diff --git a/toolkits/slack/arcade_slack/exceptions.py b/toolkits/slack/arcade_slack/exceptions.py index 106ca01e..49e40a75 100644 --- a/toolkits/slack/arcade_slack/exceptions.py +++ b/toolkits/slack/arcade_slack/exceptions.py @@ -12,3 +12,18 @@ class PaginationTimeoutError(SlackToolkitError): class ItemNotFoundError(SlackToolkitError): """Raised when an item is not found.""" + + +class UsernameNotFoundError(SlackToolkitError): + """Raised when a user is not found by the username searched""" + + def __init__(self, usernames_found: list[str]) -> None: + self.usernames_found = usernames_found + + +class ConversationNotFoundError(SlackToolkitError): + """Raised when a conversation is not found""" + + +class DirectMessageConversationNotFoundError(ConversationNotFoundError): + """Raised when a direct message conversation searched is not found""" diff --git a/toolkits/slack/arcade_slack/tools/chat.py b/toolkits/slack/arcade_slack/tools/chat.py index 3c826392..9daf5e11 100644 --- a/toolkits/slack/arcade_slack/tools/chat.py +++ b/toolkits/slack/arcade_slack/tools/chat.py @@ -9,9 +9,15 @@ from slack_sdk.errors import SlackApiError from slack_sdk.web.async_client import AsyncWebClient from arcade_slack.constants import MAX_PAGINATION_TIMEOUT_SECONDS -from arcade_slack.exceptions import ItemNotFoundError -from arcade_slack.models import ConversationType, SlackUserList -from arcade_slack.tools.users import get_user_info_by_id +from arcade_slack.exceptions import ( + ItemNotFoundError, + UsernameNotFoundError, +) +from arcade_slack.models import ( + ConversationType, + SlackUserList, +) +from arcade_slack.tools.users import get_user_info_by_id, list_users from arcade_slack.utils import ( async_paginate, convert_conversation_type_to_slack_name, @@ -21,6 +27,8 @@ from arcade_slack.utils import ( extract_conversation_metadata, format_conversations_as_csv, format_users, + get_user_by_username, + retrieve_conversations_by_user_ids, ) @@ -218,20 +226,18 @@ async def get_members_in_conversation_by_id( ], ) ) -async def get_members_in_conversation_by_name( +async def get_members_in_channel_by_name( context: ToolContext, - conversation_name: Annotated[str, "The name of the conversation to get members for"], + channel_name: Annotated[str, "The name of the channel to get members for"], limit: Annotated[Optional[int], "The maximum number of members to return."] = None, next_cursor: Annotated[Optional[str], "The cursor to use for pagination."] = None, -) -> Annotated[dict, "The conversation members' IDs and Names"]: +) -> Annotated[dict, "The channel members' IDs and Names"]: """Get the members of a conversation in Slack by the conversation's name.""" - conversation_metadata = await get_conversation_metadata_by_name( - context=context, conversation_name=conversation_name, next_cursor=next_cursor - ) + channel = await get_channel_metadata_by_name(context=context, channel_name=channel_name) return await get_members_in_conversation_by_id( # type: ignore[no-any-return] context=context, - conversation_id=conversation_metadata["id"], + conversation_id=channel["id"], limit=limit, next_cursor=next_cursor, ) @@ -433,12 +439,84 @@ async def get_messages_in_channel_by_name( 'latest_relative'. Leave all arguments with the default None to get messages without date/time filtering""" - conversation_metadata = await get_conversation_metadata_by_name( - context=context, conversation_name=channel_name - ) + channel = await get_channel_metadata_by_name(context=context, channel_name=channel_name) + return await get_messages_in_conversation_by_id( # type: ignore[no-any-return] context=context, - conversation_id=conversation_metadata["id"], + conversation_id=channel["id"], + oldest_relative=oldest_relative, + latest_relative=latest_relative, + oldest_datetime=oldest_datetime, + latest_datetime=latest_datetime, + limit=limit, + next_cursor=next_cursor, + ) + + +@tool(requires_auth=Slack(scopes=["im:history", "im:read"])) +async def get_messages_in_direct_message_conversation_by_username( + context: ToolContext, + username: Annotated[str, "The username of the user to get messages from"], + oldest_relative: Annotated[ + Optional[str], + ( + "The oldest message to include in the results, specified as a time offset from the " + "current time in the format 'DD:HH:MM'" + ), + ] = None, + latest_relative: Annotated[ + Optional[str], + ( + "The latest message to include in the results, specified as a time offset from the " + "current time in the format 'DD:HH:MM'" + ), + ] = None, + oldest_datetime: Annotated[ + Optional[str], + ( + "The oldest message to include in the results, specified as a datetime object in the " + "format 'YYYY-MM-DD HH:MM:SS'" + ), + ] = None, + latest_datetime: Annotated[ + Optional[str], + ( + "The latest message to include in the results, specified as a datetime object in the " + "format 'YYYY-MM-DD HH:MM:SS'" + ), + ] = None, + limit: Annotated[Optional[int], "The maximum number of messages to return."] = None, + next_cursor: Annotated[Optional[str], "The cursor to use for pagination."] = None, +) -> Annotated[ + dict, + ( + "The messages in a direct message conversation and next cursor for paginating results " + "(when there are additional messages to retrieve)." + ), +]: + """Get the messages in a direct conversation by the user's name. + + To filter messages by an absolute datetime, use 'oldest_datetime' and/or 'latest_datetime'. If + only 'oldest_datetime' is provided, it will return messages from the oldest_datetime to the + current time. If only 'latest_datetime' is provided, it will return messages since the + beginning of the conversation to the latest_datetime. + + To filter messages by a relative datetime (e.g. 3 days ago, 1 hour ago, etc.), use + 'oldest_relative' and/or 'latest_relative'. If only 'oldest_relative' is provided, it will + return messages from the oldest_relative to the current time. If only 'latest_relative' is + provided, it will return messages from the current time to the latest_relative. + + Do not provide both 'oldest_datetime' and 'oldest_relative' or both 'latest_datetime' and + 'latest_relative'. + + Leave all arguments with the default None to get messages without date/time filtering""" + direct_conversation = await get_direct_message_conversation_metadata_by_username( + context=context, username=username + ) + + return await get_messages_in_conversation_by_id( # type: ignore[no-any-return] + context=context, + conversation_id=direct_conversation["id"], oldest_relative=oldest_relative, latest_relative=latest_relative, oldest_datetime=oldest_datetime, @@ -490,39 +568,33 @@ async def get_conversation_metadata_by_id( return dict(**extract_conversation_metadata(response["channel"])) -@tool( - requires_auth=Slack( - scopes=["channels:read", "groups:read", "im:read", "mpim:read"], - ) -) -async def get_conversation_metadata_by_name( +@tool(requires_auth=Slack(scopes=["channels:read"])) +async def get_channel_metadata_by_name( context: ToolContext, - conversation_name: Annotated[str, "The name of the conversation to get metadata for"], + channel_name: Annotated[str, "The name of the channel to get metadata for"], next_cursor: Annotated[ Optional[str], "The cursor to use for pagination, if continuing from a previous search.", ] = None, -) -> Annotated[dict, "The conversation metadata"]: - """Get the metadata of a conversation in Slack searching by its name.""" - conversation_names: list[str] = [] +) -> Annotated[dict, "The channel metadata"]: + """Get the metadata of a channel in Slack searching by its name.""" + channel_names: list[str] = [] - async def find_conversation() -> dict: - nonlocal conversation_names, conversation_name, next_cursor + async def find_channel() -> dict: + nonlocal channel_names, channel_name, next_cursor should_continue = True while should_continue: response = await list_conversations_metadata(context, next_cursor=next_cursor) - next_cursor = response["next_cursor"] + next_cursor = response.get("next_cursor") - for conversation in response["conversations"]: - response_conversation_name = ( - "" - if not isinstance(conversation.get("name"), str) - else conversation["name"].lower() + for channel in response["conversations"]: + response_channel_name = ( + "" if not isinstance(channel.get("name"), str) else channel["name"].lower() ) - if response_conversation_name == conversation_name.lower(): - return conversation # type: ignore[no-any-return] - conversation_names.append(conversation["name"]) + if response_channel_name == channel_name.lower(): + return channel # type: ignore[no-any-return] + channel_names.append(channel["name"]) if not next_cursor: should_continue = False @@ -530,31 +602,80 @@ async def get_conversation_metadata_by_name( raise ItemNotFoundError() try: - return await asyncio.wait_for(find_conversation(), timeout=MAX_PAGINATION_TIMEOUT_SECONDS) + return await asyncio.wait_for(find_channel(), timeout=MAX_PAGINATION_TIMEOUT_SECONDS) except ItemNotFoundError: raise RetryableToolError( - "Conversation not found", - developer_message=f"Conversation with name '{conversation_name}' not found.", - additional_prompt_content=f"Available conversation names: {conversation_names}", + "Channel not found", + developer_message=f"Channel with name '{channel_name}' not found.", + additional_prompt_content=f"Available channel names: {channel_names}", retry_after_ms=500, ) except TimeoutError: raise RetryableToolError( - "Conversation not found, search timed out.", + "Channel not found, search timed out.", developer_message=( - f"Conversation with name '{conversation_name}' not found. " + f"Channel with name '{channel_name}' not found. " f"Search timed out after {MAX_PAGINATION_TIMEOUT_SECONDS} seconds." ), additional_prompt_content=( - f"Other conversation names found are: {conversation_names}. " + f"Other channel names found are: {channel_names}. " "The list is potentially non-exhaustive, since the search process timed out. " - f"Use the '{list_conversations_metadata.__name__}' tool to get a comprehensive " - "list of conversations." + f"Use the '{list_conversations_metadata.__tool_name__}' tool to get" + "a comprehensive list of channels." ), retry_after_ms=500, ) +@tool(requires_auth=Slack(scopes=["im:read"])) +async def get_direct_message_conversation_metadata_by_username( + context: ToolContext, + username: Annotated[str, "The username of the user/person to get messages with"], + next_cursor: Annotated[ + Optional[str], + "The cursor to use for pagination, if continuing from a previous search.", + ] = None, +) -> Annotated[ + Optional[dict], + "The direct message conversation metadata.", +]: + """Get the metadata of a direct message conversation in Slack by the username.""" + try: + token = ( + context.authorization.token + if context.authorization and context.authorization.token + else "" + ) + slack_client = AsyncWebClient(token=token) + + current_user, list_users_response = await asyncio.gather( + slack_client.auth_test(), list_users(context) + ) + + other_user = get_user_by_username(username, list_users_response["users"]) + + conversations_found = await retrieve_conversations_by_user_ids( + list_conversations_func=list_conversations_metadata, + get_members_in_conversation_func=get_members_in_conversation_by_id, + context=context, + conversation_types=[ConversationType.DIRECT_MESSAGE], + user_ids=[current_user["user_id"], other_user["id"]], + exact_match=True, + limit=1, + next_cursor=next_cursor, + ) + + return None if not conversations_found else conversations_found[0] + + except UsernameNotFoundError as e: + raise RetryableToolError( + f"Username '{username}' not found", + developer_message=f"User with username '{username}' not found.", + additional_prompt_content=f"Available users: {e.usernames_found}", + retry_after_ms=500, + ) + + @tool( requires_auth=Slack( scopes=["channels:read", "groups:read", "im:read", "mpim:read"], diff --git a/toolkits/slack/arcade_slack/utils.py b/toolkits/slack/arcade_slack/utils.py index 6b82b711..159cf9bc 100644 --- a/toolkits/slack/arcade_slack/utils.py +++ b/toolkits/slack/arcade_slack/utils.py @@ -2,11 +2,15 @@ import asyncio from datetime import datetime, timezone from typing import Any, Callable, Optional +from arcade.sdk import ToolContext from arcade.sdk.errors import RetryableToolError from arcade_slack.constants import MAX_PAGINATION_SIZE_LIMIT, MAX_PAGINATION_TIMEOUT_SECONDS from arcade_slack.custom_types import SlackPaginationNextCursor -from arcade_slack.exceptions import PaginationTimeoutError +from arcade_slack.exceptions import ( + PaginationTimeoutError, + UsernameNotFoundError, +) from arcade_slack.models import ( BasicUserInfo, ConversationMetadata, @@ -90,6 +94,18 @@ def get_slack_conversation_type_as_str(channel: SlackConversation) -> str: raise ValueError(f"Invalid conversation type in channel {channel.get('name')}") +def get_user_by_username(username: str, users_list: list[dict]) -> SlackUser: + usernames_found = [] + for user in users_list: + if isinstance(user.get("name"), str): + usernames_found.append(user["name"]) + username_found = user.get("name") or "" + if username.lower() == username_found.lower(): + return SlackUser(**user) + + raise UsernameNotFoundError(usernames_found) + + def convert_conversation_type_to_slack_name( conversation_type: ConversationType, ) -> ConversationTypeSlackName: @@ -174,6 +190,93 @@ def extract_basic_user_info(user_info: SlackUser) -> BasicUserInfo: ) +async def associate_members_of_multiple_conversations( + get_members_in_conversation_func: Callable, + conversations: list[dict], + context: ToolContext, +) -> list[dict]: + """Associate members to each conversation, returning the updated list.""" + return await asyncio.gather(*[ + associate_members_of_conversation(get_members_in_conversation_func, context, conv) + for conv in conversations + ]) + + +async def associate_members_of_conversation( + get_members_in_conversation_func: Callable, + context: ToolContext, + conversation: dict, +) -> dict: + response = await get_members_in_conversation_func(context, conversation["id"]) + conversation["members"] = response["members"] + return conversation + + +async def retrieve_conversations_by_user_ids( + list_conversations_func: Callable, + get_members_in_conversation_func: Callable, + context: ToolContext, + conversation_types: list[ConversationType], + user_ids: list[str], + exact_match: bool = False, + limit: Optional[int] = None, + next_cursor: Optional[str] = None, +) -> list[dict]: + """ + Retrieve conversations filtered by the given user IDs. Includes pagination support + and optionally limits the number of returned conversations. + """ + conversations_found: list[dict] = [] + + response = await list_conversations_func( + context=context, + conversation_types=conversation_types, + next_cursor=next_cursor, + ) + + # Associate members to each conversation + conversations_with_members = await associate_members_of_multiple_conversations( + get_members_in_conversation_func, response["conversations"], context + ) + + conversations_found.extend( + filter_conversations_by_user_ids(conversations_with_members, user_ids, exact_match) + ) + + return conversations_found[:limit] + + +def filter_conversations_by_user_ids( + conversations: list[dict], + user_ids: list[str], + exact_match: bool = False, +) -> list[dict]: + """ + Filter conversations by the members' user IDs. + + Args: + conversations: The list of conversations to filter. + user_ids: The user IDs to filter conversations for. + exact_match: Whether to match the exact number of members in the conversations. + + Returns: + The list of conversations found. + """ + matches = [] + for conversation in conversations: + member_ids = [member["id"] for member in conversation["members"]] + if exact_match: + same_length = len(user_ids) == len(member_ids) + has_all_members = all(user_id in member_ids for user_id in user_ids) + if same_length and has_all_members: + matches.append(conversation) + else: + if all(user_id in member_ids for user_id in user_ids): + matches.append(conversation) + + return matches + + def is_user_a_bot(user: SlackUser) -> bool: """Check if a Slack user represents a bot. diff --git a/toolkits/slack/conftest.py b/toolkits/slack/conftest.py index fb7c1c32..acda0bea 100644 --- a/toolkits/slack/conftest.py +++ b/toolkits/slack/conftest.py @@ -6,3 +6,15 @@ from arcade.sdk import ToolAuthorizationContext, ToolContext def mock_context(): mock_auth = ToolAuthorizationContext(token="fake-token") # noqa: S106 return ToolContext(authorization=mock_auth) + + +@pytest.fixture +def mock_chat_slack_client(mocker): + mock_client = mocker.patch("arcade_slack.tools.chat.AsyncWebClient", autospec=True) + return mock_client.return_value + + +@pytest.fixture +def mock_users_slack_client(mocker): + mock_client = mocker.patch("arcade_slack.tools.users.AsyncWebClient", autospec=True) + return mock_client.return_value diff --git a/toolkits/slack/evals/eval_chat.py b/toolkits/slack/evals/eval_chat.py index e83f1aeb..55b39e91 100644 --- a/toolkits/slack/evals/eval_chat.py +++ b/toolkits/slack/evals/eval_chat.py @@ -15,12 +15,14 @@ from arcade.sdk.eval import ( import arcade_slack from arcade_slack.critics import RelativeTimeBinaryCritic from arcade_slack.tools.chat import ( + get_channel_metadata_by_name, get_conversation_metadata_by_id, - get_conversation_metadata_by_name, + get_direct_message_conversation_metadata_by_username, + get_members_in_channel_by_name, get_members_in_conversation_by_id, - get_members_in_conversation_by_name, get_messages_in_channel_by_name, get_messages_in_conversation_by_id, + get_messages_in_direct_message_conversation_by_username, list_conversations_metadata, list_direct_message_conversations_metadata, list_group_direct_message_conversations_metadata, @@ -433,19 +435,19 @@ def get_conversations_metadata_eval_suite() -> EvalSuite: """Create an evaluation suite for tools getting conversations metadata.""" suite = EvalSuite( name="Slack Tools Evaluation", - system_message="You are an AI assistant that can interact with Slack to send messages and get information from conversations, users, etc.", + system_message="You are an AI assistant that can interact with Slack to get information from conversations, users, etc.", catalog=catalog, rubric=rubric, ) suite.add_case( - name="Get conversation metadata by name", + name="Get channel metadata by name", user_message="Get the metadata of the #general channel", expected_tool_calls=[ ExpectedToolCall( - func=get_conversation_metadata_by_name, + func=get_channel_metadata_by_name, args={ - "conversation_name": "general", + "channel_name": "general", }, ), ], @@ -470,6 +472,26 @@ def get_conversations_metadata_eval_suite() -> EvalSuite: ], ) + get_metadata_by_username_user_messages = [ + "get the metadata of the direct message conversation with the user 'jane.doe'" + "get data about my private conversation with the user 'jane.doe'", + "get data about my IM conversation with the 'jane.doe'", + ] + + for i, user_message in enumerate(get_metadata_by_username_user_messages): + suite.add_case( + name=f"Get direct message conversation metadata by username {i}", + user_message=user_message, + expected_tool_calls=[ + ExpectedToolCall( + func=get_direct_message_conversation_metadata_by_username, + args={ + "username": "jane.doe", + }, + ), + ], + ) + return suite @@ -496,13 +518,13 @@ def get_conversations_members_eval_suite() -> EvalSuite: for user_message in user_messages: suite.add_case( - name=f"Get conversation members by name: {user_message}", + name=f"Get channel members by name: {user_message}", user_message=user_message, expected_tool_calls=[ ExpectedToolCall( - func=get_members_in_conversation_by_name, + func=get_members_in_channel_by_name, args={ - "conversation_name": "general", + "channel_name": "general", }, ), ], @@ -531,8 +553,8 @@ def get_conversations_members_eval_suite() -> EvalSuite: @tool_eval() -def get_conversation_history_eval_suite() -> EvalSuite: - """Create an evaluation suite for tools getting conversations history.""" +def get_messages_in_channel_eval_suite() -> EvalSuite: + """Create an evaluation suite for tools getting messages in channels.""" suite = EvalSuite( name="Slack Chat Tools Evaluation", system_message="You are an AI assistant that can interact with Slack to send messages and get information from conversations, users, etc.", @@ -540,16 +562,16 @@ def get_conversation_history_eval_suite() -> EvalSuite: rubric=rubric, ) - no_arguments_user_messages_by_conversation_name = [ - "Get the history of the #general channel", - "Get the history of the general channel", + no_arguments_user_messages_by_channel_name = [ + "what are the latest messages in the #general channel", + "show me the messages in the general channel", "list the messages in the #general channel", "list the messages in the general channel", ] - for user_message in no_arguments_user_messages_by_conversation_name: + for i, user_message in enumerate(no_arguments_user_messages_by_channel_name): suite.add_case( - name=f"Get conversation history by name: '{user_message}'", + name=f"Get messages in conversation by name {i}: '{user_message}'", user_message=user_message, expected_tool_calls=[ ExpectedToolCall( @@ -985,3 +1007,82 @@ def get_conversation_history_eval_suite() -> EvalSuite: ) return suite + + +@tool_eval() +def get_messages_in_direct_message_eval_suite() -> EvalSuite: + """Create an evaluation suite for tools getting messages in direct messages.""" + suite = EvalSuite( + name="Slack Chat Tools Evaluation", + system_message="You are an AI assistant that can interact with Slack to send messages and get information from conversations, users, etc.", + catalog=catalog, + rubric=rubric, + ) + + no_arguments_user_messages_by_username = [ + "what are the latest messages I exchanged with jane.doe", + "show my messages with jane.doe on Slack", + "list the messages I exchanged with jane.doe", + "list the message history with jane.doe", + ] + + for i, user_message in enumerate(no_arguments_user_messages_by_username): + suite.add_case( + name=f"{user_message} [{i}]", + user_message=user_message, + expected_tool_calls=[ + ExpectedToolCall( + func=get_messages_in_direct_message_conversation_by_username, + args={ + "username": "jane.doe", + }, + ), + ], + critics=[ + BinaryCritic(critic_field="username", weight=1.0), + ], + ) + + suite.add_case( + name="get messages in direct conversation by username (on a specific date)", + user_message="get the messages I exchanged with jane.doe on 2025-01-31", + expected_tool_calls=[ + ExpectedToolCall( + func=get_messages_in_direct_message_conversation_by_username, + args={ + "username": "jane.doe", + "oldest_datetime": "2025-01-31 00:00:00", + "latest_datetime": "2025-01-31 23:59:59", + }, + ), + ], + critics=[ + BinaryCritic(critic_field="username", weight=1 / 3), + DatetimeCritic( + critic_field="oldest_datetime", weight=1 / 3, max_difference=timedelta(minutes=2) + ), + DatetimeCritic( + critic_field="latest_datetime", weight=1 / 3, max_difference=timedelta(minutes=2) + ), + ], + ) + + suite.add_case( + name="Get conversation history oldest relative by username (2 days ago)", + user_message="Get the messages I exchanged with jane.doe starting 2 days ago", + expected_tool_calls=[ + ExpectedToolCall( + func=get_messages_in_direct_message_conversation_by_username, + args={ + "username": "jane.doe", + "oldest_relative": "02:00:00", + }, + ), + ], + critics=[ + BinaryCritic(critic_field="username", weight=0.5), + RelativeTimeBinaryCritic(critic_field="oldest_relative", weight=0.5), + ], + ) + + return suite diff --git a/toolkits/slack/evals/eval_slack_chat.py b/toolkits/slack/evals/eval_slack_chat.py deleted file mode 100644 index 0cfc7a65..00000000 --- a/toolkits/slack/evals/eval_slack_chat.py +++ /dev/null @@ -1,1020 +0,0 @@ -import json -from datetime import timedelta -from typing import Any - -from arcade.sdk import ToolCatalog -from arcade.sdk.eval import ( - BinaryCritic, - DatetimeCritic, - EvalRubric, - EvalSuite, - ExpectedToolCall, - SimilarityCritic, - tool_eval, -) - -import arcade_slack -from arcade_slack.tools.chat import ( - get_conversation_metadata_by_id, - get_conversation_metadata_by_name, - get_members_in_conversation_by_id, - get_members_in_conversation_by_name, - get_messages_in_channel_by_name, - get_messages_in_conversation_by_id, - list_conversations_metadata, - list_direct_message_conversations_metadata, - list_group_direct_message_conversations_metadata, - list_private_channels_metadata, - list_public_channels_metadata, - send_dm_to_user, - send_message_to_channel, -) - - -class RelativeTimeBinaryCritic(BinaryCritic): - def evaluate(self, expected: Any, actual: Any) -> dict[str, float | bool]: - """ - Evaluates whether the expected and actual relative time strings are equivalent after casting. - - Args: - expected: The expected value. - actual: The actual value to compare, cast to the type of expected. - - Returns: - dict: A dictionary containing the match status and score. - """ - # Cast actual to the type of expected - try: - actual_casted = self.cast_actual(expected, actual) - # TODO log or something better here - except TypeError: - actual_casted = actual - - expected_parts = tuple(map(int, expected.split(":"))) - actual_parts = tuple(map(int, actual_casted.split(":"))) - - if len(expected_parts) != 3 or len(actual_parts) != 3: - return {"match": False, "score": 0.0} - - exp_days, exp_hours, exp_minutes = expected_parts - act_days, act_hours, act_minutes = actual_parts - - match = exp_days == act_days and exp_hours == act_hours and exp_minutes == act_minutes - return {"match": match, "score": self.weight if match else 0.0} - - -# Evaluation rubric -rubric = EvalRubric( - fail_threshold=0.8, - warn_threshold=0.9, -) - - -catalog = ToolCatalog() -# Register the Slack tools -catalog.add_module(arcade_slack) - - -@tool_eval() -def send_message_eval_suite() -> EvalSuite: - """Create an evaluation suite for Slack messaging tools.""" - suite = EvalSuite( - name="Slack Messaging Tools Evaluation", - system_message="You are an AI assistant that can send direct messages and post messages to channels in Slack using the provided tools.", - catalog=catalog, - rubric=rubric, - ) - - # Send DM to User Scenarios - suite.add_case( - name="Send DM to user with clear username", - user_message="Send a direct message to johndoe saying 'Hello, can we meet at 3 PM?'", - expected_tool_calls=[ - ExpectedToolCall( - func=send_dm_to_user, - args={ - "user_name": "johndoe", - "message": "Hello, can we meet at 3 PM?", - }, - ) - ], - critics=[ - BinaryCritic(critic_field="user_name", weight=0.5), - SimilarityCritic(critic_field="message", weight=0.5, similarity_threshold=0.9), - ], - ) - - suite.add_case( - name="Send DM with ambiguous username", - user_message="ask him for an update on the project", - expected_tool_calls=[ - ExpectedToolCall( - func=send_dm_to_user, - args={ - "user_name": "john", - "message": "Hi John, could you please provide an update on the Acme project?", - }, - ) - ], - critics=[ - BinaryCritic(critic_field="user_name", weight=0.75), - SimilarityCritic(critic_field="message", weight=0.25, similarity_threshold=0.6), - ], - additional_messages=[ - {"role": "user", "content": "Message John about the Acme project deadline"}, - { - "role": "assistant", - "content": "", - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": { - "name": "Slack_ListUsers", - "arguments": '{"exclude_bots":true}', - }, - } - ], - }, - { - "role": "tool", - "content": json.dumps({ - "users": [ - { - "display_name": "john", - "email": "john@randomtech.com", - "id": "abc123", - "is_bot": False, - "name": "john", - "real_name": "John Doe", - "timezone": "America/Los_Angeles", - }, - { - "display_name": "jack", - "email": "jack@randomtech.com", - "id": "def456", - "is_bot": False, - "name": "jack", - "real_name": "Jack Doe", - "timezone": "America/Los_Angeles", - }, - ] - }), - "tool_call_id": "call_1", - "name": "Slack_ListUsers", - }, - { - "role": "assistant", - "content": "What would you like to include in the message to John about the Acme project deadline?", - }, - ], - ) - - suite.add_case( - name="Send DM with username in different format", - user_message="yes, send it", - expected_tool_calls=[ - ExpectedToolCall( - func=send_dm_to_user, - args={ - "user_name": "jane.doe", - "message": "Hi Jane, I need to reschedule our meeting. When are you available?", - }, - ) - ], - critics=[ - BinaryCritic(critic_field="user_name", weight=0.75), - SimilarityCritic(critic_field="message", weight=0.25, similarity_threshold=0.6), - ], - additional_messages=[ - {"role": "user", "content": "Message Jane.Doe asking to reschedule our meeting"}, - { - "role": "assistant", - "content": "", - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": { - "name": "Slack_ListUsers", - "arguments": '{"exclude_bots":true}', - }, - } - ], - }, - { - "role": "tool", - "content": json.dumps({ - "users": [ - { - "display_name": "jane.doe", - "email": "jane@randomtech.com", - "id": "abc123", - "is_bot": False, - "name": "jane.doe", - "real_name": "Jane Doe", - "timezone": "America/Los_Angeles", - }, - { - "display_name": "jack", - "email": "jack@randomtech.com", - "id": "def456", - "is_bot": False, - "name": "jack", - "real_name": "Jack Doe", - "timezone": "America/Los_Angeles", - }, - ] - }), - "tool_call_id": "call_1", - "name": "Slack_ListUsers", - }, - { - "role": "assistant", - "content": "I found a user with the name 'jane.doe'. Would you like to send a message to them?", - }, - ], - ) - - # Send Message to Channel Scenarios - suite.add_case( - name="Send message to channel with clear name", - user_message="Post 'The new feature is now live!' in the #announcements channel", - expected_tool_calls=[ - ExpectedToolCall( - func=send_message_to_channel, - args={ - "channel_name": "announcements", - "message": "The new feature is now live!", - }, - ) - ], - critics=[ - BinaryCritic(critic_field="channel_name", weight=0.5), - SimilarityCritic(critic_field="message", weight=0.5), - ], - ) - - suite.add_case( - name="Send message to channel with ambiguous name", - user_message="Inform the team in the general channel about the upcoming maintenance", - expected_tool_calls=[ - ExpectedToolCall( - func=send_message_to_channel, - args={ - "channel_name": "general", - "message": "Attention team: There will be upcoming maintenance. Please save your work and expect some downtime.", - }, - ) - ], - critics=[ - SimilarityCritic(critic_field="channel_name", weight=0.8), - SimilarityCritic(critic_field="message", weight=0.2, similarity_threshold=0.6), - ], - ) - - # Adversarial Scenarios - suite.add_case( - name="Ambiguous between DM and channel message", - user_message="general", - expected_tool_calls=[ - ExpectedToolCall( - func=send_message_to_channel, - args={ - "channel_name": "general", - "message": "Great job on the presentation!", - }, - ) - ], - critics=[ - SimilarityCritic(critic_field="channel_name", weight=0.4), - SimilarityCritic(critic_field="message", weight=0.6), - ], - additional_messages=[ - {"role": "user", "content": "Send 'Great job on the presentation!' to the team"}, - { - "role": "assistant", - "content": "", - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": { - "name": "Slack_ListPublicChannelsMetadata", - "arguments": '{"limit":20}', - }, - } - ], - }, - { - "role": "tool", - "content": json.dumps({ - "conversations": [ - { - "conversation_type": "public_channel", - "id": "channel1", - "is_archived": False, - "is_member": True, - "is_private": False, - "name": "random", - "num_members": 999, - "purpose": "Random stuff", - }, - { - "conversation_type": "public_channel", - "id": "channel2", - "is_archived": False, - "is_member": True, - "is_private": False, - "name": "general", - "num_members": 999, - "purpose": "Just a general channel", - }, - ], - "next_cursor": "", - }), - "tool_call_id": "call_1", - "name": "Slack_ListPublicChannelsMetadata", - }, - { - "role": "assistant", - "content": 'To send the message "Great job on the presentation!" to the team, please let me know which Slack channel you\'d like to use:\n\n1. #random\n2. #general\n\nPlease let me know your choice!', - }, - ], - ) - - # Multiple recipients in DM request - suite.add_case( - name="Multiple recipients in DM request", - user_message="Send DMs to the users 'alice' and 'bob' about pushing the meeting tomorrow. I have to much work to do.", - expected_tool_calls=[ - ExpectedToolCall( - func=send_dm_to_user, - args={ - "user_name": "alice", - "message": "Hi Alice, about our meeting tomorrow, let's reschedule? I am swamped with work.", - }, - ), - ExpectedToolCall( - func=send_dm_to_user, - args={ - "user_name": "bob", - "message": "Hi Bob, about our meeting tomorrow, let's reschedule? I am swamped with work.", - }, - ), - ], - critics=[ - SimilarityCritic(critic_field="user_name", weight=0.75), - SimilarityCritic(critic_field="message", weight=0.25, similarity_threshold=0.5), - ], - ) - - suite.add_case( - name="Channel name similar to username", - user_message="Post 'sounds great!' in john-project channel", - expected_tool_calls=[ - ExpectedToolCall( - func=send_message_to_channel, - args={ - "channel_name": "john-project", - "message": "Sounds great!", - }, - ) - ], - critics=[ - BinaryCritic(critic_field="channel_name", weight=0.5), - SimilarityCritic(critic_field="message", weight=0.5), - ], - ) - - return suite - - -@tool_eval() -def list_conversations_eval_suite() -> EvalSuite: - """Create an evaluation suite for tools listing conversations.""" - suite = EvalSuite( - name="Slack Messaging Tools Evaluation", - system_message="You are an AI assistant that can interact with Slack to send messages and get information from conversations, users, etc.", - catalog=catalog, - rubric=rubric, - ) - - cases = [ - ( - "List my conversations", - "List all conversations I am a member of", - list_conversations_metadata, - ), - ( - "List public channels", - "List all public channels", - list_public_channels_metadata, - ), - ( - "List private channels", - "List all private channels", - list_private_channels_metadata, - ), - ( - "List group direct message channels", - "List all group direct message channels", - list_group_direct_message_conversations_metadata, - ), - ( - "List individual direct message channels", - "List all individual direct message channels", - list_direct_message_conversations_metadata, - ), - ( - "List direct message channels", - "List all direct message channels", - list_direct_message_conversations_metadata, - ), - ( - "List public and private channels", - "List public and private channels I am a member of", - list_public_channels_metadata, - list_private_channels_metadata, - ), - ( - "List public channels and direct message conversations", - "List public channels and direct message conversations I am a member of", - list_public_channels_metadata, - list_direct_message_conversations_metadata, - ), - ] - - for name, user_message, *expect_called_tool_functions in cases: - suite.add_case( - name=name, - user_message=user_message, - expected_tool_calls=[ - ExpectedToolCall( - func=tool_function, - args={}, - ) - for tool_function in expect_called_tool_functions - ], - ) - - return suite - - -@tool_eval() -def get_conversations_metadata_eval_suite() -> EvalSuite: - """Create an evaluation suite for tools getting conversations metadata.""" - suite = EvalSuite( - name="Slack Tools Evaluation", - system_message="You are an AI assistant that can interact with Slack to send messages and get information from conversations, users, etc.", - catalog=catalog, - rubric=rubric, - ) - - suite.add_case( - name="Get conversation metadata by name", - user_message="Get the metadata of the #general channel", - expected_tool_calls=[ - ExpectedToolCall( - func=get_conversation_metadata_by_name, - args={ - "conversation_name": "general", - }, - ), - ], - critics=[ - BinaryCritic(critic_field="conversation_name", weight=1.0), - ], - ) - - suite.add_case( - name="Get conversation metadata by id", - user_message="Get the metadata of the conversation with id '1234567890'", - expected_tool_calls=[ - ExpectedToolCall( - func=get_conversation_metadata_by_id, - args={ - "conversation_id": "1234567890", - }, - ), - ], - critics=[ - BinaryCritic(critic_field="conversation_id", weight=1.0), - ], - ) - - return suite - - -@tool_eval() -def get_conversations_members_eval_suite() -> EvalSuite: - """Create an evaluation suite for tools getting conversations members.""" - suite = EvalSuite( - name="Slack Tools Evaluation", - system_message="You are an AI assistant that can interact with Slack to send messages and get information from conversations, users, etc.", - catalog=catalog, - rubric=rubric, - ) - - user_messages = [ - "Get the members of the #general channel", - "Get the members of the general channel", - "Get a list of people in the #general channel", - "Get a list of people in the general channel", - "Show me who's in the #general channel", - "Show me who's in the general channel", - "Who is in the #general channel?", - "Who is in the general channel?", - ] - - for user_message in user_messages: - suite.add_case( - name=f"Get conversation members by name: {user_message}", - user_message=user_message, - expected_tool_calls=[ - ExpectedToolCall( - func=get_members_in_conversation_by_name, - args={ - "conversation_name": "general", - }, - ), - ], - critics=[ - BinaryCritic(critic_field="conversation_name", weight=1.0), - ], - ) - - suite.add_case( - name="Get conversation members by id", - user_message="Get the members of the conversation with id '1234567890'", - expected_tool_calls=[ - ExpectedToolCall( - func=get_members_in_conversation_by_id, - args={ - "conversation_id": "1234567890", - }, - ), - ], - critics=[ - BinaryCritic(critic_field="conversation_id", weight=1.0), - ], - ) - - return suite - - -@tool_eval() -def get_conversation_history_eval_suite() -> EvalSuite: - """Create an evaluation suite for tools getting conversations history.""" - suite = EvalSuite( - name="Slack Tools Evaluation", - system_message="You are an AI assistant that can interact with Slack to send messages and get information from conversations, users, etc.", - catalog=catalog, - rubric=rubric, - ) - - no_arguments_user_messages_by_conversation_name = [ - "Get the history of the #general channel", - "Get the history of the general channel", - "list the messages in the #general channel", - "list the messages in the general channel", - ] - - for user_message in no_arguments_user_messages_by_conversation_name: - suite.add_case( - name=f"Get conversation history by name: '{user_message}'", - user_message=user_message, - expected_tool_calls=[ - ExpectedToolCall( - func=get_messages_in_channel_by_name, - args={ - "conversation_name": "general", - }, - ), - ], - critics=[ - BinaryCritic(critic_field="conversation_name", weight=1.0), - ], - ) - - no_arguments_user_messages_by_conversation_id = [ - "Get the history of the conversation with id '1234567890'", - "Get the history of the conversation with id '1234567890'", - "list the messages in the conversation with id '1234567890'", - "list the messages in the conversation with id '1234567890'", - ] - - for user_message in no_arguments_user_messages_by_conversation_id: - suite.add_case( - name=f"Get conversation history by id: '{user_message}'", - user_message=user_message, - expected_tool_calls=[ - ExpectedToolCall( - func=get_messages_in_conversation_by_id, - args={ - "conversation_id": "1234567890", - }, - ), - ], - critics=[ - BinaryCritic(critic_field="conversation_id", weight=1.0), - ], - ) - - suite.add_case( - name="Get conversation history with limit by name", - user_message="Get the last 10 messages in the #general channel", - expected_tool_calls=[ - ExpectedToolCall( - func=get_messages_in_channel_by_name, - args={ - "conversation_name": "general", - "limit": 10, - }, - ), - ], - critics=[ - BinaryCritic(critic_field="conversation_name", weight=1.0), - ], - ) - - suite.add_case( - name="Get conversation history with limit by id", - user_message="Get the last 25 messages in the conversation with id '1234567890'", - expected_tool_calls=[ - ExpectedToolCall( - func=get_messages_in_conversation_by_id, - args={ - "conversation_id": "1234567890", - "limit": 25, - }, - ), - ], - critics=[ - BinaryCritic(critic_field="conversation_id", weight=1.0), - ], - ) - - # Relative time eval cases by id - - suite.add_case( - name="Get conversation history oldest relative by id (2 days ago)", - user_message="Get the messages in the conversation with id '1234567890' starting 2 days ago", - expected_tool_calls=[ - ExpectedToolCall( - func=get_messages_in_conversation_by_id, - args={ - "conversation_id": "1234567890", - "oldest_relative": "02:00:00", - }, - ), - ], - critics=[ - BinaryCritic(critic_field="conversation_id", weight=0.5), - RelativeTimeBinaryCritic(critic_field="oldest_relative", weight=0.5), - ], - ) - - suite.add_case( - name="Get conversation history oldest and latest relative by id", - user_message="Get the messages in the conversation with id '1234567890' from 2 days ago to 3 hours ago", - expected_tool_calls=[ - ExpectedToolCall( - func=get_messages_in_conversation_by_id, - args={ - "conversation_id": "1234567890", - "oldest_relative": "02:00:00", - "latest_relative": "00:03:00", - }, - ), - ], - critics=[ - BinaryCritic(critic_field="conversation_id", weight=1 / 3), - RelativeTimeBinaryCritic(critic_field="oldest_relative", weight=1 / 3), - RelativeTimeBinaryCritic(critic_field="latest_relative", weight=1 / 3), - ], - ) - - suite.add_case( - name="Get conversation history oldest relative by id (1 week ago)", - user_message="Get the messages in the conversation with id '1234567890' starting 1 week ago", - expected_tool_calls=[ - ExpectedToolCall( - func=get_messages_in_conversation_by_id, - args={ - "conversation_id": "1234567890", - "oldest_relative": "07:00:00", - }, - ), - ], - critics=[ - BinaryCritic(critic_field="conversation_id", weight=0.5), - RelativeTimeBinaryCritic(critic_field="oldest_relative", weight=0.5), - ], - ) - - suite.add_case( - name="Get conversation history oldest relative by id (yesterday)", - user_message="Get the messages in the conversation with id '1234567890' from yesterday", - expected_tool_calls=[ - ExpectedToolCall( - func=get_messages_in_conversation_by_id, - args={ - "conversation_id": "1234567890", - "oldest_relative": "01:00:00", - }, - ), - ], - critics=[ - BinaryCritic(critic_field="conversation_id", weight=0.5), - RelativeTimeBinaryCritic(critic_field="oldest_relative", weight=0.5), - ], - ) - - # Relative time eval cases by name - - suite.add_case( - name="Get conversation history oldest relative by name (2 days ago)", - user_message="Get the messages in the #general channel starting 2 days ago", - expected_tool_calls=[ - ExpectedToolCall( - func=get_messages_in_channel_by_name, - args={ - "conversation_name": "general", - "oldest_relative": "02:00:00", - }, - ), - ], - critics=[ - BinaryCritic(critic_field="conversation_name", weight=0.5), - RelativeTimeBinaryCritic(critic_field="oldest_relative", weight=0.5), - ], - ) - - suite.add_case( - name="Get conversation history oldest and latest relative by name", - user_message="Get the messages in the #general channel from 2 days ago to 3 hours ago", - expected_tool_calls=[ - ExpectedToolCall( - func=get_messages_in_channel_by_name, - args={ - "conversation_name": "general", - "oldest_relative": "02:00:00", - "latest_relative": "00:03:00", - }, - ), - ], - critics=[ - BinaryCritic(critic_field="conversation_name", weight=1 / 3), - RelativeTimeBinaryCritic(critic_field="oldest_relative", weight=1 / 3), - RelativeTimeBinaryCritic(critic_field="latest_relative", weight=1 / 3), - ], - ) - - suite.add_case( - name="Get conversation history oldest relative by name (yesterday)", - user_message="Get the messages in the #general channel from yesterday", - expected_tool_calls=[ - ExpectedToolCall( - func=get_messages_in_channel_by_name, - args={ - "conversation_name": "general", - "oldest_relative": "01:00:00", - }, - ), - ], - critics=[ - BinaryCritic(critic_field="conversation_name", weight=0.5), - RelativeTimeBinaryCritic(critic_field="oldest_relative", weight=0.5), - ], - ) - - suite.add_case( - name="Get conversation history oldest relative by name (last week)", - user_message="Get the messages in the #general channel from last week", - expected_tool_calls=[ - ExpectedToolCall( - func=get_messages_in_channel_by_name, - args={ - "conversation_name": "general", - "oldest_relative": "07:00:00", - }, - ), - ], - critics=[ - BinaryCritic(critic_field="conversation_name", weight=0.5), - RelativeTimeBinaryCritic(critic_field="oldest_relative", weight=0.5), - ], - ) - - # Absolute time eval cases by id - - suite.add_case( - name="Get conversation history oldest absolute by id (on a specific date)", - user_message="Get the messages in the conversation with id '1234567890' from 2025-01-20", - expected_tool_calls=[ - ExpectedToolCall( - func=get_messages_in_conversation_by_id, - args={ - "conversation_id": "1234567890", - "oldest_datetime": "2025-01-20 00:00:00", - "latest_datetime": "2025-01-20 23:59:59", - }, - ), - ], - critics=[ - BinaryCritic(critic_field="conversation_id", weight=1 / 3), - DatetimeCritic( - critic_field="oldest_datetime", weight=1 / 3, max_difference=timedelta(minutes=2) - ), - DatetimeCritic( - critic_field="latest_datetime", weight=1 / 3, max_difference=timedelta(minutes=2) - ), - ], - ) - - suite.add_case( - name="Get conversation history oldest absolute by id (between a date range)", - user_message="Get the messages in the conversation with id '1234567890' from 2025-01-20 to 2025-01-25", - expected_tool_calls=[ - ExpectedToolCall( - func=get_messages_in_conversation_by_id, - args={ - "conversation_id": "1234567890", - "oldest_datetime": "2025-01-20 00:00:00", - "latest_datetime": "2025-01-25 23:59:59", - }, - ), - ], - critics=[ - BinaryCritic(critic_field="conversation_id", weight=1 / 3), - DatetimeCritic( - critic_field="oldest_datetime", weight=1 / 3, max_difference=timedelta(minutes=2) - ), - DatetimeCritic( - critic_field="latest_datetime", weight=1 / 3, max_difference=timedelta(minutes=2) - ), - ], - ) - - suite.add_case( - name="Get conversation history oldest absolute by name (on a specific date)", - user_message="Get the messages in the #general channel from 2025-01-20", - expected_tool_calls=[ - ExpectedToolCall( - func=get_messages_in_channel_by_name, - args={ - "conversation_name": "general", - "oldest_datetime": "2025-01-20 00:00:00", - "latest_datetime": "2025-01-20 23:59:59", - }, - ), - ], - critics=[ - BinaryCritic(critic_field="conversation_name", weight=1 / 3), - # We use a timedelta of 10 seconds because sometimes the LLM will select the limit - # date at 23:59:59, other times it'll select the next day at 00:00:00. - DatetimeCritic( - critic_field="oldest_datetime", weight=1 / 3, max_difference=timedelta(seconds=10) - ), - DatetimeCritic( - critic_field="latest_datetime", weight=1 / 3, max_difference=timedelta(seconds=10) - ), - ], - ) - - suite.add_case( - name="Get conversation history oldest absolute by name (between a date range)", - user_message="Get the messages in the #general channel from 2025-01-20 to 2025-01-25", - expected_tool_calls=[ - ExpectedToolCall( - func=get_messages_in_channel_by_name, - args={ - "conversation_name": "general", - "oldest_datetime": "2025-01-20 00:00:00", - "latest_datetime": "2025-01-25 23:59:59", - }, - ), - ], - critics=[ - BinaryCritic(critic_field="conversation_name", weight=1 / 3), - # We use a timedelta of 10 seconds because sometimes the LLM will select the limit - # date at 23:59:59, other times it'll select the next day at 00:00:00. - DatetimeCritic( - critic_field="oldest_datetime", weight=1 / 3, max_difference=timedelta(seconds=10) - ), - DatetimeCritic( - critic_field="latest_datetime", weight=1 / 3, max_difference=timedelta(seconds=10) - ), - ], - ) - - # Eval case for pagination - - suite.add_case( - name="Get conversation history with pagination", - user_message="get the next 5 messages", - expected_tool_calls=[ - ExpectedToolCall( - func=get_messages_in_channel_by_name, - args={ - "conversation_name": "general", - "limit": 5, - "cursor": "cursor_xyz", - }, - ), - ], - critics=[ - BinaryCritic(critic_field="conversation_name", weight=1 / 3), - BinaryCritic(critic_field="cursor", weight=1 / 3), - BinaryCritic(critic_field="limit", weight=1 / 3), - ], - additional_messages=[ - {"role": "user", "content": "Get the last 2 messages on the general channel"}, - { - "role": "assistant", - "content": "", - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": { - "name": "Slack_GetConversationHistoryByName", - "arguments": json.dumps({ - "conversation_name": "general", - "limit": 2, - }), - }, - } - ], - }, - { - "role": "tool", - "content": json.dumps({ - "messages": [ - { - "blocks": [ - { - "block_id": "abc123", - "elements": [ - { - "elements": [ - { - "text": "Almost there, Boss, need to get some evals in!", - "type": "text", - } - ], - "type": "rich_text_section", - } - ], - "type": "rich_text", - } - ], - "client_msg_id": "msg_id_0", - "datetime_timestamp": "2025-01-21 16:59:55", - "team": "617263616465207465616D20697320617420626F7373206C6576656C", - "text": "Almost there, Boss, need to get some evals in!", - "ts": "1737507595.598529", - "type": "message", - "user": "77686F2069732074686520626F73733F", - }, - { - "blocks": [ - { - "block_id": "xyz456", - "elements": [ - { - "elements": [ - { - "text": "hey, are the Slack Tools ready yet?", - "type": "text", - } - ], - "type": "rich_text_section", - } - ], - "type": "rich_text", - } - ], - "client_msg_id": "msg_id_1", - "datetime_timestamp": "2025-01-21 16:57:35", - "team": "617263616465207465616D20697320617420626F7373206C6576656C", - "text": "hey, are the Slack Tools ready yet?", - "ts": "1737507595.598529", - "type": "message", - "user": "73616D2069732074686520626F7373", - }, - ], - "next_cursor": "cursor_xyz", - }), - "tool_call_id": "call_1", - "name": "Slack_GetConversationHistoryByName", - }, - { - "role": "assistant", - "content": 'Here are the last 2 messages from the general channel:\n\n1. **User:** 77686F2069732074686520626F73733F \n **Message:** "Almost there, Boss, need to get some evals in!" \n **Timestamp:** 2025-01-21 16:59:55\n\n2. **User:** 73616D2069732074686520626F7373 \n **Message:** "hey, are the Slack Tools ready yet?" \n **Timestamp:** 2025-01-21 16:57:35', - }, - ], - ) - - return suite diff --git a/toolkits/slack/tests/test_chat.py b/toolkits/slack/tests/test_chat.py index 19111b4d..c1fe3ca5 100644 --- a/toolkits/slack/tests/test_chat.py +++ b/toolkits/slack/tests/test_chat.py @@ -10,12 +10,14 @@ from slack_sdk.web.async_slack_response import AsyncSlackResponse from arcade_slack.constants import MAX_PAGINATION_SIZE_LIMIT from arcade_slack.models import ConversationType, ConversationTypeSlackName from arcade_slack.tools.chat import ( + get_channel_metadata_by_name, get_conversation_metadata_by_id, - get_conversation_metadata_by_name, + get_direct_message_conversation_metadata_by_username, + get_members_in_channel_by_name, get_members_in_conversation_by_id, - get_members_in_conversation_by_name, get_messages_in_channel_by_name, get_messages_in_conversation_by_id, + get_messages_in_direct_message_conversation_by_username, list_conversations_metadata, list_direct_message_conversations_metadata, list_group_direct_message_conversations_metadata, @@ -37,37 +39,31 @@ def mock_channel_info() -> dict: return {"name": "general", "id": "C12345", "is_member": True, "is_channel": True} -@pytest.fixture -def mock_slack_client(mocker): - mock_client = mocker.patch("arcade_slack.tools.chat.AsyncWebClient", autospec=True) - return mock_client.return_value - - @pytest.mark.asyncio -async def test_send_dm_to_user(mock_context, mock_slack_client): - mock_slack_client.users_list.return_value = { +async def test_send_dm_to_user(mock_context, mock_chat_slack_client): + mock_chat_slack_client.users_list.return_value = { "ok": True, "members": [{"name": "testuser", "id": "U12345"}], } - mock_slack_client.conversations_open.return_value = { + mock_chat_slack_client.conversations_open.return_value = { "ok": True, "channel": {"id": "D12345"}, } mock_slack_response = Mock(spec=AsyncSlackResponse) mock_slack_response.data = {"ok": True} - mock_slack_client.chat_postMessage.return_value = mock_slack_response + mock_chat_slack_client.chat_postMessage.return_value = mock_slack_response response = await send_dm_to_user(mock_context, "testuser", "Hello!") assert response["response"]["ok"] is True - mock_slack_client.users_list.assert_called_once() - mock_slack_client.conversations_open.assert_called_once_with(users=["U12345"]) - mock_slack_client.chat_postMessage.assert_called_once_with(channel="D12345", text="Hello!") + mock_chat_slack_client.users_list.assert_called_once() + mock_chat_slack_client.conversations_open.assert_called_once_with(users=["U12345"]) + mock_chat_slack_client.chat_postMessage.assert_called_once_with(channel="D12345", text="Hello!") @pytest.mark.asyncio -async def test_send_dm_to_inexistent_user(mock_context, mock_slack_client): - mock_slack_client.users_list.return_value = { +async def test_send_dm_to_inexistent_user(mock_context, mock_chat_slack_client): + mock_chat_slack_client.users_list.return_value = { "ok": True, "members": [{"name": "testuser", "id": "U12345"}], } @@ -75,33 +71,33 @@ async def test_send_dm_to_inexistent_user(mock_context, mock_slack_client): with pytest.raises(RetryableToolError): await send_dm_to_user(mock_context, "inexistent_user", "Hello!") - mock_slack_client.users_list.assert_called_once() - mock_slack_client.conversations_open.assert_not_called() - mock_slack_client.chat_postMessage.assert_not_called() + mock_chat_slack_client.users_list.assert_called_once() + mock_chat_slack_client.conversations_open.assert_not_called() + mock_chat_slack_client.chat_postMessage.assert_not_called() @pytest.mark.asyncio -async def test_send_message_to_channel(mock_context, mock_slack_client): - mock_slack_client.conversations_list.return_value = { +async def test_send_message_to_channel(mock_context, mock_chat_slack_client): + mock_chat_slack_client.conversations_list.return_value = { "ok": True, "channels": [{"id": "C12345", "name": "general"}], } mock_slack_response = Mock(spec=AsyncSlackResponse) mock_slack_response.data = {"ok": True} - mock_slack_client.chat_postMessage.return_value = mock_slack_response + mock_chat_slack_client.chat_postMessage.return_value = mock_slack_response response = await send_message_to_channel(mock_context, "general", "Hello, channel!") assert response["response"]["ok"] is True - mock_slack_client.conversations_list.assert_called_once() - mock_slack_client.chat_postMessage.assert_called_once_with( + mock_chat_slack_client.conversations_list.assert_called_once() + mock_chat_slack_client.chat_postMessage.assert_called_once_with( channel="C12345", text="Hello, channel!" ) @pytest.mark.asyncio -async def test_send_message_to_inexistent_channel(mock_context, mock_slack_client): - mock_slack_client.conversations_list.return_value = { +async def test_send_message_to_inexistent_channel(mock_context, mock_chat_slack_client): + mock_chat_slack_client.conversations_list.return_value = { "ok": True, "channels": [], } @@ -109,15 +105,15 @@ async def test_send_message_to_inexistent_channel(mock_context, mock_slack_clien with pytest.raises(RetryableToolError): await send_message_to_channel(mock_context, "inexistent_channel", "Hello!") - mock_slack_client.conversations_list.assert_called_once() - mock_slack_client.chat_postMessage.assert_not_called() + mock_chat_slack_client.conversations_list.assert_called_once() + mock_chat_slack_client.chat_postMessage.assert_not_called() @pytest.mark.asyncio async def test_list_conversations_metadata_with_default_args( - mock_context, mock_slack_client, mock_channel_info + mock_context, mock_chat_slack_client, mock_channel_info ): - mock_slack_client.conversations_list.return_value = { + mock_chat_slack_client.conversations_list.return_value = { "ok": True, "channels": [mock_channel_info], } @@ -127,7 +123,7 @@ async def test_list_conversations_metadata_with_default_args( assert response["conversations"] == [extract_conversation_metadata(mock_channel_info)] assert response["next_cursor"] is None - mock_slack_client.conversations_list.assert_called_once_with( + mock_chat_slack_client.conversations_list.assert_called_once_with( types=",".join([conv_type.value for conv_type in ConversationTypeSlackName]), exclude_archived=True, limit=MAX_PAGINATION_SIZE_LIMIT, @@ -137,9 +133,9 @@ async def test_list_conversations_metadata_with_default_args( @pytest.mark.asyncio async def test_list_conversations_metadata_filtering_single_conversation_type( - mock_context, mock_slack_client, mock_channel_info + mock_context, mock_chat_slack_client, mock_channel_info ): - mock_slack_client.conversations_list.return_value = { + mock_chat_slack_client.conversations_list.return_value = { "ok": True, "channels": [mock_channel_info], } @@ -151,7 +147,7 @@ async def test_list_conversations_metadata_filtering_single_conversation_type( assert response["conversations"] == [extract_conversation_metadata(mock_channel_info)] assert response["next_cursor"] is None - mock_slack_client.conversations_list.assert_called_once_with( + mock_chat_slack_client.conversations_list.assert_called_once_with( types=ConversationTypeSlackName.PUBLIC_CHANNEL.value, exclude_archived=True, limit=MAX_PAGINATION_SIZE_LIMIT, @@ -161,9 +157,9 @@ async def test_list_conversations_metadata_filtering_single_conversation_type( @pytest.mark.asyncio async def test_list_conversations_metadata_filtering_multiple_conversation_types( - mock_context, mock_slack_client, mock_channel_info + mock_context, mock_chat_slack_client, mock_channel_info ): - mock_slack_client.conversations_list.return_value = { + mock_chat_slack_client.conversations_list.return_value = { "ok": True, "channels": [mock_channel_info], } @@ -179,7 +175,7 @@ async def test_list_conversations_metadata_filtering_multiple_conversation_types assert response["conversations"] == [extract_conversation_metadata(mock_channel_info)] assert response["next_cursor"] is None - mock_slack_client.conversations_list.assert_called_once_with( + mock_chat_slack_client.conversations_list.assert_called_once_with( types=f"{ConversationTypeSlackName.PUBLIC_CHANNEL.value},{ConversationTypeSlackName.PRIVATE_CHANNEL.value}", exclude_archived=True, limit=MAX_PAGINATION_SIZE_LIMIT, @@ -189,9 +185,9 @@ async def test_list_conversations_metadata_filtering_multiple_conversation_types @pytest.mark.asyncio async def test_list_conversations_metadata_with_custom_pagination_args( - mock_context, mock_slack_client, mock_channel_info + mock_context, mock_chat_slack_client, mock_channel_info ): - mock_slack_client.conversations_list.return_value = { + mock_chat_slack_client.conversations_list.return_value = { "ok": True, "channels": [mock_channel_info] * 3, "response_metadata": {"next_cursor": "456"}, @@ -204,7 +200,7 @@ async def test_list_conversations_metadata_with_custom_pagination_args( ] assert response["next_cursor"] == "456" - mock_slack_client.conversations_list.assert_called_once_with( + mock_chat_slack_client.conversations_list.assert_called_once_with( types=",".join([conv_type.value for conv_type in ConversationTypeSlackName]), exclude_archived=True, limit=3, @@ -221,9 +217,9 @@ async def test_list_conversations_metadata_with_custom_pagination_args( ], ) async def test_tools_with_slack_error( - mock_context, mock_slack_client, faulty_slack_function_name, tool_function, tool_args + mock_context, mock_chat_slack_client, faulty_slack_function_name, tool_function, tool_args ): - getattr(mock_slack_client, faulty_slack_function_name).side_effect = SlackApiError( + getattr(mock_chat_slack_client, faulty_slack_function_name).side_effect = SlackApiError( message="test_slack_error", response={"ok": False, "error": "test_slack_error"}, ) @@ -262,8 +258,10 @@ async def test_list_channels_metadata( @pytest.mark.asyncio -async def test_get_conversation_metadata_by_id(mock_context, mock_slack_client, mock_channel_info): - mock_slack_client.conversations_info.return_value = { +async def test_get_conversation_metadata_by_id( + mock_context, mock_chat_slack_client, mock_channel_info +): + mock_chat_slack_client.conversations_info.return_value = { "ok": True, "channel": mock_channel_info, } @@ -271,7 +269,7 @@ async def test_get_conversation_metadata_by_id(mock_context, mock_slack_client, response = await get_conversation_metadata_by_id(mock_context, "C12345") assert response == extract_conversation_metadata(mock_channel_info) - mock_slack_client.conversations_info.assert_called_once_with( + mock_chat_slack_client.conversations_info.assert_called_once_with( channel="C12345", include_locale=True, include_num_members=True, @@ -281,14 +279,14 @@ async def test_get_conversation_metadata_by_id(mock_context, mock_slack_client, @pytest.mark.asyncio @patch("arcade_slack.tools.chat.list_conversations_metadata") async def test_get_conversation_metadata_by_id_slack_api_error( - mock_list_conversations_metadata, mock_context, mock_slack_client, mock_channel_info + mock_list_conversations_metadata, mock_context, mock_chat_slack_client, mock_channel_info ): mock_channel_info["name"] = "whatever_conversation_should_be_present_in_additional_prompt" mock_list_conversations_metadata.return_value = { "conversations": [extract_conversation_metadata(mock_channel_info)], "response_metadata": {"next_cursor": None}, } - mock_slack_client.conversations_info.side_effect = SlackApiError( + mock_chat_slack_client.conversations_info.side_effect = SlackApiError( message="channel_not_found", response={"ok": False, "error": "channel_not_found"}, ) @@ -301,7 +299,7 @@ async def test_get_conversation_metadata_by_id_slack_api_error( in e.additional_prompt_content ) - mock_slack_client.conversations_info.assert_called_once_with( + mock_chat_slack_client.conversations_info.assert_called_once_with( channel="C12345", include_locale=True, include_num_members=True, @@ -319,34 +317,34 @@ async def test_get_conversation_metadata_by_name( "next_cursor": None, } - response = await get_conversation_metadata_by_name(mock_context, sample_conversation["name"]) + response = await get_channel_metadata_by_name(mock_context, sample_conversation["name"]) assert response == sample_conversation mock_list_conversations_metadata.assert_called_once_with(mock_context, next_cursor=None) @pytest.mark.asyncio -async def test_get_conversation_metadata_by_name_triggering_pagination( +async def test_get_channel_metadata_by_name_triggering_pagination( mock_context, mock_list_conversations_metadata, mock_channel_info ): - target_conversation = extract_conversation_metadata(mock_channel_info) - another_conversation = extract_conversation_metadata(mock_channel_info) - another_conversation["name"] = "another_conversation" + target_channel = extract_conversation_metadata(mock_channel_info) + another_channel = extract_conversation_metadata(mock_channel_info) + another_channel["name"] = "another_channel" mock_list_conversations_metadata.side_effect = [ { - "conversations": [another_conversation], + "conversations": [another_channel], "next_cursor": "123", }, { - "conversations": [target_conversation], + "conversations": [target_channel], "next_cursor": None, }, ] - response = await get_conversation_metadata_by_name(mock_context, target_conversation["name"]) + response = await get_channel_metadata_by_name(mock_context, target_channel["name"]) - assert response == target_conversation + assert response == target_channel assert mock_list_conversations_metadata.call_count == 2 mock_list_conversations_metadata.assert_has_calls([ call(mock_context, next_cursor=None), @@ -355,26 +353,26 @@ async def test_get_conversation_metadata_by_name_triggering_pagination( @pytest.mark.asyncio -async def test_get_conversation_metadata_by_name_not_found( +async def test_get_channel_metadata_by_name_not_found( mock_context, mock_list_conversations_metadata, mock_channel_info ): - first_conversation = extract_conversation_metadata(mock_channel_info) - second_conversation = extract_conversation_metadata(mock_channel_info) - second_conversation["name"] = "second_conversation" + first_channel = extract_conversation_metadata(mock_channel_info) + second_channel = extract_conversation_metadata(mock_channel_info) + second_channel["name"] = "second_channel" mock_list_conversations_metadata.side_effect = [ { - "conversations": [second_conversation], + "conversations": [second_channel], "next_cursor": "123", }, { - "conversations": [first_conversation], + "conversations": [first_channel], "next_cursor": None, }, ] with pytest.raises(RetryableToolError): - await get_conversation_metadata_by_name(mock_context, "inexistent_conversation") + await get_channel_metadata_by_name(mock_context, "inexistent_channel") assert mock_list_conversations_metadata.call_count == 2 mock_list_conversations_metadata.assert_has_calls([ @@ -387,7 +385,7 @@ async def test_get_conversation_metadata_by_name_not_found( @patch("arcade_slack.tools.chat.async_paginate") @patch("arcade_slack.tools.chat.get_user_info_by_id") async def test_get_members_from_conversation_id( - mock_get_user_info_by_id, mock_async_paginate, mock_context, mock_slack_client + mock_get_user_info_by_id, mock_async_paginate, mock_context, mock_chat_slack_client ): member1 = {"id": "U123", "name": "testuser123"} member1_info = extract_basic_user_info(member1) @@ -406,7 +404,7 @@ async def test_get_members_from_conversation_id( "next_cursor": "token123", } mock_async_paginate.assert_called_once_with( - mock_slack_client.conversations_members, + mock_chat_slack_client.conversations_members, "members", limit=2, next_cursor=None, @@ -427,7 +425,7 @@ async def test_get_members_from_conversation_id_channel_not_found( mock_get_user_info_by_id, mock_async_paginate, mock_context, - mock_slack_client, + mock_chat_slack_client, mock_channel_info, ): conversations = [extract_conversation_metadata(mock_channel_info)] * 2 @@ -451,7 +449,7 @@ async def test_get_members_from_conversation_id_channel_not_found( await get_members_in_conversation_by_id(mock_context, conversation_id="C12345", limit=2) mock_async_paginate.assert_called_once_with( - mock_slack_client.conversations_members, + mock_chat_slack_client.conversations_members, "members", limit=2, next_cursor=None, @@ -463,7 +461,7 @@ async def test_get_members_from_conversation_id_channel_not_found( @pytest.mark.asyncio @patch("arcade_slack.tools.chat.list_conversations_metadata") @patch("arcade_slack.tools.chat.get_members_in_conversation_by_id") -async def test_get_members_in_conversation_by_name( +async def test_get_members_in_channel_by_name( mock_get_members_in_conversation_by_id, mock_list_conversations_metadata, mock_context, @@ -474,7 +472,7 @@ async def test_get_members_in_conversation_by_name( "next_cursor": None, } - response = await get_members_in_conversation_by_name( + response = await get_members_in_channel_by_name( mock_context, mock_channel_info["name"], limit=2 ) @@ -491,7 +489,7 @@ async def test_get_members_in_conversation_by_name( @pytest.mark.asyncio @patch("arcade_slack.tools.chat.list_conversations_metadata") @patch("arcade_slack.tools.chat.get_members_in_conversation_by_id") -async def test_get_members_in_conversation_by_name_triggering_pagination( +async def test_get_members_in_channel_by_name_triggering_pagination( mock_get_members_in_conversation_by_id, mock_list_conversations_metadata, mock_context, @@ -513,9 +511,7 @@ async def test_get_members_in_conversation_by_name_triggering_pagination( }, ] - response = await get_members_in_conversation_by_name( - mock_context, conversation2["name"], limit=2 - ) + response = await get_members_in_channel_by_name(mock_context, conversation2["name"], limit=2) assert response == mock_get_members_in_conversation_by_id.return_value mock_list_conversations_metadata.assert_has_calls([ @@ -531,8 +527,8 @@ async def test_get_members_in_conversation_by_name_triggering_pagination( @pytest.mark.asyncio -async def test_get_conversation_history_by_id(mock_context, mock_slack_client): - mock_slack_client.conversations_history.return_value = { +async def test_get_conversation_history_by_id(mock_context, mock_chat_slack_client): + mock_chat_slack_client.conversations_history.return_value = { "ok": True, "messages": [{"text": "Hello, world!"}], } @@ -540,7 +536,7 @@ async def test_get_conversation_history_by_id(mock_context, mock_slack_client): response = await get_messages_in_conversation_by_id(mock_context, "C12345", limit=1) assert response == {"messages": [{"text": "Hello, world!"}], "next_cursor": None} - mock_slack_client.conversations_history.assert_called_once_with( + mock_chat_slack_client.conversations_history.assert_called_once_with( channel="C12345", include_all_metadata=True, inclusive=True, @@ -555,9 +551,12 @@ async def test_get_conversation_history_by_id(mock_context, mock_slack_client): @patch("arcade_slack.tools.chat.convert_relative_datetime_to_unix_timestamp") @patch("arcade_slack.tools.chat.datetime") async def test_get_conversation_history_by_id_with_relative_datetime_args( - mock_datetime, mock_convert_relative_datetime_to_unix_timestamp, mock_context, mock_slack_client + mock_datetime, + mock_convert_relative_datetime_to_unix_timestamp, + mock_context, + mock_chat_slack_client, ): - mock_slack_client.conversations_history.return_value = { + mock_chat_slack_client.conversations_history.return_value = { "ok": True, "messages": [{"text": "Hello, world!"}], } @@ -584,7 +583,7 @@ async def test_get_conversation_history_by_id_with_relative_datetime_args( call("01:00:00", expected_current_unix_timestamp), call("02:00:00", expected_current_unix_timestamp), ]) - mock_slack_client.conversations_history.assert_called_once_with( + mock_chat_slack_client.conversations_history.assert_called_once_with( channel="C12345", include_all_metadata=True, inclusive=True, @@ -600,9 +599,9 @@ async def test_get_conversation_history_by_id_with_relative_datetime_args( @pytest.mark.asyncio @patch("arcade_slack.tools.chat.convert_datetime_to_unix_timestamp") async def test_get_conversation_history_by_id_with_absolute_datetime_args( - mock_convert_datetime_to_unix_timestamp, mock_context, mock_slack_client + mock_convert_datetime_to_unix_timestamp, mock_context, mock_chat_slack_client ): - mock_slack_client.conversations_history.return_value = { + mock_chat_slack_client.conversations_history.return_value = { "ok": True, "messages": [{"text": "Hello, world!"}], } @@ -631,7 +630,7 @@ async def test_get_conversation_history_by_id_with_absolute_datetime_args( call("2025-01-02 00:00:00"), call("2025-01-01 00:00:00"), ]) - mock_slack_client.conversations_history.assert_called_once_with( + mock_chat_slack_client.conversations_history.assert_called_once_with( channel="C12345", include_all_metadata=True, inclusive=True, @@ -644,7 +643,7 @@ async def test_get_conversation_history_by_id_with_absolute_datetime_args( @pytest.mark.asyncio async def test_get_conversation_history_by_id_with_messed_oldest_args( - mock_context, mock_slack_client + mock_context, mock_chat_slack_client ): with pytest.raises(ToolExecutionError): await get_messages_in_conversation_by_id( @@ -657,7 +656,7 @@ async def test_get_conversation_history_by_id_with_messed_oldest_args( @pytest.mark.asyncio async def test_get_conversation_history_by_id_with_messed_latest_args( - mock_context, mock_slack_client + mock_context, mock_chat_slack_client ): with pytest.raises(ToolExecutionError): await get_messages_in_conversation_by_id( @@ -669,8 +668,8 @@ async def test_get_conversation_history_by_id_with_messed_latest_args( @pytest.mark.asyncio -async def test_get_conversation_history_by_name(mock_context, mock_slack_client): - mock_slack_client.conversations_list.return_value = { +async def test_get_conversation_history_by_name(mock_context, mock_chat_slack_client): + mock_chat_slack_client.conversations_list.return_value = { "ok": True, "channels": [ { @@ -681,7 +680,7 @@ async def test_get_conversation_history_by_name(mock_context, mock_slack_client) } ], } - mock_slack_client.conversations_history.return_value = { + mock_chat_slack_client.conversations_history.return_value = { "ok": True, "messages": [{"text": "Hello, world!"}], } @@ -689,6 +688,133 @@ async def test_get_conversation_history_by_name(mock_context, mock_slack_client) response = await get_messages_in_channel_by_name(mock_context, "general", limit=1) assert response == {"messages": [{"text": "Hello, world!"}], "next_cursor": None} - mock_slack_client.conversations_history.assert_called_once_with( + mock_chat_slack_client.conversations_history.assert_called_once_with( channel="C12345", include_all_metadata=True, inclusive=True, limit=1, cursor=None ) + + +@pytest.mark.asyncio +@patch("arcade_slack.tools.chat.retrieve_conversations_by_user_ids") +async def test_get_direct_message_conversation_metadata_by_username( + mock_retrieve_conversations_by_user_ids, + mock_context, + mock_chat_slack_client, + mock_users_slack_client, +): + mock_chat_slack_client.auth_test.return_value = { + "ok": True, + "user_id": "U1", + "team_id": "T1", + "user": "user1", + } + + mock_users_slack_client.users_list.return_value = { + "ok": True, + "members": [ + {"id": "U1", "name": "user1"}, + {"id": "U2", "name": "user2"}, + ], + "response_metadata": {"next_cursor": None}, + } + + conversation = { + "id": "C12345", + "type": ConversationTypeSlackName.IM.value, + "is_im": True, + "members": ["U1", "U2"], + } + + mock_retrieve_conversations_by_user_ids.return_value = [conversation] + + response = await get_direct_message_conversation_metadata_by_username( + context=mock_context, username="user2" + ) + + assert response == conversation + mock_retrieve_conversations_by_user_ids.assert_called_once_with( + list_conversations_func=list_conversations_metadata, + get_members_in_conversation_func=get_members_in_conversation_by_id, + context=mock_context, + conversation_types=[ConversationType.DIRECT_MESSAGE], + user_ids=["U1", "U2"], + exact_match=True, + limit=1, + next_cursor=None, + ) + + +@pytest.mark.asyncio +@patch("arcade_slack.tools.chat.retrieve_conversations_by_user_ids") +async def test_get_direct_message_conversation_metadata_by_username_username_not_found( + mock_retrieve_conversations_by_user_ids, + mock_context, + mock_chat_slack_client, + mock_users_slack_client, +): + mock_chat_slack_client.users_identity.return_value = { + "ok": True, + "user": {"id": "U1", "name": "user1"}, + "team": {"id": "T1", "name": "team1"}, + } + + mock_users_slack_client.users_list.return_value = { + "ok": True, + "members": [ + {"id": "U1", "name": "user1"}, + {"id": "U2", "name": "user2"}, + ], + "response_metadata": {"next_cursor": None}, + } + + mock_retrieve_conversations_by_user_ids.side_effect = TimeoutError() + + with pytest.raises(RetryableToolError): + await get_direct_message_conversation_metadata_by_username( + context=mock_context, username="user999" + ) + + +@pytest.mark.asyncio +@patch("arcade_slack.tools.chat.get_messages_in_conversation_by_id") +@patch("arcade_slack.tools.chat.get_direct_message_conversation_metadata_by_username") +async def test_get_messages_in_direct_conversation_by_username( + mock_get_direct_message_conversation_metadata_by_username, + mock_get_messages_in_conversation_by_id, + mock_context, +): + mock_get_direct_message_conversation_metadata_by_username.return_value = { + "id": "C12345", + } + + response = await get_messages_in_direct_message_conversation_by_username( + context=mock_context, username="user2" + ) + + assert response == mock_get_messages_in_conversation_by_id.return_value + mock_get_direct_message_conversation_metadata_by_username.assert_called_once_with( + context=mock_context, username="user2" + ) + mock_get_messages_in_conversation_by_id.assert_called_once_with( + context=mock_context, + conversation_id="C12345", + oldest_relative=None, + latest_relative=None, + oldest_datetime=None, + latest_datetime=None, + limit=None, + next_cursor=None, + ) + + +@pytest.mark.asyncio +@patch("arcade_slack.tools.chat.get_direct_message_conversation_metadata_by_username") +async def test_get_messages_in_direct_conversation_by_username_not_found( + mock_get_direct_message_conversation_metadata_by_username, + mock_context, +): + mock_get_direct_message_conversation_metadata_by_username.return_value = None + + with pytest.raises(ToolExecutionError): + await get_messages_in_direct_message_conversation_by_username( + context=mock_context, username="user2" + ) diff --git a/toolkits/slack/tests/test_utils.py b/toolkits/slack/tests/test_utils.py index f7c83ec2..c569e0aa 100644 --- a/toolkits/slack/tests/test_utils.py +++ b/toolkits/slack/tests/test_utils.py @@ -1,12 +1,26 @@ import asyncio -from unittest.mock import AsyncMock, call, patch +from unittest.mock import AsyncMock, MagicMock, call, patch import pytest +from arcade.sdk import ToolContext from slack_sdk.errors import SlackApiError from slack_sdk.web.async_client import AsyncWebClient from arcade_slack.exceptions import PaginationTimeoutError -from arcade_slack.utils import async_paginate +from arcade_slack.models import ( + ConversationType, + ConversationTypeSlackName, +) +from arcade_slack.tools.chat import ( + get_members_in_conversation_by_id, + list_conversations_metadata, +) +from arcade_slack.utils import ( + async_paginate, + convert_conversation_type_to_slack_name, + filter_conversations_by_user_ids, + retrieve_conversations_by_user_ids, +) @pytest.mark.asyncio @@ -185,3 +199,486 @@ async def test_async_paginate_with_timeout(): str(exc_info.value) == f"The pagination process timed out after {max_pagination_timeout_seconds} seconds." ) + + +def test_filter_conversations_by_user_ids(): + conversations = [ + {"id": "123", "members": [{"id": "user1"}, {"id": "user2"}, {"id": "user3"}]}, + {"id": "456", "members": [{"id": "user2"}, {"id": "user3"}]}, + ] + response = filter_conversations_by_user_ids( + conversations=conversations, + user_ids=["user1", "user2"], + exact_match=False, + ) + assert response == [ + {"id": "123", "members": [{"id": "user1"}, {"id": "user2"}, {"id": "user3"}]}, + ] + + +def test_filter_conversations_by_user_ids_empty_response(): + conversations = [ + {"id": "123", "members": [{"id": "user1"}, {"id": "user3"}, {"id": "user4"}]}, + {"id": "456", "members": [{"id": "user2"}, {"id": "user3"}, {"id": "user4"}]}, + ] + response = filter_conversations_by_user_ids( + conversations=conversations, + user_ids=["user1", "user2"], + exact_match=False, + ) + assert response == [] + + +def test_filter_conversations_by_user_ids_multiple_matches(): + conversations = [ + {"id": "123", "members": [{"id": "user1"}, {"id": "user2"}, {"id": "user3"}]}, + {"id": "456", "members": [{"id": "user2"}, {"id": "user3"}]}, + { + "id": "789", + "members": [{"id": "user4"}, {"id": "user1"}, {"id": "user2"}, {"id": "user3"}], + }, + ] + response = filter_conversations_by_user_ids( + conversations=conversations, + user_ids=["user1", "user2"], + exact_match=False, + ) + assert response == [ + {"id": "123", "members": [{"id": "user1"}, {"id": "user2"}, {"id": "user3"}]}, + { + "id": "789", + "members": [{"id": "user4"}, {"id": "user1"}, {"id": "user2"}, {"id": "user3"}], + }, + ] + + +def test_filter_conversations_by_user_ids_exact_match(): + conversations = [ + {"id": "123", "members": [{"id": "user1"}, {"id": "user2"}]}, + {"id": "456", "members": [{"id": "user2"}, {"id": "user3"}]}, + ] + response = filter_conversations_by_user_ids( + conversations=conversations, + user_ids=["user1", "user2"], + exact_match=True, + ) + assert response == [{"id": "123", "members": [{"id": "user1"}, {"id": "user2"}]}] + + +def test_filter_conversations_by_user_ids_exact_match_empty_response(): + conversations = [ + {"id": "123", "members": [{"id": "user1"}, {"id": "user2"}, {"id": "user3"}]}, + {"id": "456", "members": [{"id": "user2"}, {"id": "user3"}]}, + ] + response = filter_conversations_by_user_ids( + conversations=conversations, + user_ids=["user1", "user2"], + exact_match=True, + ) + assert response == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "search_user_ids, conversation_types, exact_match, limit, expected_conversation_ids", + [ + (["U1", "U2"], [ConversationType.DIRECT_MESSAGE], False, 1, ["C1"]), + (["U1", "U2"], [ConversationType.DIRECT_MESSAGE], True, 1, ["C1"]), + (["U1", "U2", "U3"], [ConversationType.DIRECT_MESSAGE], False, 1, []), + ( + ["U1", "U2"], + [ConversationType.DIRECT_MESSAGE, ConversationType.PUBLIC_CHANNEL], + False, + 10, + ["C1", "C3", "C4"], + ), + ( + ["U1", "U2"], + [ConversationType.DIRECT_MESSAGE, ConversationType.PUBLIC_CHANNEL], + True, + 10, + ["C1", "C3"], + ), + ], +) +async def test_retrieve_conversations_by_user_ids( + mock_chat_slack_client, + mock_users_slack_client, + search_user_ids, + conversation_types, + exact_match, + limit, + expected_conversation_ids, +): + context = MagicMock(spec=ToolContext) + context.authorization = MagicMock() + context.authorization.token = MagicMock() + + conversation_types_slack_name_str = [ + convert_conversation_type_to_slack_name(conv_type).value + for conv_type in conversation_types or ConversationType + ] + + conversations = [ + { + "conversation": { + "id": "C1", + "type": ConversationTypeSlackName.IM.value, + "name": "im-1", + "is_channel": False, + "is_im": True, + "is_member": True, + }, + "members": { + "ok": True, + "members": ["U1", "U2"], + "response_metadata": {"next_cursor": None}, + }, + "users": [ + {"ok": True, "user": {"id": "U1", "team_id": "T123", "name": "user1"}}, + {"ok": True, "user": {"id": "U2", "team_id": "T123", "name": "user2"}}, + ], + }, + { + "conversation": { + "id": "C2", + "type": ConversationTypeSlackName.IM.value, + "name": "im-2", + "is_channel": False, + "is_im": True, + "is_member": True, + }, + "members": { + "ok": True, + "members": ["U2", "U3"], + "response_metadata": {"next_cursor": None}, + }, + "users": [ + {"ok": True, "user": {"id": "U2", "team_id": "T123", "name": "user2"}}, + {"ok": True, "user": {"id": "U3", "team_id": "T123", "name": "user3"}}, + ], + }, + { + "conversation": { + "id": "C3", + "type": ConversationTypeSlackName.PUBLIC_CHANNEL.value, + "name": "general", + "is_channel": True, + "is_im": False, + "is_member": True, + }, + "members": { + "ok": True, + "members": ["U1", "U2"], + "response_metadata": {"next_cursor": None}, + }, + "users": [ + {"ok": True, "user": {"id": "U1", "team_id": "T123", "name": "user1"}}, + {"ok": True, "user": {"id": "U2", "team_id": "T123", "name": "user2"}}, + ], + }, + { + "conversation": { + "id": "C4", + "type": ConversationTypeSlackName.PUBLIC_CHANNEL.value, + "name": "random", + "is_channel": True, + "is_im": False, + "is_member": True, + }, + "members": { + "ok": True, + "members": ["U1", "U2", "U3", "U4"], + "response_metadata": {"next_cursor": None}, + }, + "users": [ + {"ok": True, "user": {"id": "U1", "team_id": "T123", "name": "user1"}}, + {"ok": True, "user": {"id": "U2", "team_id": "T123", "name": "user2"}}, + {"ok": True, "user": {"id": "U3", "team_id": "T123", "name": "user3"}}, + {"ok": True, "user": {"id": "U4", "team_id": "T123", "name": "user4"}}, + ], + }, + ] + + conversations_listed = [ + conversation + for conversation in conversations + if conversation["conversation"]["type"] in conversation_types_slack_name_str + ] + + mock_chat_slack_client.conversations_list.return_value = { + "ok": True, + "channels": [conversation["conversation"] for conversation in conversations_listed], + "response_metadata": {"next_cursor": None}, + } + + mock_chat_slack_client.conversations_members.side_effect = [ + conversation["members"] for conversation in conversations_listed + ] + + mock_users_slack_client.users_info.side_effect = [ + user for conversation in conversations_listed for user in conversation["users"] + ] + + conversations_found = await retrieve_conversations_by_user_ids( + list_conversations_func=list_conversations_metadata, + get_members_in_conversation_func=get_members_in_conversation_by_id, + context=context, + conversation_types=conversation_types, + user_ids=search_user_ids, + exact_match=exact_match, + limit=limit, + next_cursor=None, + ) + + assert [conversation["id"] for conversation in conversations_found] == expected_conversation_ids + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ( + "search_user_ids, conversation_types, exact_match, limit, " + "expected_conversation_ids, expected_conversations_list_calls" + ), + [ + ( + ["U1", "U2", "U3"], + [ConversationType.MULTI_PERSON_DIRECT_MESSAGE], + False, + None, + ["C1", "C3"], + 2, + ), + ( + ["U1", "U2", "U3"], + [ConversationType.MULTI_PERSON_DIRECT_MESSAGE], + True, + None, + ["C1"], + 2, + ), + (["U1", "U2", "U99"], [ConversationType.MULTI_PERSON_DIRECT_MESSAGE], False, None, [], 2), + ( + ["U1", "U2"], + [ConversationType.MULTI_PERSON_DIRECT_MESSAGE, ConversationType.PUBLIC_CHANNEL], + False, + None, + ["C1", "C3", "C4", "C6"], + 2, + ), + ( + ["U1", "U2"], + [ConversationType.MULTI_PERSON_DIRECT_MESSAGE, ConversationType.PUBLIC_CHANNEL], + False, + 1, + ["C1"], + 2, + ), + ( + ["U1", "U2"], + [ConversationType.MULTI_PERSON_DIRECT_MESSAGE, ConversationType.PUBLIC_CHANNEL], + False, + 3, + ["C1", "C3", "C4"], + 2, + ), + ( + ["U1", "U2"], + [ConversationType.MULTI_PERSON_DIRECT_MESSAGE, ConversationType.PUBLIC_CHANNEL], + True, + None, + ["C4"], + 2, + ), + ], +) +async def test_retrieve_conversations_by_user_ids_with_pagination( + mock_chat_slack_client, + mock_users_slack_client, + search_user_ids, + conversation_types, + exact_match, + limit, + expected_conversation_ids, + expected_conversations_list_calls, +): + context = MagicMock(spec=ToolContext) + context.authorization = MagicMock() + context.authorization.token = MagicMock() + + conversation_types_slack_name_str = [ + convert_conversation_type_to_slack_name(conv_type).value + for conv_type in conversation_types or ConversationType + ] + + conversations = [ + { + "conversation": { + "id": "C1", + "type": ConversationTypeSlackName.MPIM.value, + "name": "mpim-1", + "is_channel": False, + "is_im": False, + "is_mpim": True, + "is_member": True, + }, + "members": { + "ok": True, + "members": ["U1", "U2", "U3"], + "response_metadata": {"next_cursor": None}, + }, + "users": [ + {"ok": True, "user": {"id": "U1", "team_id": "T123", "name": "user1"}}, + {"ok": True, "user": {"id": "U2", "team_id": "T123", "name": "user2"}}, + {"ok": True, "user": {"id": "U3", "team_id": "T123", "name": "user3"}}, + ], + }, + { + "conversation": { + "id": "C2", + "type": ConversationTypeSlackName.MPIM.value, + "name": "mpim-2", + "is_channel": False, + "is_im": False, + "is_mpim": True, + "is_member": True, + }, + "members": { + "ok": True, + "members": ["U2", "U3"], + "response_metadata": {"next_cursor": None}, + }, + "users": [ + {"ok": True, "user": {"id": "U2", "team_id": "T123", "name": "user2"}}, + {"ok": True, "user": {"id": "U3", "team_id": "T123", "name": "user3"}}, + ], + }, + { + "conversation": { + "id": "C3", + "type": ConversationTypeSlackName.MPIM.value, + "name": "mpim-3", + "is_channel": False, + "is_im": False, + "is_mpim": True, + "is_member": True, + }, + "members": { + "ok": True, + "members": ["U1", "U2", "U3", "U4"], + "response_metadata": {"next_cursor": None}, + }, + "users": [ + {"ok": True, "user": {"id": "U1", "team_id": "T123", "name": "user1"}}, + {"ok": True, "user": {"id": "U2", "team_id": "T123", "name": "user2"}}, + {"ok": True, "user": {"id": "U3", "team_id": "T123", "name": "user3"}}, + {"ok": True, "user": {"id": "U4", "team_id": "T123", "name": "user4"}}, + ], + }, + { + "conversation": { + "id": "C4", + "type": ConversationTypeSlackName.PUBLIC_CHANNEL.value, + "name": "channel-4", + "is_channel": True, + "is_im": False, + "is_member": True, + }, + "members": { + "ok": True, + "members": ["U1", "U2"], + "response_metadata": {"next_cursor": None}, + }, + "users": [ + {"ok": True, "user": {"id": "U1", "team_id": "T123", "name": "user1"}}, + {"ok": True, "user": {"id": "U2", "team_id": "T123", "name": "user2"}}, + ], + }, + { + "conversation": { + "id": "C5", + "type": ConversationTypeSlackName.PUBLIC_CHANNEL.value, + "name": "channel-5", + "is_channel": True, + "is_im": False, + "is_member": True, + }, + "members": { + "ok": True, + "members": ["U2", "U3", "U4"], + "response_metadata": {"next_cursor": None}, + }, + "users": [ + {"ok": True, "user": {"id": "U2", "team_id": "T123", "name": "user2"}}, + {"ok": True, "user": {"id": "U3", "team_id": "T123", "name": "user3"}}, + {"ok": True, "user": {"id": "U4", "team_id": "T123", "name": "user4"}}, + ], + }, + { + "conversation": { + "id": "C6", + "type": ConversationTypeSlackName.PUBLIC_CHANNEL.value, + "name": "channel-6", + "is_channel": True, + "is_im": False, + "is_member": True, + }, + "members": { + "ok": True, + "members": ["U1", "U2", "U3", "U4"], + "response_metadata": {"next_cursor": None}, + }, + "users": [ + {"ok": True, "user": {"id": "U1", "team_id": "T123", "name": "user1"}}, + {"ok": True, "user": {"id": "U2", "team_id": "T123", "name": "user2"}}, + {"ok": True, "user": {"id": "U3", "team_id": "T123", "name": "user3"}}, + {"ok": True, "user": {"id": "U4", "team_id": "T123", "name": "user4"}}, + ], + }, + ] + + conversations_listed = [ + conversation + for conversation in conversations + if conversation["conversation"]["type"] in conversation_types_slack_name_str + ] + + split_size = len(conversations_listed) // 2 + + conversations_listed_1 = conversations_listed[:split_size] + conversations_listed_2 = conversations_listed[split_size:] + + mock_chat_slack_client.conversations_list.side_effect = [ + { + "ok": True, + "channels": [conversation["conversation"] for conversation in conversations_listed_1], + "response_metadata": {"next_cursor": "cursor_1"}, + }, + { + "ok": True, + "channels": [conversation["conversation"] for conversation in conversations_listed_2], + "response_metadata": {"next_cursor": None}, + }, + ] + + mock_chat_slack_client.conversations_members.side_effect = [ + conversation["members"] for conversation in conversations_listed + ] + + mock_users_slack_client.users_info.side_effect = [ + user for conversation in conversations_listed for user in conversation["users"] + ] + + conversations_found = await retrieve_conversations_by_user_ids( + list_conversations_func=list_conversations_metadata, + get_members_in_conversation_func=get_members_in_conversation_by_id, + context=context, + conversation_types=conversation_types, + user_ids=search_user_ids, + exact_match=exact_match, + limit=limit, + next_cursor=None, + ) + + assert [conversation["id"] for conversation in conversations_found] == expected_conversation_ids + assert mock_chat_slack_client.conversations_list.call_count == expected_conversations_list_calls