admin-valentin/arcade-mcp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Slack tools to retrieve messages & metadata from multi-person DM conversation (#254)

Browse source
This commit is contained in:
Renato Byrro 2025-02-19 16:51:45 -03:00 committed by GitHub
parent becd86da0c
commit 8efa9a51df
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 370 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
toolkits/slack/arcade_slack/exceptions.py
View file

@ -17,8 +17,9 @@ class ItemNotFoundError(SlackToolkitError):
class UsernameNotFoundError(SlackToolkitError):
"""Raised when a user is not found by the username searched"""
def __init__(self, usernames_found: list[str]) -> None:
def __init__(self, usernames_found: list[str], username_not_found: str) -> None:
self.usernames_found = usernames_found
self.username_not_found = username_not_found
class ConversationNotFoundError(SlackToolkitError):

159
toolkits/slack/arcade_slack/tools/chat.py
View file

@ -491,7 +491,7 @@ async def get_messages_in_direct_message_conversation_by_username(
dict,
(
"The messages in a direct message conversation and next cursor for paginating results "
"(when there are additional messages to retrieve)."
"when there are additional messages to retrieve."
),
]:
"""Get the messages in a direct conversation by the user's name.
@ -526,6 +526,79 @@ async def get_messages_in_direct_message_conversation_by_username(
)
@tool(requires_auth=Slack(scopes=["im:history", "im:read"]))
async def get_messages_in_multi_person_dm_conversation_by_usernames(
context: ToolContext,
usernames: Annotated[list[str], "The usernames of the users to get messages from"],
oldest_relative: Annotated[
Optional[str],
(
"The oldest message to include in the results, specified as a time offset from the "
"current time in the format 'DD:HH:MM'"
),
] = None,
latest_relative: Annotated[
Optional[str],
(
"The latest message to include in the results, specified as a time offset from the "
"current time in the format 'DD:HH:MM'"
),
] = None,
oldest_datetime: Annotated[
Optional[str],
(
"The oldest message to include in the results, specified as a datetime object in the "
"format 'YYYY-MM-DD HH:MM:SS'"
),
] = None,
latest_datetime: Annotated[
Optional[str],
(
"The latest message to include in the results, specified as a datetime object in the "
"format 'YYYY-MM-DD HH:MM:SS'"
),
] = None,
limit: Annotated[Optional[int], "The maximum number of messages to return."] = None,
next_cursor: Annotated[Optional[str], "The cursor to use for pagination."] = None,
) -> Annotated[
dict,
(
"The messages in a multi-person direct message conversation and next cursor for "
"paginating results (when there are additional messages to retrieve)."
),
]:
"""Get the messages in a multi-person direct message conversation by the usernames.
To filter messages by an absolute datetime, use 'oldest_datetime' and/or 'latest_datetime'. If
only 'oldest_datetime' is provided, it will return messages from the oldest_datetime to the
current time. If only 'latest_datetime' is provided, it will return messages since the
beginning of the conversation to the latest_datetime.
To filter messages by a relative datetime (e.g. 3 days ago, 1 hour ago, etc.), use
'oldest_relative' and/or 'latest_relative'. If only 'oldest_relative' is provided, it will
return messages from the oldest_relative to the current time. If only 'latest_relative' is
provided, it will return messages from the current time to the latest_relative.
Do not provide both 'oldest_datetime' and 'oldest_relative' or both 'latest_datetime' and
'latest_relative'.
Leave all arguments with the default None to get messages without date/time filtering"""
direct_conversation = await get_multi_person_dm_conversation_metadata_by_usernames(
context=context, usernames=usernames
)
return await get_messages_in_conversation_by_id( # type: ignore[no-any-return]
context=context,
conversation_id=direct_conversation["id"],
oldest_relative=oldest_relative,
latest_relative=latest_relative,
oldest_datetime=oldest_datetime,
latest_datetime=latest_datetime,
limit=limit,
next_cursor=next_cursor,
)
@tool(
requires_auth=Slack(
scopes=["channels:read", "groups:read", "im:read", "mpim:read"],
@ -535,7 +608,10 @@ async def get_conversation_metadata_by_id(
context: ToolContext,
conversation_id: Annotated[str, "The ID of the conversation to get metadata for"],
) -> Annotated[dict, "The conversation metadata"]:
"""Get the metadata of a conversation in Slack searching by its ID."""
"""Get the metadata of a conversation in Slack searching by its ID.
This tool does not return the messages in a conversation. To get the messages, use the
`get_messages_in_conversation_by_id` tool."""
token = (
context.authorization.token if context.authorization and context.authorization.token else ""
)
@ -577,7 +653,10 @@ async def get_channel_metadata_by_name(
"The cursor to use for pagination, if continuing from a previous search.",
] = None,
) -> Annotated[dict, "The channel metadata"]:
"""Get the metadata of a channel in Slack searching by its name."""
"""Get the metadata of a channel in Slack searching by its name.
This tool does not return the messages in a channel. To get the messages, use the
`get_messages_in_channel_by_name` tool."""
channel_names: list[str] = []
async def find_channel() -> dict:
@ -639,7 +718,10 @@ async def get_direct_message_conversation_metadata_by_username(
Optional[dict],
"The direct message conversation metadata.",
]:
"""Get the metadata of a direct message conversation in Slack by the username."""
"""Get the metadata of a direct message conversation in Slack by the username.
This tool does not return the messages in a conversation. To get the messages, use the
`get_messages_in_direct_message_conversation_by_username` tool."""
try:
token = (
context.authorization.token
@ -669,8 +751,73 @@ async def get_direct_message_conversation_metadata_by_username(
except UsernameNotFoundError as e:
raise RetryableToolError(
f"Username '{username}' not found",
developer_message=f"User with username '{username}' not found.",
f"Username '{e.username_not_found}' not found",
developer_message=f"User with username '{e.username_not_found}' not found.",
additional_prompt_content=f"Available users: {e.usernames_found}",
retry_after_ms=500,
)
@tool(requires_auth=Slack(scopes=["im:read"]))
async def get_multi_person_dm_conversation_metadata_by_usernames(
context: ToolContext,
usernames: Annotated[list[str], "The usernames of the users/people to get messages with"],
next_cursor: Annotated[
Optional[str],
"The cursor to use for pagination, if continuing from a previous search.",
] = None,
) -> Annotated[
Optional[dict],
"The multi-person direct message conversation metadata.",
]:
"""Get the metadata of a multi-person direct message conversation in Slack by the usernames.
This tool does not return the messages in a conversation. To get the messages, use the
`get_messages_in_multi_person_dm_conversation_by_usernames` tool.
"""
try:
token = (
context.authorization.token
if context.authorization and context.authorization.token
else ""
)
slack_client = AsyncWebClient(token=token)
current_user, list_users_response = await asyncio.gather(
slack_client.auth_test(), list_users(context)
)
other_users = [
get_user_by_username(username, list_users_response["users"]) for username in usernames
]
conversations_found = await retrieve_conversations_by_user_ids(
list_conversations_func=list_conversations_metadata,
get_members_in_conversation_func=get_members_in_conversation_by_id,
context=context,
conversation_types=[ConversationType.MULTI_PERSON_DIRECT_MESSAGE],
user_ids=[
current_user["user_id"],
*[user["id"] for user in other_users if user["id"] != current_user["user_id"]],
],
exact_match=True,
limit=1,
next_cursor=next_cursor,
)
if not conversations_found:
raise RetryableToolError(
"Conversation not found with the usernames provided",
developer_message="Conversation not found with the usernames provided",
retry_after_ms=500,
)
return conversations_found[0]
except UsernameNotFoundError as e:
raise RetryableToolError(
f"Username '{e.username_not_found}' not found",
developer_message=f"User with username '{e.username_not_found}' not found.",
additional_prompt_content=f"Available users: {e.usernames_found}",
retry_after_ms=500,
)

2
toolkits/slack/arcade_slack/utils.py
View file

@ -103,7 +103,7 @@ def get_user_by_username(username: str, users_list: list[dict]) -> SlackUser:
if username.lower() == username_found.lower():
return SlackUser(**user)
raise UsernameNotFoundError(usernames_found)
raise UsernameNotFoundError(usernames_found=usernames_found, username_not_found=username)
def convert_conversation_type_to_slack_name(

80
toolkits/slack/evals/eval_chat.py
View file

@ -23,6 +23,7 @@ from arcade_slack.tools.chat import (
get_messages_in_channel_by_name,
get_messages_in_conversation_by_id,
get_messages_in_direct_message_conversation_by_username,
get_messages_in_multi_person_dm_conversation_by_usernames,
list_conversations_metadata,
list_direct_message_conversations_metadata,
list_group_direct_message_conversations_metadata,
@ -1086,3 +1087,82 @@ def get_messages_in_direct_message_eval_suite() -> EvalSuite:
)
return suite
@tool_eval()
def get_messages_in_multi_person_direct_message_eval_suite() -> EvalSuite:
"""Create an evaluation suite for tools getting messages in multi-person direct messages."""
suite = EvalSuite(
name="Slack Chat Tools Evaluation",
system_message="You are an AI assistant that can interact with Slack to send messages and get information from conversations, users, etc.",
catalog=catalog,
rubric=rubric,
)
no_arguments_user_messages_by_username = [
"what are the latest messages I exchanged together with the usernames john, ryan, and jennifer",
"show the messages in the multi person dm with the usernames john, ryan, and jennifer on Slack",
"list the messages I exchanged together with the usernames john, ryan, and jennifer",
"list the message history together with the usernames john, ryan, and jennifer",
]
for i, user_message in enumerate(no_arguments_user_messages_by_username):
suite.add_case(
name=f"{user_message} [{i}]",
user_message=user_message,
expected_tool_calls=[
ExpectedToolCall(
func=get_messages_in_multi_person_dm_conversation_by_usernames,
args={
"usernames": ["john", "ryan", "jennifer"],
},
),
],
critics=[
BinaryCritic(critic_field="usernames", weight=1.0),
],
)
suite.add_case(
name="get messages in direct conversation by username (on a specific date)",
user_message="get the messages I exchanged together with the usernames john, ryan, and jennifer on 2025-01-31",
expected_tool_calls=[
ExpectedToolCall(
func=get_messages_in_multi_person_dm_conversation_by_usernames,
args={
"usernames": ["john", "ryan", "jennifer"],
"oldest_datetime": "2025-01-31 00:00:00",
"latest_datetime": "2025-01-31 23:59:59",
},
),
],
critics=[
BinaryCritic(critic_field="usernames", weight=1 / 3),
DatetimeCritic(
critic_field="oldest_datetime", weight=1 / 3, max_difference=timedelta(minutes=2)
),
DatetimeCritic(
critic_field="latest_datetime", weight=1 / 3, max_difference=timedelta(minutes=2)
),
],
)
suite.add_case(
name="Get conversation history oldest relative by username (2 days ago)",
user_message="Get the messages I exchanged together with the usernames john, ryan, and jennifer starting 2 days ago",
expected_tool_calls=[
ExpectedToolCall(
func=get_messages_in_multi_person_dm_conversation_by_usernames,
args={
"usernames": ["john", "ryan", "jennifer"],
"oldest_relative": "02:00:00",
},
),
],
critics=[
BinaryCritic(critic_field="usernames", weight=0.5),
RelativeTimeBinaryCritic(critic_field="oldest_relative", weight=0.5),
],
)
return suite

134
toolkits/slack/tests/test_chat.py
View file

@ -18,6 +18,8 @@ from arcade_slack.tools.chat import (
get_messages_in_channel_by_name,
get_messages_in_conversation_by_id,
get_messages_in_direct_message_conversation_by_username,
get_messages_in_multi_person_dm_conversation_by_usernames,
get_multi_person_dm_conversation_metadata_by_usernames,
list_conversations_metadata,
list_direct_message_conversations_metadata,
list_group_direct_message_conversations_metadata,
@ -818,3 +820,135 @@ async def test_get_messages_in_direct_conversation_by_username_not_found(
await get_messages_in_direct_message_conversation_by_username(
context=mock_context, username="user2"
)
@pytest.mark.asyncio
@patch("arcade_slack.tools.chat.retrieve_conversations_by_user_ids")
async def test_get_multi_person_direct_message_conversation_metadata_by_username(
mock_retrieve_conversations_by_user_ids,
mock_context,
mock_chat_slack_client,
mock_users_slack_client,
):
mock_chat_slack_client.auth_test.return_value = {
"ok": True,
"user_id": "U1",
"team_id": "T1",
"user": "user1",
}
mock_users_slack_client.users_list.return_value = {
"ok": True,
"members": [
{"id": "U1", "name": "user1"},
{"id": "U2", "name": "user2"},
{"id": "U3", "name": "user3"},
{"id": "U4", "name": "user4"},
{"id": "U5", "name": "user5"},
],
"response_metadata": {"next_cursor": None},
}
conversation = {
"id": "C12345",
"type": ConversationTypeSlackName.MPIM.value,
"is_mpim": True,
"members": ["U1", "U4", "U5"],
}
mock_retrieve_conversations_by_user_ids.return_value = [conversation]
response = await get_multi_person_dm_conversation_metadata_by_usernames(
context=mock_context, usernames=["user1", "user4", "user5"]
)
assert response == conversation
mock_retrieve_conversations_by_user_ids.assert_called_once_with(
list_conversations_func=list_conversations_metadata,
get_members_in_conversation_func=get_members_in_conversation_by_id,
context=mock_context,
conversation_types=[ConversationType.MULTI_PERSON_DIRECT_MESSAGE],
user_ids=["U1", "U4", "U5"],
exact_match=True,
limit=1,
next_cursor=None,
)
@pytest.mark.asyncio
@patch("arcade_slack.tools.chat.retrieve_conversations_by_user_ids")
async def test_get_multi_person_direct_message_conversation_metadata_by_username_username_not_found(
mock_retrieve_conversations_by_user_ids,
mock_context,
mock_chat_slack_client,
mock_users_slack_client,
):
mock_chat_slack_client.users_identity.return_value = {
"ok": True,
"user": {"id": "U1", "name": "user1"},
"team": {"id": "T1", "name": "team1"},
}
mock_users_slack_client.users_list.return_value = {
"ok": True,
"members": [
{"id": "U1", "name": "user1"},
{"id": "U2", "name": "user2"},
],
"response_metadata": {"next_cursor": None},
}
mock_retrieve_conversations_by_user_ids.side_effect = TimeoutError()
with pytest.raises(RetryableToolError):
await get_multi_person_dm_conversation_metadata_by_usernames(
context=mock_context, usernames=["user999", "user1", "user2"]
)
@pytest.mark.asyncio
@patch("arcade_slack.tools.chat.get_messages_in_conversation_by_id")
@patch("arcade_slack.tools.chat.get_multi_person_dm_conversation_metadata_by_usernames")
async def test_get_messages_in_multi_person_dm_conversation_by_usernames(
mock_get_multi_person_dm_conversation_metadata_by_usernames,
mock_get_messages_in_conversation_by_id,
mock_context,
):
mock_get_multi_person_dm_conversation_metadata_by_usernames.return_value = {
"id": "C12345",
}
response = await get_messages_in_multi_person_dm_conversation_by_usernames(
context=mock_context, usernames=["user1", "user4", "user5"]
)
assert response == mock_get_messages_in_conversation_by_id.return_value
mock_get_multi_person_dm_conversation_metadata_by_usernames.assert_called_once_with(
context=mock_context, usernames=["user1", "user4", "user5"]
)
mock_get_messages_in_conversation_by_id.assert_called_once_with(
context=mock_context,
conversation_id="C12345",
oldest_relative=None,
latest_relative=None,
oldest_datetime=None,
latest_datetime=None,
limit=None,
next_cursor=None,
)
@pytest.mark.asyncio
@patch("arcade_slack.tools.chat.get_multi_person_dm_conversation_metadata_by_usernames")
async def test_get_messages_in_multi_person_dm_conversation_by_usernames_not_found(
mock_get_multi_person_dm_conversation_metadata_by_usernames,
mock_context,
):
mock_get_multi_person_dm_conversation_metadata_by_usernames.return_value = None
with pytest.raises(ToolExecutionError):
await get_messages_in_direct_message_conversation_by_username(
context=mock_context, username="user2"
)