30739dc44a
Adds `Jira.GetAvailableAtlassianClouds` tool, which provides a list of clouds available (checking which Clouds were actually authorized by the current auth token). Refactors the interface of every tool to accept an `atlassian_cloud_id` argument (when not provided, try to get a unique cloud ID - if multiple are available, raises a Retryable error with the list of Clouds available instructing to select one). Gets rid of all caching. Now storing the global semaphore to the context object. The global semaphore is important because some tools depend on others, and each tool instantiates its own Jira HTTP client. Storing the semaphore in the context object ensures that all HTTP clients will respect the concurrency limit. Removes from tool responses the Atlassian URLs linking to objects in the Jira GUI (users, projects, issues, etc. We do not keep track of the cloud name anymore, which is required to build the objects' URLs. Extends/refactors unit tests accordingly. Evals checking LLM behavior when: - a cloud ID is explicitly mentioned in the prompt; - no cloud ID is mentioned; - a "multiple clouds available" error is raised and the user is prompted to pick one; - user request triggers another tool call after having previously picked a cloud ID (in the same chat context);
185 lines
5.5 KiB
Python
185 lines
5.5 KiB
Python
from collections.abc import Callable
|
|
|
|
import pytest
|
|
from arcade_tdk import ToolContext
|
|
|
|
from arcade_jira.exceptions import MultipleItemsFoundError, NotFoundError
|
|
from arcade_jira.utils import (
|
|
clean_user_dict,
|
|
find_multiple_unique_users,
|
|
find_unique_user,
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_find_unique_user_by_id_success(
|
|
mock_context: ToolContext,
|
|
mock_httpx_client,
|
|
mock_httpx_response: Callable,
|
|
build_user_dict: Callable,
|
|
):
|
|
sample_user = build_user_dict()
|
|
user_response = mock_httpx_response(200, sample_user)
|
|
mock_httpx_client.get.return_value = user_response
|
|
|
|
response = await find_unique_user(mock_context, sample_user["accountId"])
|
|
assert response == clean_user_dict(sample_user)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_find_unique_user_by_name_with_a_single_match(
|
|
mock_context: ToolContext,
|
|
mock_httpx_client,
|
|
mock_httpx_response: Callable,
|
|
build_user_dict: Callable,
|
|
):
|
|
sample_user = build_user_dict()
|
|
get_user_by_id_response = mock_httpx_response(404, {})
|
|
get_users_without_id_response = mock_httpx_response(200, [sample_user])
|
|
mock_httpx_client.get.side_effect = [get_user_by_id_response, get_users_without_id_response]
|
|
|
|
response = await find_unique_user(mock_context, sample_user["displayName"].lower())
|
|
assert response == clean_user_dict(sample_user)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_find_unique_user_by_name_with_multiple_matches(
|
|
mock_context: ToolContext,
|
|
mock_httpx_client,
|
|
mock_httpx_response: Callable,
|
|
build_user_dict: Callable,
|
|
generate_random_str: Callable,
|
|
):
|
|
user_name = generate_random_str()
|
|
sample_users = [
|
|
build_user_dict(display_name=user_name),
|
|
build_user_dict(display_name=user_name),
|
|
]
|
|
get_user_by_id_response = mock_httpx_response(404, {})
|
|
get_users_without_id_response = mock_httpx_response(200, sample_users)
|
|
mock_httpx_client.get.side_effect = [get_user_by_id_response, get_users_without_id_response]
|
|
|
|
with pytest.raises(MultipleItemsFoundError) as exc:
|
|
await find_unique_user(mock_context, sample_users[0]["displayName"].lower())
|
|
|
|
assert sample_users[0]["accountId"] in exc.value.message
|
|
assert sample_users[1]["accountId"] in exc.value.message
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_find_unique_user_by_name_without_matches(
|
|
mock_context: ToolContext,
|
|
mock_httpx_client,
|
|
mock_httpx_response: Callable,
|
|
generate_random_str: Callable,
|
|
):
|
|
get_user_by_id_response = mock_httpx_response(404, {})
|
|
get_users_without_id_response = mock_httpx_response(200, [])
|
|
mock_httpx_client.get.side_effect = [get_user_by_id_response, get_users_without_id_response]
|
|
|
|
with pytest.raises(NotFoundError):
|
|
await find_unique_user(mock_context, generate_random_str())
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_find_multiple_users_when_all_names_match_one_result(
|
|
mock_context: ToolContext,
|
|
mock_httpx_client,
|
|
mock_httpx_response: Callable,
|
|
build_user_dict: Callable,
|
|
):
|
|
user1 = build_user_dict()
|
|
user2 = build_user_dict()
|
|
|
|
mock_httpx_client.get.side_effect = [
|
|
mock_httpx_response(200, [user1]),
|
|
mock_httpx_response(200, [user2]),
|
|
]
|
|
|
|
response = await find_multiple_unique_users(
|
|
mock_context, [user1["displayName"], user2["displayName"]]
|
|
)
|
|
|
|
assert response == [
|
|
clean_user_dict(user1),
|
|
clean_user_dict(user2),
|
|
]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_find_multiple_users_when_a_name_match_multiple_results(
|
|
mock_context: ToolContext,
|
|
mock_httpx_client,
|
|
mock_httpx_response: Callable,
|
|
build_user_dict: Callable,
|
|
):
|
|
user1 = build_user_dict()
|
|
user2 = build_user_dict()
|
|
user3 = build_user_dict()
|
|
|
|
mock_httpx_client.get.side_effect = [
|
|
mock_httpx_response(200, [user1]),
|
|
mock_httpx_response(200, [user2, user3]),
|
|
]
|
|
|
|
with pytest.raises(MultipleItemsFoundError) as exc:
|
|
await find_multiple_unique_users(mock_context, [user1["displayName"], user2["displayName"]])
|
|
|
|
assert user2["accountId"] in exc.value.message
|
|
assert user3["accountId"] in exc.value.message
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_find_multiple_users_when_user_is_not_found_by_name_but_found_by_id(
|
|
mock_context: ToolContext,
|
|
mock_httpx_client,
|
|
mock_httpx_response: Callable,
|
|
build_user_dict: Callable,
|
|
):
|
|
user1 = build_user_dict()
|
|
user2 = build_user_dict()
|
|
|
|
mock_httpx_client.get.side_effect = [
|
|
mock_httpx_response(200, [user1]),
|
|
mock_httpx_response(200, []),
|
|
mock_httpx_response(200, user2),
|
|
]
|
|
|
|
response = await find_multiple_unique_users(
|
|
mock_context, [user1["displayName"], user2["accountId"]]
|
|
)
|
|
|
|
assert response == [
|
|
clean_user_dict(user1),
|
|
clean_user_dict(user2),
|
|
]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_find_multiple_users_when_various_users_are_not_found_by_name_but_found_by_id(
|
|
mock_context: ToolContext,
|
|
mock_httpx_client,
|
|
mock_httpx_response: Callable,
|
|
build_user_dict: Callable,
|
|
):
|
|
user1 = build_user_dict()
|
|
user2 = build_user_dict()
|
|
user3 = build_user_dict()
|
|
|
|
mock_httpx_client.get.side_effect = [
|
|
mock_httpx_response(200, [user1]),
|
|
mock_httpx_response(200, []),
|
|
mock_httpx_response(200, []),
|
|
mock_httpx_response(200, user2),
|
|
mock_httpx_response(200, user3),
|
|
]
|
|
|
|
response = await find_multiple_unique_users(
|
|
mock_context, [user1["displayName"], user2["accountId"], user3["accountId"]]
|
|
)
|
|
|
|
assert response == [
|
|
clean_user_dict(user1),
|
|
clean_user_dict(user2),
|
|
clean_user_dict(user3),
|
|
]
|