30739dc44a
Adds `Jira.GetAvailableAtlassianClouds` tool, which provides a list of clouds available (checking which Clouds were actually authorized by the current auth token). Refactors the interface of every tool to accept an `atlassian_cloud_id` argument (when not provided, try to get a unique cloud ID - if multiple are available, raises a Retryable error with the list of Clouds available instructing to select one). Gets rid of all caching. Now storing the global semaphore to the context object. The global semaphore is important because some tools depend on others, and each tool instantiates its own Jira HTTP client. Storing the semaphore in the context object ensures that all HTTP clients will respect the concurrency limit. Removes from tool responses the Atlassian URLs linking to objects in the Jira GUI (users, projects, issues, etc. We do not keep track of the cloud name anymore, which is required to build the objects' URLs. Extends/refactors unit tests accordingly. Evals checking LLM behavior when: - a cloud ID is explicitly mentioned in the prompt; - no cloud ID is mentioned; - a "multiple clouds available" error is raised and the user is prompted to pick one; - user request triggers another tool call after having previously picked a cloud ID (in the same chat context);
291 lines
10 KiB
Python
291 lines
10 KiB
Python
import json
|
|
import uuid
|
|
|
|
from arcade_evals import (
|
|
EvalRubric,
|
|
EvalSuite,
|
|
ExpectedToolCall,
|
|
tool_eval,
|
|
)
|
|
from arcade_evals.critic import BinaryCritic
|
|
from arcade_tdk import ToolCatalog
|
|
|
|
import arcade_jira
|
|
from arcade_jira.tools.comments import get_issue_comments
|
|
from arcade_jira.tools.issues import get_issue_by_id
|
|
|
|
# Evaluation rubric
|
|
rubric = EvalRubric(
|
|
fail_threshold=0.85,
|
|
warn_threshold=0.95,
|
|
)
|
|
|
|
|
|
catalog = ToolCatalog()
|
|
catalog.add_module(arcade_jira)
|
|
|
|
|
|
@tool_eval()
|
|
def multi_cloud_eval_suite() -> EvalSuite:
|
|
suite = EvalSuite(
|
|
name="Atlassian multi-cloud evaluation suite",
|
|
system_message=(
|
|
"You are an AI assistant with access to Jira tools. "
|
|
"Use them to help the user with their tasks."
|
|
),
|
|
catalog=catalog,
|
|
rubric=rubric,
|
|
)
|
|
|
|
suite.add_case(
|
|
name="Test calling tool without specifying a cloud id",
|
|
user_message="Get the issue with ID '10000'.",
|
|
expected_tool_calls=[
|
|
ExpectedToolCall(
|
|
func=get_issue_by_id,
|
|
args={
|
|
"issue_id": "10000",
|
|
"atlassian_cloud_id": None,
|
|
},
|
|
),
|
|
],
|
|
rubric=rubric,
|
|
critics=[
|
|
BinaryCritic(critic_field="issue_id", weight=0.5),
|
|
BinaryCritic(critic_field="atlassian_cloud_id", weight=0.5),
|
|
],
|
|
)
|
|
|
|
cloud_id = str(uuid.uuid4())
|
|
|
|
suite.add_case(
|
|
name="Test calling tool specifying a cloud id directly with the request",
|
|
user_message=f"Get the issue with ID '10000' in the Cloud with ID '{cloud_id}'.",
|
|
expected_tool_calls=[
|
|
ExpectedToolCall(
|
|
func=get_issue_by_id,
|
|
args={
|
|
"issue_id": "10000",
|
|
"atlassian_cloud_id": cloud_id,
|
|
},
|
|
),
|
|
],
|
|
rubric=rubric,
|
|
critics=[
|
|
BinaryCritic(critic_field="issue_id", weight=0.5),
|
|
BinaryCritic(critic_field="atlassian_cloud_id", weight=0.5),
|
|
],
|
|
)
|
|
|
|
cloud_1_id = str(uuid.uuid4())
|
|
cloud_2_id = str(uuid.uuid4())
|
|
available_clouds = [
|
|
{
|
|
"atlassian_cloud_id": cloud_1_id,
|
|
"atlassian_cloud_name": "Foobar",
|
|
"atlassian_cloud_url": "https://foobar.atlassian.com",
|
|
},
|
|
{
|
|
"atlassian_cloud_id": cloud_2_id,
|
|
"atlassian_cloud_name": "Quick Brown Fox",
|
|
"atlassian_cloud_url": "https://quickbrownfox.atlassian.com",
|
|
},
|
|
]
|
|
available_clouds_str = json.dumps(available_clouds)
|
|
|
|
suite.add_case(
|
|
name="Test calling tool with multiple clouds error and specifying which cloud to use",
|
|
user_message="Let's use the Foobar Cloud",
|
|
expected_tool_calls=[
|
|
ExpectedToolCall(
|
|
func=get_issue_by_id,
|
|
args={
|
|
"issue_id": "10000",
|
|
"atlassian_cloud_id": cloud_1_id,
|
|
},
|
|
),
|
|
],
|
|
rubric=rubric,
|
|
critics=[
|
|
BinaryCritic(critic_field="issue_id", weight=0.5),
|
|
BinaryCritic(critic_field="atlassian_cloud_id", weight=0.5),
|
|
],
|
|
additional_messages=[
|
|
{"role": "user", "content": "Get the issue with id '10000' in Jira"},
|
|
{
|
|
"role": "assistant",
|
|
"content": "",
|
|
"tool_calls": [
|
|
{
|
|
"id": "call_1",
|
|
"type": "function",
|
|
"function": {
|
|
"name": "Jira_GetIssueById",
|
|
"arguments": json.dumps({
|
|
"issue": "10000",
|
|
}),
|
|
},
|
|
}
|
|
],
|
|
},
|
|
{
|
|
"role": "tool",
|
|
"content": json.dumps({
|
|
"name": "retryable_tool_call_error",
|
|
"message": (
|
|
"Multiple Atlassian Clouds are available. One Cloud ID has to be selected "
|
|
"and provided in the tool call using the `atlassian_cloud_id` argument.",
|
|
),
|
|
"developer_message": (
|
|
"Multiple Atlassian Clouds are available. One Cloud ID has to be selected "
|
|
"and provided in the tool call using the `atlassian_cloud_id` argument.",
|
|
),
|
|
"additional_prompt_content": (
|
|
f"Available Atlassian Clouds:\n\n```json\n{available_clouds_str}\n```"
|
|
),
|
|
}),
|
|
"tool_call_id": "call_1",
|
|
"name": "Jira_GetIssueById",
|
|
},
|
|
{
|
|
"role": "assistant",
|
|
"content": (
|
|
"Here is the list of available Atlassian clouds:\n\n"
|
|
"1. **Name:** Foobar\n"
|
|
" - **URL:** https://foobar.atlassian.com\n"
|
|
"2. **Name:** Quick Brown Fox\n"
|
|
" - **URL:** https://quickbrownfox.atlassian.com\n"
|
|
"Please select one of the above Clouds to get the Jira issue."
|
|
),
|
|
},
|
|
],
|
|
)
|
|
|
|
suite.add_case(
|
|
name="Test calling tool one interaction after specifying a cloud id",
|
|
user_message="Get the comments on this issue",
|
|
expected_tool_calls=[
|
|
ExpectedToolCall(
|
|
func=get_issue_comments,
|
|
args={
|
|
"issue": "10000",
|
|
"atlassian_cloud_id": cloud_1_id,
|
|
},
|
|
),
|
|
],
|
|
rubric=rubric,
|
|
critics=[
|
|
BinaryCritic(critic_field="issue", weight=0.5),
|
|
BinaryCritic(critic_field="atlassian_cloud_id", weight=0.5),
|
|
],
|
|
additional_messages=[
|
|
{"role": "user", "content": "Get the issue with id '10000' in Jira"},
|
|
{
|
|
"role": "assistant",
|
|
"content": "",
|
|
"tool_calls": [
|
|
{
|
|
"id": "call_1",
|
|
"type": "function",
|
|
"function": {
|
|
"name": "Jira_GetIssueById",
|
|
"arguments": json.dumps({
|
|
"issue": "10000",
|
|
}),
|
|
},
|
|
}
|
|
],
|
|
},
|
|
{
|
|
"role": "tool",
|
|
"content": json.dumps({
|
|
"name": "retryable_tool_call_error",
|
|
"message": (
|
|
"Multiple Atlassian Clouds are available. One Cloud ID has to be selected "
|
|
"and provided in the tool call using the `atlassian_cloud_id` argument.",
|
|
),
|
|
"developer_message": (
|
|
"Multiple Atlassian Clouds are available. One Cloud ID has to be selected "
|
|
"and provided in the tool call using the `atlassian_cloud_id` argument.",
|
|
),
|
|
"additional_prompt_content": (
|
|
f"Available Atlassian Clouds:\n\n```json\n{available_clouds_str}\n```"
|
|
),
|
|
}),
|
|
"tool_call_id": "call_1",
|
|
"name": "Jira_GetIssueById",
|
|
},
|
|
{
|
|
"role": "assistant",
|
|
"content": (
|
|
"Here is the list of available Atlassian clouds:\n\n"
|
|
"1. **Name:** Foobar\n"
|
|
" - **URL:** https://foobar.atlassian.com\n"
|
|
"2. **Name:** Quick Brown Fox\n"
|
|
" - **URL:** https://quickbrownfox.atlassian.com\n"
|
|
"Please select one of the above Clouds to get the Jira issue."
|
|
),
|
|
},
|
|
{"role": "user", "content": "Let's use the Foobar Cloud from now on."},
|
|
{
|
|
"role": "assistant",
|
|
"content": "",
|
|
"tool_calls": [
|
|
{
|
|
"id": "call_2",
|
|
"type": "function",
|
|
"function": {
|
|
"name": "Jira_GetIssueById",
|
|
"arguments": json.dumps({
|
|
"issue": "10000",
|
|
"atlassian_cloud_id": cloud_1_id,
|
|
}),
|
|
},
|
|
}
|
|
],
|
|
},
|
|
{
|
|
"role": "tool",
|
|
"content": json.dumps({
|
|
"id": "10000",
|
|
"key": "ENG-101",
|
|
"assignee": {
|
|
"id": "10010",
|
|
"name": "John Doe",
|
|
"email": "john.doe@example.com",
|
|
},
|
|
"description": "Implement the message queue",
|
|
"status": {
|
|
"id": "10020",
|
|
"name": "In Progress",
|
|
},
|
|
"issuetype": {
|
|
"id": "10030",
|
|
"name": "Task",
|
|
},
|
|
"project": {
|
|
"id": "10040",
|
|
"key": "ENG",
|
|
"name": "Engineering",
|
|
},
|
|
}),
|
|
"tool_call_id": "call_2",
|
|
"name": "Jira_GetIssueById",
|
|
},
|
|
{
|
|
"role": "assistant",
|
|
"content": (
|
|
"Here is the issue:\n\n"
|
|
"1. **ID:** 10000\n"
|
|
" - **Key:** ENG-101\n"
|
|
" - **Assignee:** John Doe\n"
|
|
" - **Description:** Implement the message queue\n"
|
|
" - **Status:** In Progress\n"
|
|
" - **Issue Type:** Task\n"
|
|
" - **Project:** Engineering"
|
|
),
|
|
},
|
|
],
|
|
)
|
|
|
|
return suite
|