6bc6f0c574
12 tools for the new Confluence toolkit. | Name | Description | |----------------------------------|------------------------------------------------------------------------------------| | Confluence.CreatePage | Create a new page at the root of the given space. | | Confluence.UpdatePageContent | Update a page's content. | | Confluence.RenamePage | Rename a page by changing its title. | | Confluence.GetPage | Retrieve a SINGLE page's content by its ID or title. | | Confluence.GetPagesById | Get the content of MULTIPLE pages by their ID in a single efficient request. | | Confluence.ListPages | Get the content of multiple pages by their ID. | | Confluence.ListAttachments | List attachments in a workspace. | | Confluence.GetAttachmentsForPage | Get attachments for a page by its ID or title. | | Confluence.SearchContent | Search for content in Confluence. | | Confluence.GetSpace | Get the details of a space by its ID or key. | | Confluence.ListSpaces | List all spaces sorted by name in ascending order. | | Confluence.GetSpaceHierarchy | Retrieve the full hierarchical structure of a Confluence space as a tree structure.| ### Confluence Clients Confluence has deprecated most of their V1 endpoints, so most of the tools use V2. However, we still need a V1 client to support search. The V1 search API has not been deprecated yet, because there is no V2 equivalent. But we need to be aware in the future for when this deprecation happens. ### Future work * Content of pages are returned in the Confluence `storage` format. This is the format that Confluence uses to store pages internally. We should understand the storage format more deeply, and write utility function to transform this format into plain text. * Better protections against extremely large pages. I've tested up to 6,000 word pages. * Tools for blog posts * Tool for getting the children of a page. (`get_space_hierarchy` will suffice for now) * Allow for numerical titles
111 lines
3.1 KiB
Python
111 lines
3.1 KiB
Python
from arcade.sdk import ToolCatalog
|
|
from arcade.sdk.eval import (
|
|
EvalRubric,
|
|
EvalSuite,
|
|
ExpectedToolCall,
|
|
tool_eval,
|
|
)
|
|
from arcade.sdk.eval.critic import BinaryCritic
|
|
|
|
import arcade_confluence
|
|
from arcade_confluence.tools import (
|
|
search_content,
|
|
)
|
|
from evals.critics import ListCritic
|
|
|
|
# Evaluation rubric
|
|
rubric = EvalRubric(
|
|
fail_threshold=0.85,
|
|
warn_threshold=0.95,
|
|
)
|
|
|
|
|
|
catalog = ToolCatalog()
|
|
catalog.add_module(arcade_confluence)
|
|
|
|
|
|
@tool_eval()
|
|
def confluence_search_eval_suite() -> EvalSuite:
|
|
suite = EvalSuite(
|
|
name="Confluence search content tool evaluation",
|
|
system_message="You are an AI assistant with access to Confluence tools.",
|
|
catalog=catalog,
|
|
rubric=rubric,
|
|
)
|
|
|
|
suite.add_case(
|
|
name="Search for content - easy case",
|
|
user_message="Find all pages that contain 'Arcade.dev'",
|
|
expected_tool_calls=[
|
|
ExpectedToolCall(func=search_content, args={"can_contain_any": ["Arcade.dev"]})
|
|
],
|
|
rubric=rubric,
|
|
critics=[
|
|
BinaryCritic(critic_field="can_contain_any", weight=1),
|
|
],
|
|
)
|
|
|
|
suite.add_case(
|
|
name="Search for content - medium case",
|
|
user_message=("Find 20 pages that contain 'Arcade' or 'AI', or 'tool calls'"),
|
|
expected_tool_calls=[
|
|
ExpectedToolCall(
|
|
func=search_content,
|
|
args={
|
|
"can_contain_any": ["Arcade", "AI", "tool calls"],
|
|
"limit": 20,
|
|
},
|
|
)
|
|
],
|
|
rubric=rubric,
|
|
critics=[
|
|
ListCritic(
|
|
critic_field="can_contain_any",
|
|
weight=0.9,
|
|
case_sensitive=False,
|
|
order_matters=False,
|
|
),
|
|
BinaryCritic(critic_field="limit", weight=0.1),
|
|
],
|
|
)
|
|
|
|
suite.add_case(
|
|
name="Search for content - hard case",
|
|
user_message=(
|
|
"Look for 25 databases that have 'How to', "
|
|
"and also have 'carborator' in the content and "
|
|
"also has one of the following: 'money', 'redbull gives you wings', "
|
|
"'honey hole', 'don't snap the pasta!'."
|
|
),
|
|
expected_tool_calls=[
|
|
ExpectedToolCall(
|
|
func=search_content,
|
|
args={
|
|
"must_contain_all": ["carborator", "How to"],
|
|
"can_contain_any": [
|
|
"money",
|
|
"redbull gives you wings",
|
|
"honey hole",
|
|
"don't snap the pasta!",
|
|
],
|
|
},
|
|
),
|
|
],
|
|
rubric=rubric,
|
|
critics=[
|
|
ListCritic(
|
|
critic_field="must_contain_all",
|
|
weight=0.5,
|
|
case_sensitive=False,
|
|
order_matters=False,
|
|
),
|
|
ListCritic(
|
|
critic_field="can_contain_any",
|
|
weight=0.5,
|
|
case_sensitive=False,
|
|
order_matters=False,
|
|
),
|
|
],
|
|
)
|
|
|
|
return suite
|