6bc6f0c574
12 tools for the new Confluence toolkit. | Name | Description | |----------------------------------|------------------------------------------------------------------------------------| | Confluence.CreatePage | Create a new page at the root of the given space. | | Confluence.UpdatePageContent | Update a page's content. | | Confluence.RenamePage | Rename a page by changing its title. | | Confluence.GetPage | Retrieve a SINGLE page's content by its ID or title. | | Confluence.GetPagesById | Get the content of MULTIPLE pages by their ID in a single efficient request. | | Confluence.ListPages | Get the content of multiple pages by their ID. | | Confluence.ListAttachments | List attachments in a workspace. | | Confluence.GetAttachmentsForPage | Get attachments for a page by its ID or title. | | Confluence.SearchContent | Search for content in Confluence. | | Confluence.GetSpace | Get the details of a space by its ID or key. | | Confluence.ListSpaces | List all spaces sorted by name in ascending order. | | Confluence.GetSpaceHierarchy | Retrieve the full hierarchical structure of a Confluence space as a tree structure.| ### Confluence Clients Confluence has deprecated most of their V1 endpoints, so most of the tools use V2. However, we still need a V1 client to support search. The V1 search API has not been deprecated yet, because there is no V2 equivalent. But we need to be aware in the future for when this deprecation happens. ### Future work * Content of pages are returned in the Confluence `storage` format. This is the format that Confluence uses to store pages internally. We should understand the storage format more deeply, and write utility function to transform this format into plain text. * Better protections against extremely large pages. I've tested up to 6,000 word pages. * Tools for blog posts * Tool for getting the children of a page. (`get_space_hierarchy` will suffice for now) * Allow for numerical titles
113 lines
3.2 KiB
Python
113 lines
3.2 KiB
Python
from arcade.sdk import ToolCatalog
|
|
from arcade.sdk.eval import (
|
|
EvalRubric,
|
|
EvalSuite,
|
|
ExpectedToolCall,
|
|
tool_eval,
|
|
)
|
|
from arcade.sdk.eval.critic import BinaryCritic
|
|
|
|
import arcade_confluence
|
|
from arcade_confluence.tools import (
|
|
get_space,
|
|
get_space_hierarchy,
|
|
list_spaces,
|
|
)
|
|
from evals.conversations import list_spaces_1
|
|
|
|
# Evaluation rubric
|
|
rubric = EvalRubric(
|
|
fail_threshold=0.85,
|
|
warn_threshold=0.95,
|
|
)
|
|
|
|
|
|
catalog = ToolCatalog()
|
|
catalog.add_module(arcade_confluence)
|
|
|
|
|
|
@tool_eval()
|
|
def confluence_get_space_eval_suite() -> EvalSuite:
|
|
suite = EvalSuite(
|
|
name="Confluence get_space tool evaluation",
|
|
system_message="You are an AI assistant with access to Confluence tools.",
|
|
catalog=catalog,
|
|
rubric=rubric,
|
|
)
|
|
|
|
suite.add_case(
|
|
name="Get two spaces - one by ID, and one by key",
|
|
user_message="Get spaces 3498573 and 'Poetry'",
|
|
expected_tool_calls=[
|
|
ExpectedToolCall(func=get_space, args={"space_identifier": 3498573}),
|
|
ExpectedToolCall(func=get_space, args={"space_identifier": "Poetry"}),
|
|
],
|
|
rubric=rubric,
|
|
critics=[BinaryCritic(critic_field="space_identifier", weight=1)],
|
|
)
|
|
|
|
return suite
|
|
|
|
|
|
@tool_eval()
|
|
def confluence_list_spaces_eval_suite() -> EvalSuite:
|
|
suite = EvalSuite(
|
|
name="Confluence list_spaces tool evaluation",
|
|
system_message="You are an AI assistant with access to Confluence tools.",
|
|
catalog=catalog,
|
|
rubric=rubric,
|
|
)
|
|
|
|
suite.add_case(
|
|
name="List the next space using a pagination token",
|
|
user_message="get the next one",
|
|
expected_tool_calls=[
|
|
ExpectedToolCall(
|
|
func=list_spaces,
|
|
args={
|
|
"limit": 1,
|
|
"pagination_token": "eyJpZCI6MjMyNjc5NCwic3BhY2VTb3J0T3JkZXIiOnsiZmllbGQiOiJOQU1FIiwiZGlyZWN0aW9uIjoiQVNDRU5ESU5HIn0sInNwYWNlU29ydE9yZGVyVmFsdWUiOiJlcmljYXJjYWRlIn0=", # noqa: E501
|
|
},
|
|
),
|
|
],
|
|
rubric=rubric,
|
|
critics=[
|
|
BinaryCritic(critic_field="limit", weight=0.3),
|
|
BinaryCritic(critic_field="pagination_token", weight=0.7),
|
|
],
|
|
additional_messages=list_spaces_1,
|
|
)
|
|
|
|
return suite
|
|
|
|
|
|
@tool_eval()
|
|
def confluence_get_space_hierarchy_eval_suite() -> EvalSuite:
|
|
suite = EvalSuite(
|
|
name="Confluence get_space_hierarchy tool evaluation",
|
|
system_message="You are an AI assistant with access to Confluence tools.",
|
|
catalog=catalog,
|
|
rubric=rubric,
|
|
)
|
|
|
|
suite.add_case(
|
|
name="Get the hierarchy of a space",
|
|
user_message=(
|
|
"What is the best file location within my 'Poetry' space to create a new page "
|
|
"named 'Rough Draft - Poem for King Henry VIII'?"
|
|
),
|
|
expected_tool_calls=[
|
|
ExpectedToolCall(
|
|
func=get_space_hierarchy,
|
|
args={
|
|
"space_identifier": "Poetry",
|
|
},
|
|
),
|
|
],
|
|
rubric=rubric,
|
|
critics=[
|
|
BinaryCritic(critic_field="space_identifier", weight=1),
|
|
],
|
|
)
|
|
|
|
return suite
|