Updates to Notion toolkit (#476)
### Adds one new tool to the Notion toolkit * **AppendContentToEndOfPage**: Append markdown content to the end of a Notion page by its ID or title ### Fixes a bug * The **CreatePage** would previously fail if the content provided was broken into more than 100 blocks. I added 'chunking' logic to support > 100 blocks.
This commit is contained in:
Eric Gustin
GitHub
parent
173cc6994e
commit
4598e1e4d4
9 changed files with 291 additions and 69 deletions
|
|
@ -12,6 +12,8 @@ ENDPOINTS = {
|
|||
"create_comment": "/comments",
|
||||
"retrieve_a_page": "/pages/{page_id}",
|
||||
"retrieve_a_block": "/blocks/{block_id}",
|
||||
"delete_a_block": "/blocks/{block_id}",
|
||||
"update_a_block": "/blocks/{block_id}",
|
||||
}
|
||||
|
||||
UNTITLED_TITLE = "New Page"
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
from arcade_notion_toolkit.tools.pages import (
|
||||
append_content_to_end_of_page,
|
||||
create_page,
|
||||
get_page_content_by_id,
|
||||
get_page_content_by_title,
|
||||
|
|
@ -10,10 +11,11 @@ from arcade_notion_toolkit.tools.search import (
|
|||
)
|
||||
|
||||
__all__ = [
|
||||
"append_content_to_end_of_page",
|
||||
"create_page",
|
||||
"get_object_metadata",
|
||||
"get_page_content_by_id",
|
||||
"get_page_content_by_title",
|
||||
"search_by_title",
|
||||
"get_workspace_structure",
|
||||
"search_by_title",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -15,7 +15,9 @@ from arcade_notion_toolkit.utils import (
|
|||
extract_title,
|
||||
get_headers,
|
||||
get_next_page,
|
||||
get_page_url,
|
||||
get_url,
|
||||
is_page_id,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -136,12 +138,19 @@ async def create_page(
|
|||
else:
|
||||
properties = PageWithPageParentProperties(title=title).to_dict()
|
||||
|
||||
children = convert_markdown_to_blocks(content) if content else None
|
||||
children = convert_markdown_to_blocks(content) if content else []
|
||||
|
||||
# Split children into chunks of 100 due to Notion API limit
|
||||
chunk_size = 100
|
||||
first_chunk = children[:chunk_size] if children else []
|
||||
remaining_chunks = [
|
||||
children[i : i + chunk_size] for i in range(chunk_size, len(children), chunk_size)
|
||||
]
|
||||
|
||||
body = {
|
||||
"parent": parent.to_dict(),
|
||||
"properties": properties,
|
||||
"children": children,
|
||||
"children": first_chunk,
|
||||
}
|
||||
|
||||
url = get_url("create_a_page")
|
||||
|
|
@ -149,4 +158,55 @@ async def create_page(
|
|||
async with httpx.AsyncClient() as client:
|
||||
response = await client.post(url, headers=headers, json=body)
|
||||
response.raise_for_status()
|
||||
return f"Successfully created page with ID: {response.json()['id']}"
|
||||
page_id = response.json()["id"]
|
||||
|
||||
# Append remaining chunks if any
|
||||
if remaining_chunks:
|
||||
append_url = get_url("append_block_children", block_id=page_id)
|
||||
for chunk in remaining_chunks:
|
||||
chunk_body = {"children": chunk}
|
||||
append_response = await client.patch(append_url, headers=headers, json=chunk_body)
|
||||
append_response.raise_for_status()
|
||||
|
||||
return f"Successfully created page with ID: {page_id}"
|
||||
|
||||
|
||||
@tool(requires_auth=Notion())
|
||||
async def append_content_to_end_of_page(
|
||||
context: ToolContext,
|
||||
page_id_or_title: Annotated[str, "ID or title of the page to append content to"],
|
||||
content: Annotated[str, "The markdown content to append to the end of the page"],
|
||||
) -> Annotated[dict[str, str], "A dictionary containing a success message and the URL to the page"]:
|
||||
"""Append markdown content to the end of a Notion page by its ID or title"""
|
||||
# Determine if the provided identifier is an ID or a title
|
||||
page_id = page_id_or_title
|
||||
if not is_page_id(page_id_or_title):
|
||||
page_metadata = await get_object_metadata(
|
||||
context,
|
||||
object_title=page_id_or_title,
|
||||
object_type=ObjectType.PAGE,
|
||||
)
|
||||
page_id = page_metadata["id"]
|
||||
|
||||
headers = get_headers(context)
|
||||
# the Notion API endpoint conveniently also accepts page ID for the block_id path parameter
|
||||
url = get_url("append_block_children", block_id=page_id)
|
||||
|
||||
children = convert_markdown_to_blocks(content)
|
||||
|
||||
# Split children into chunks of 100 due to Notion API limit
|
||||
chunk_size = 100
|
||||
async with httpx.AsyncClient() as client:
|
||||
for i in range(0, len(children), chunk_size):
|
||||
chunk = children[i : i + chunk_size]
|
||||
body = {"children": chunk}
|
||||
|
||||
response = await client.patch(url, headers=headers, json=body)
|
||||
response.raise_for_status()
|
||||
|
||||
page_url = await get_page_url(context, page_id)
|
||||
|
||||
return {
|
||||
"message": f"Successfully appended content to page with ID: {page_id}",
|
||||
"url": page_url,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
import httpx
|
||||
from arcade_tdk import ToolContext
|
||||
|
|
@ -6,6 +7,25 @@ from arcade_tdk import ToolContext
|
|||
from arcade_notion_toolkit.constants import ENDPOINTS, NOTION_API_URL, UNTITLED_TITLE
|
||||
|
||||
|
||||
def is_page_id(candidate: str) -> bool:
|
||||
"""
|
||||
Determine if the provided candidate string has the structure of a valid Notion page ID.
|
||||
Page IDs are UUID values.
|
||||
|
||||
Args:
|
||||
candidate (str): The candidate string to check.
|
||||
|
||||
Returns:
|
||||
bool: True if the candidate has the structure of a valid Notion page ID, False otherwise.
|
||||
"""
|
||||
try:
|
||||
UUID(candidate)
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def get_url(endpoint: str, **kwargs: Any) -> str:
|
||||
"""
|
||||
Constructs the full URL for a specified notion endpoint.
|
||||
|
|
|
|||
65
toolkits/notion/evals/constants.py
Normal file
65
toolkits/notion/evals/constants.py
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
SMALL_PAGE_CONTENT = """## Why Build Tools with Arcade?
|
||||
|
||||
Arcade solves key challenges for agent developers:
|
||||
|
||||
1. **Auth Native to Agents**: Authentication designed for agentic workflows — the right token is always available for each user without complex integration work.
|
||||
|
||||
2. **Multi-Tenant Tool Calling**: Enable your agent to take actions AS the specific user of the agent
|
||||
|
||||
3. **Better Agent Capabilities**: Build tools that securely connect to the services your users want your agent to integrate with (Gmail, Slack, Google Drive, Zoom, etc.) without complex integration code.
|
||||
|
||||
4. **Clean Codebase**: Eliminate environment variables full of API keys and complex OAuth implementations from your application code.
|
||||
|
||||
5. **Flexible Integration**: Choose your integration approach:
|
||||
|
||||
- LLM API for the simplest experience with hundreds of pre-built tools
|
||||
- Tools API for direct execution control
|
||||
- Auth API for authentication-only integration
|
||||
- Framework connectors for LangChain, CrewAI and others
|
||||
|
||||
6. **Zero Schema Maintenance**: Tool definitions generate automatically from code annotations and translate to any LLM format.
|
||||
|
||||
7. **Built-in Evaluation**: Evaluate your tools across user scenarios, llms, and context with Arcade's tool calling evaluation framework. Ensure your tools are working as expected and are useful for your agents.
|
||||
|
||||
8. **Complete Tooling Ecosystem**: Built-in evaluation framework, scalable execution infrastructure, and flexible deployment options (including VPC, Docker, and Kubernetes).
|
||||
|
||||
Arcade lets you focus on creating useful tool functionality rather than solving complex authentication, deployment, and integration challenges.
|
||||
""" # noqa: E501
|
||||
|
||||
# A conversation where a user asks the AI to get the content of a page named 'Arcade Notes'
|
||||
GET_SMALL_PAGE_CONTENT_CONVERSATION = [
|
||||
{"role": "user", "content": "get 'Arcade Notes'"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_92Vhl75I8KEKQfjihS7l53DL",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "Notion_GetPageContentByTitle",
|
||||
"arguments": '{"title":"Arcade Notes"}',
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"content": "# Arcade Notes\nFirst, make sure you have these pre-requisites installed on your system: \n- **Python 3.10**\xa0or higherVerify your Python version by running\xa0`python --version`\xa0or\xa0`python3 --version`\xa0in your terminal. \n- **pip**: The Python package installer should be available. It's typically included with Python. \n- **Arcade Account**: Sign up for an\xa0[Arcade account](https://api.arcade.dev/signup?utm_source=docs&utm_medium=page&utm_campaign=custom-tools)\xa0if you haven't already. \nLet's set up Arcade and give it a try! \n### **Obtain an API key** \nInstall the Arcade CLI and SDK and log in. Your Arcade API key will be printed to the console as well as saved to\xa0`~/.arcade/credentials.yaml`. \npip install arcade-aiarcade login\n### **Try\xa0**`arcade chat` \nWith Arcade CLI installed, you can test outour API with the\xa0`arcade chat`\xa0command: \narcade chat\nThis launches a chat with the Arcade Cloud Engine (hosted at\xa0`api.arcade.dev`). All pre-built Arcade tools are available to use. \nFor example, try asking: \nstar the ArcadeAI/arcade-ai repo on Github\nArcade will ask you to authorize with GitHub, and then the AI assistant will star the\xa0[ArcadeAI/arcade-ai](https://github.com/ArcadeAI/arcade-ai)\xa0repo on your behalf. \nYou'll see output similar to this: \nAssistant (gpt-4o):I starred the ArcadeAI/arcade-ai repo on Github for you!\nYou can use Ctrl-C to exit the chat at any time. \n### **Try more tools** \nTry prompting the AI with different requests: \n- Search the web to find when The Winds of Winter will be released \n- Summarize my emails from today \n- What's playing on my Spotify right now? \nTo see all tools available in the Cloud, run: \narcade show\nGreat job! 🎉 You've set up Arcade and used your first tool. \n### **Next steps** \n- [Learn about tools](https://docs.arcade.dev/home/use-tools/tools-overview)\xa0andhow they work \n- Learn how to\xa0[call tools with models](https://docs.arcade.dev/home/use-tools/call-tools-with-models) \n- Or, jump ahead to building your own\xa0[custom tools](https://docs.arcade.dev/home/build-tools/create-a-toolkit) \n", # noqa: E501
|
||||
"tool_call_id": "call_92Vhl75I8KEKQfjihS7l53DL",
|
||||
"name": "Notion_GetPageContentByTitle",
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "Here are the notes on setting up and using Arcade:\n\n### Pre-requisites\nMake sure you have the following installed on your system:\n-**Python 3.10 or higher**: Verify using `python --version` or `python3 --version` in your terminal.\n- **pip**: The Python package installer, typically included with Python.\n- **Arcade Account**: Sign up for an [Arcade account](https://api.arcade.dev/signup?utm_source=docs&utm_medium=page&utm_campaign=custom-tools) if you haven't already.\n\n### Setting up Arcade\n\n#### Obtain an API key\n1. Install the Arcade CLI and SDK.\n2. Log in to Arcade: Run the command `arcade login`.\n3. Your Arcade API key will be printed to the console and saved in `~/.arcade/credentials.yaml`.\n\n#### Try `arcade chat`\n- With Arcade CLI installed, you can test the API using `arcade chat`.\n- This command launches a chat with the Arcade Cloud Engine (hosted at `api.arcade.dev`), giving you access to all pre-built Arcade tools.\n \nExample prompt: \n- Ask to star the `ArcadeAI/arcade-ai` repo on GitHub. The AI assistant will star the repo for you after authorization.\n\nEvent example output:\n```\nAssistant (gpt-4o): I starred the ArcadeAI/arcade-ai repo on Github for you!\n```\n\nUse `Ctrl-C` to exit the chat anytime.\n\n### Explore More Tools\n- Try different requests like searching the web, summarizing emails, or checking your current Spotify play.\n- Run `arcade show` to see all available tools in the Cloud.\n\n### Next Steps\n- [Learn about tools](https://docs.arcade.dev/home/use-tools/tools-overview) and how they function.\n- Understand how to [call tools with models](https://docs.arcade.dev/home/use-tools/call-tools-with-models).\n- Begin building your own [custom tools](https://docs.arcade.dev/home/build-tools/create-a-toolkit).\n\nCongratulations on setting up Arcade and trying your first tool! 🎉", # noqa: E501
|
||||
},
|
||||
]
|
||||
|
||||
SMALL_PAGE_CONTENT_TO_APPEND = """
|
||||
# Next steps
|
||||
Here are the next steps
|
||||
1. Add ability to append to the end of a page
|
||||
2. Make it Pythonic
|
||||
## Next next steps
|
||||
**Write** [evals](https://github.com/ArcadeAI/arcade-ai)
|
||||
"""
|
||||
|
|
@ -10,10 +10,16 @@ from arcade_tdk import ToolCatalog
|
|||
|
||||
import arcade_notion_toolkit
|
||||
from arcade_notion_toolkit.tools import (
|
||||
append_content_to_end_of_page,
|
||||
create_page,
|
||||
get_page_content_by_id,
|
||||
get_page_content_by_title,
|
||||
)
|
||||
from evals.constants import (
|
||||
GET_SMALL_PAGE_CONTENT_CONVERSATION,
|
||||
SMALL_PAGE_CONTENT,
|
||||
SMALL_PAGE_CONTENT_TO_APPEND,
|
||||
)
|
||||
|
||||
# Evaluation rubric
|
||||
rubric = EvalRubric(
|
||||
|
|
@ -25,63 +31,6 @@ rubric = EvalRubric(
|
|||
catalog = ToolCatalog()
|
||||
catalog.add_module(arcade_notion_toolkit)
|
||||
|
||||
PAGE_CONTENT = """## Why Build Tools with Arcade?
|
||||
|
||||
Arcade solves key challenges for agent developers:
|
||||
|
||||
1. **Auth Native to Agents**: Authentication designed for agentic workflows — the right token is always available for each user without complex integration work.
|
||||
|
||||
2. **Multi-Tenant Tool Calling**: Enable your agent to take actions AS the specific user of the agent
|
||||
|
||||
3. **Better Agent Capabilities**: Build tools that securely connect to the services your users want your agent to integrate with (Gmail, Slack, Google Drive, Zoom, etc.) without complex integration code.
|
||||
|
||||
4. **Clean Codebase**: Eliminate environment variables full of API keys and complex OAuth implementations from your application code.
|
||||
|
||||
5. **Flexible Integration**: Choose your integration approach:
|
||||
|
||||
- LLM API for the simplest experience with hundreds of pre-built tools
|
||||
- Tools API for direct execution control
|
||||
- Auth API for authentication-only integration
|
||||
- Framework connectors for LangChain, CrewAI and others
|
||||
|
||||
6. **Zero Schema Maintenance**: Tool definitions generate automatically from code annotations and translate to any LLM format.
|
||||
|
||||
7. **Built-in Evaluation**: Evaluate your tools across user scenarios, llms, and context with Arcade's tool calling evaluation framework. Ensure your tools are working as expected and are useful for your agents.
|
||||
|
||||
8. **Complete Tooling Ecosystem**: Built-in evaluation framework, scalable execution infrastructure, and flexible deployment options (including VPC, Docker, and Kubernetes).
|
||||
|
||||
Arcade lets you focus on creating useful tool functionality rather than solving complex authentication, deployment, and integration challenges.
|
||||
""" # noqa: E501
|
||||
|
||||
# A conversation where a user asks the AI to get the content of a page named 'Arcade Notes'
|
||||
GET_PAGE_CONVERSATION = [
|
||||
{"role": "user", "content": "get 'Arcade Notes'"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_92Vhl75I8KEKQfjihS7l53DL",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "Notion_GetPageContentByTitle",
|
||||
"arguments": '{"title":"Arcade Notes"}',
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"content": "# Arcade Notes\nFirst, make sure you have these pre-requisites installed on your system: \n- **Python 3.10**\xa0or higherVerify your Python version by running\xa0`python --version`\xa0or\xa0`python3 --version`\xa0in your terminal. \n- **pip**: The Python package installer should be available. It's typically included with Python. \n- **Arcade Account**: Sign up for an\xa0[Arcade account](https://api.arcade.dev/signup?utm_source=docs&utm_medium=page&utm_campaign=custom-tools)\xa0if you haven't already. \nLet's set up Arcade and give it a try! \n### **Obtain an API key** \nInstall the Arcade CLI and SDK and log in. Your Arcade API key will be printed to the console as well as saved to\xa0`~/.arcade/credentials.yaml`. \npip install arcade-aiarcade login\n### **Try\xa0**`arcade chat` \nWith Arcade CLI installed, you can test outour API with the\xa0`arcade chat`\xa0command: \narcade chat\nThis launches a chat with the Arcade Cloud Engine (hosted at\xa0`api.arcade.dev`). All pre-built Arcade tools are available to use. \nFor example, try asking: \nstar the ArcadeAI/arcade-ai repo on Github\nArcade will ask you to authorize with GitHub, and then the AI assistant will star the\xa0[ArcadeAI/arcade-ai](https://github.com/ArcadeAI/arcade-ai)\xa0repo on your behalf. \nYou'll see output similar to this: \nAssistant (gpt-4o):I starred the ArcadeAI/arcade-ai repo on Github for you!\nYou can use Ctrl-C to exit the chat at any time. \n### **Try more tools** \nTry prompting the AI with different requests: \n- Search the web to find when The Winds of Winter will be released \n- Summarize my emails from today \n- What's playing on my Spotify right now? \nTo see all tools available in the Cloud, run: \narcade show\nGreat job! 🎉 You've set up Arcade and used your first tool. \n### **Next steps** \n- [Learn about tools](https://docs.arcade.dev/home/use-tools/tools-overview)\xa0andhow they work \n- Learn how to\xa0[call tools with models](https://docs.arcade.dev/home/use-tools/call-tools-with-models) \n- Or, jump ahead to building your own\xa0[custom tools](https://docs.arcade.dev/home/build-tools/create-a-toolkit) \n", # noqa: E501
|
||||
"tool_call_id": "call_92Vhl75I8KEKQfjihS7l53DL",
|
||||
"name": "Notion_GetPageContentByTitle",
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "Here are the notes on setting up and using Arcade:\n\n### Pre-requisites\nMake sure you have the following installed on your system:\n-**Python 3.10 or higher**: Verify using `python --version` or `python3 --version` in your terminal.\n- **pip**: The Python package installer, typically included with Python.\n- **Arcade Account**: Sign up for an [Arcade account](https://api.arcade.dev/signup?utm_source=docs&utm_medium=page&utm_campaign=custom-tools) if you haven't already.\n\n### Setting up Arcade\n\n#### Obtain an API key\n1. Install the Arcade CLI and SDK.\n2. Log in to Arcade: Run the command `arcade login`.\n3. Your Arcade API key will be printed to the console and saved in `~/.arcade/credentials.yaml`.\n\n#### Try `arcade chat`\n- With Arcade CLI installed, you can test the API using `arcade chat`.\n- This command launches a chat with the Arcade Cloud Engine (hosted at `api.arcade.dev`), giving you access to all pre-built Arcade tools.\n \nExample prompt: \n- Ask to star the `ArcadeAI/arcade-ai` repo on GitHub. The AI assistant will star the repo for you after authorization.\n\nEvent example output:\n```\nAssistant (gpt-4o): I starred the ArcadeAI/arcade-ai repo on Github for you!\n```\n\nUse `Ctrl-C` to exit the chat anytime.\n\n### Explore More Tools\n- Try different requests like searching the web, summarizing emails, or checking your current Spotify play.\n- Run `arcade show` to see all available tools in the Cloud.\n\n### Next Steps\n- [Learn about tools](https://docs.arcade.dev/home/use-tools/tools-overview) and how they function.\n- Understand how to [call tools with models](https://docs.arcade.dev/home/use-tools/call-tools-with-models).\n- Begin building your own [custom tools](https://docs.arcade.dev/home/build-tools/create-a-toolkit).\n\nCongratulations on setting up Arcade and trying your first tool! 🎉", # noqa: E501
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
@tool_eval()
|
||||
def create_page_eval_suite() -> EvalSuite:
|
||||
|
|
@ -124,7 +73,7 @@ def create_page_eval_suite() -> EvalSuite:
|
|||
suite.add_case(
|
||||
name="Create page medium difficulty",
|
||||
user_message=(
|
||||
f"Create a page with the title 'Why Use Arcade?' and content {PAGE_CONTENT}"
|
||||
f"Create a page with the title 'Why Use Arcade?' and content {SMALL_PAGE_CONTENT}"
|
||||
"under the parent page 'Arcade Notes'."
|
||||
),
|
||||
expected_tool_calls=[
|
||||
|
|
@ -133,7 +82,7 @@ def create_page_eval_suite() -> EvalSuite:
|
|||
args={
|
||||
"parent_title": "Arcade Notes",
|
||||
"title": "Why Use Arcade?",
|
||||
"content": PAGE_CONTENT,
|
||||
"content": SMALL_PAGE_CONTENT,
|
||||
},
|
||||
),
|
||||
],
|
||||
|
|
@ -147,14 +96,14 @@ def create_page_eval_suite() -> EvalSuite:
|
|||
# Hard case
|
||||
suite.add_case(
|
||||
name="Create page hard difficulty",
|
||||
user_message=(f"Add {PAGE_CONTENT} as a subpage. Name it 'Why Use Arcade?'"),
|
||||
user_message=(f"Add {SMALL_PAGE_CONTENT} as a subpage. Name it 'Why Use Arcade?'"),
|
||||
expected_tool_calls=[
|
||||
ExpectedToolCall(
|
||||
func=create_page,
|
||||
args={
|
||||
"parent_title": "Arcade Notes",
|
||||
"title": "Why Use Arcade?",
|
||||
"content": PAGE_CONTENT,
|
||||
"content": SMALL_PAGE_CONTENT,
|
||||
},
|
||||
),
|
||||
],
|
||||
|
|
@ -163,7 +112,7 @@ def create_page_eval_suite() -> EvalSuite:
|
|||
SimilarityCritic(critic_field="title", weight=0.33, similarity_threshold=0.95),
|
||||
SimilarityCritic(critic_field="content", weight=0.33, similarity_threshold=0.95),
|
||||
],
|
||||
additional_messages=GET_PAGE_CONVERSATION,
|
||||
additional_messages=GET_SMALL_PAGE_CONTENT_CONVERSATION,
|
||||
)
|
||||
return suite
|
||||
|
||||
|
|
@ -249,7 +198,41 @@ def get_page_content_eval_suite() -> EvalSuite:
|
|||
critics=[
|
||||
BinaryCritic(critic_field="title", weight=1),
|
||||
],
|
||||
additional_messages=GET_PAGE_CONVERSATION,
|
||||
additional_messages=GET_SMALL_PAGE_CONTENT_CONVERSATION,
|
||||
)
|
||||
|
||||
return suite
|
||||
|
||||
|
||||
@tool_eval()
|
||||
def append_page_content_eval_suite() -> EvalSuite:
|
||||
"""Create an evaluation suite for tools appending content to an existing Notion page"""
|
||||
suite = EvalSuite(
|
||||
name="Notion Append Content To End Of Page",
|
||||
system_message=(
|
||||
"You are an AI assistant that has access to the user's Notion workspace. "
|
||||
"You can take actions on the user's Notion workspace on behalf of the user."
|
||||
),
|
||||
catalog=catalog,
|
||||
rubric=rubric,
|
||||
)
|
||||
|
||||
suite.add_case(
|
||||
name="Append page content",
|
||||
user_message=f"Add this to the end of that page:\n{SMALL_PAGE_CONTENT_TO_APPEND}",
|
||||
expected_tool_calls=[
|
||||
ExpectedToolCall(
|
||||
func=append_content_to_end_of_page,
|
||||
args={
|
||||
"page_id_or_title": "Arcade Notes",
|
||||
"content": SMALL_PAGE_CONTENT_TO_APPEND,
|
||||
},
|
||||
),
|
||||
],
|
||||
critics=[
|
||||
BinaryCritic(critic_field="page_id_or_title", weight=0.5),
|
||||
SimilarityCritic(critic_field="content", weight=0.5, similarity_threshold=0.95),
|
||||
],
|
||||
additional_messages=GET_SMALL_PAGE_CONTENT_CONVERSATION,
|
||||
)
|
||||
return suite
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|||
|
||||
[project]
|
||||
name = "arcade_notion_toolkit"
|
||||
version = "0.1.6"
|
||||
version = "0.2.0"
|
||||
description = "Arcade.dev LLM tools for Notion"
|
||||
requires-python = ">=3.10"
|
||||
dependencies = [
|
||||
|
|
|
|||
|
|
@ -167,3 +167,80 @@ async def test_get_page_content_by_id_nested(mock_context, monkeypatch, setup_no
|
|||
result = await pages.get_page_content_by_id(mock_context, "test_nested")
|
||||
expected = "# Test Nested\nParent Block \n Child Block \n"
|
||||
assert result == expected
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_append_content_to_end_of_page_with_large_content(
|
||||
mock_context, monkeypatch, setup_notion_pages
|
||||
):
|
||||
pages = setup_notion_pages
|
||||
|
||||
# Mock is_page_id to return True
|
||||
monkeypatch.setattr(pages, "is_page_id", lambda x: True)
|
||||
|
||||
# Create 150 dummy blocks (more than the 100 chunk size)
|
||||
dummy_blocks = []
|
||||
for i in range(150):
|
||||
dummy_blocks.append({
|
||||
"object": "block",
|
||||
"type": "paragraph",
|
||||
"paragraph": {"rich_text": [{"type": "text", "text": {"content": f"Block {i}"}}]},
|
||||
})
|
||||
|
||||
# Mock convert_markdown_to_blocks to return our 150 blocks
|
||||
def fake_convert_markdown_to_blocks(content):
|
||||
return dummy_blocks
|
||||
|
||||
monkeypatch.setattr(pages, "convert_markdown_to_blocks", fake_convert_markdown_to_blocks)
|
||||
|
||||
# Mock get_page_url to return a dummy URL
|
||||
async def fake_get_page_url(context, page_id):
|
||||
return f"https://notion.so/page/{page_id}"
|
||||
|
||||
monkeypatch.setattr(pages, "get_page_url", fake_get_page_url)
|
||||
|
||||
# Track the HTTP requests
|
||||
request_count = 0
|
||||
request_payloads = []
|
||||
|
||||
class MockResponse:
|
||||
def raise_for_status(self):
|
||||
pass
|
||||
|
||||
class MockClient:
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *args):
|
||||
pass
|
||||
|
||||
async def patch(self, url, headers, json):
|
||||
nonlocal request_count
|
||||
request_count += 1
|
||||
request_payloads.append(json)
|
||||
return MockResponse()
|
||||
|
||||
# Mock httpx.AsyncClient
|
||||
monkeypatch.setattr(pages.httpx, "AsyncClient", MockClient)
|
||||
|
||||
_ = await pages.append_content_to_end_of_page(
|
||||
mock_context, "test_page_id", "Large content that will be converted to 150 blocks"
|
||||
)
|
||||
|
||||
# Verify chunking behavior: 150 blocks should be split into 2 requests
|
||||
# First request: 100 blocks (0-99)
|
||||
# Second request: 50 blocks (100-149)
|
||||
assert request_count == 2
|
||||
assert len(request_payloads) == 2
|
||||
|
||||
# Verify first chunk has 100 blocks
|
||||
first_chunk = request_payloads[0]["children"]
|
||||
assert len(first_chunk) == 100
|
||||
assert first_chunk[0]["paragraph"]["rich_text"][0]["text"]["content"] == "Block 0"
|
||||
assert first_chunk[99]["paragraph"]["rich_text"][0]["text"]["content"] == "Block 99"
|
||||
|
||||
# Verify second chunk has 50 blocks
|
||||
second_chunk = request_payloads[1]["children"]
|
||||
assert len(second_chunk) == 50
|
||||
assert second_chunk[0]["paragraph"]["rich_text"][0]["text"]["content"] == "Block 100"
|
||||
assert second_chunk[49]["paragraph"]["rich_text"][0]["text"]["content"] == "Block 149"
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import pytest
|
||||
|
||||
from arcade_notion_toolkit.utils import simplify_search_result
|
||||
from arcade_notion_toolkit.utils import is_page_id, simplify_search_result
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
@ -84,3 +84,16 @@ def test_simplify_search_result(item, expected_title):
|
|||
assert simplified["last_edited_time"] == item.get("last_edited_time")
|
||||
assert simplified["url"] == item.get("url")
|
||||
assert simplified["public_url"] == item.get("public_url")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"candidate, expected_result",
|
||||
[
|
||||
("1ae7a62b04d480cd8f30fe64b5354cc0", True),
|
||||
("1b37a62b-04d4-8079-a902-ce69ed7e7240", True),
|
||||
("1b37a62b04d48079ad7e7240", False),
|
||||
("OAuth 2 In Plain English", False),
|
||||
],
|
||||
)
|
||||
def test_is_page_id(candidate, expected_result):
|
||||
assert is_page_id(candidate) == expected_result
|
||||
|
|
|
|||
Loading…
Reference in a new issue