b6b4cd0a4c
### Overview Major restructuring from monolithic `arcade-ai` package to modular library architecture with standardized uv-based dependency management.  ### New Package Structure - **`arcade-tdk`** - Lightweight toolkit development kit (core decorators, auth) - **`arcade-core`** - Core execution engine and catalog functionality - **`arcade-serve`** - FastAPI/MCP server components - **`arcade-ai`** - Meta package that includes CLI functionality. Optionally include evals via the `evals` extra. Optionally include all packages via the `all` extra. ### Key Benefits - **Lighter Dependencies**: Toolkits now depend only on `arcade-tdk` (~2 deps) vs full `arcade-ai` (~30+ deps) - **Faster Builds**: uv provides 10-100x faster dependency resolution and installation - **Better Modularity**: Clear separation of concerns, consumers import only what they need - **Standard Tooling**: Eliminates custom poetry scripts, uses standard Python packaging ### Migration Impact - All 20 toolkits converted from poetry → uv with `arcade-tdk` dependencies plus `arcade-ai[evals]` and `arcade-serve` dev dependencies. When developing locally, devs should install toolkits via `make install-local`. - Modern Python 3.10+ type hints throughout - Standardized build system with hatchling backend - Enhanced Makefile with robust toolkit management commands - Removed `arcade dev` CLI command - Reduce the number of files created by `arcade new` and add an option to not generate a tests and evals folder. This foundation enables faster development cycles and cleaner dependency chains for the growing toolkit ecosystem. ### Todo After this PR is merged - [ ] Post-merge workflow(s) (release & publish containers, etc) - [ ] Release order plan. @EricGustin suggests releasing in the following order: 1. `arcade-core` version 0.1.0 2. `arcade-serve` version 0.1.0 and `arcade-tdk` version 0.1.0 3. `arcade-ai` version 2.0.0 4. Patch release for all toolkits (all changes in toolkits are internal refactors) - [ ] [Update docs](https://github.com/ArcadeAI/docs/pull/318) --------- Co-authored-by: Eric Gustin <eric@arcade.dev> Co-authored-by: Eric Gustin <34000337+EricGustin@users.noreply.github.com>
219 lines
8.6 KiB
Python
219 lines
8.6 KiB
Python
from arcade_evals import (
|
||
BinaryCritic,
|
||
EvalRubric,
|
||
EvalSuite,
|
||
ExpectedToolCall,
|
||
tool_eval,
|
||
)
|
||
from arcade_tdk import ToolCatalog
|
||
|
||
import arcade_x
|
||
from arcade_x.tools.tweets import (
|
||
delete_tweet_by_id,
|
||
lookup_tweet_by_id,
|
||
post_tweet,
|
||
search_recent_tweets_by_keywords,
|
||
search_recent_tweets_by_username,
|
||
)
|
||
from arcade_x.tools.users import lookup_single_user_by_username
|
||
|
||
# Evaluation rubric
|
||
rubric = EvalRubric(
|
||
fail_threshold=0.7,
|
||
warn_threshold=0.9,
|
||
)
|
||
|
||
catalog = ToolCatalog()
|
||
# Register the X tools
|
||
catalog.add_module(arcade_x)
|
||
|
||
search_recent_tweets_by_username_history = [
|
||
{"role": "user", "content": "list 1 tweet from elonmusk"},
|
||
{
|
||
"role": "assistant",
|
||
"content": "",
|
||
"tool_calls": [
|
||
{
|
||
"id": "call_kineaPbYCAof3n6qCwnYSKBb",
|
||
"type": "function",
|
||
"function": {
|
||
"name": "X_SearchRecentTweetsByUsername",
|
||
"arguments": '{"max_results":1,"username":"elonmusk"}',
|
||
},
|
||
}
|
||
],
|
||
},
|
||
{
|
||
"role": "tool",
|
||
"content": '{"data":[{"author_id":"44196397","author_name":"Elon Musk","author_username":"elonmusk","edit_history_tweet_ids":["1866572304320466985"],"id":"1866572304320466985","text":"RT @chamath: Meanwhile the State of California is going to spend almost double this ($35B) to build a 171 mile stretch of rail between Merc…","tweet_url":"https://x.com/x/status/1866572304320466985"},{"author_id":"44196397","edit_history_tweet_ids":["1866571568266219998"],"id":"1866571568266219998","text":"This is awesome 🚀🇺🇸 https://twitter.com/cb_doge/status/1866565984502550905","tweet_url":"https://x.com/x/status/1866571568266219998"},{"author_id":"44196397","edit_history_tweet_ids":["1866571416969285954"],"id":"1866571416969285954","text":"@ajtourville @Tesla I’ve always felt that the climate predictions were too pessimistic and bound to backfire. \\n\\nExtreme environmentalists can’t say ridiculous things like the world is doomed in 5 years, because 5 years goes by, the world is ok and they lose credibility. \\n\\nIf we transition to… https://x.com/i/web/status/1866571416969285954","tweet_url":"https://x.com/x/status/1866571416969285954"},{"author_id":"44196397","edit_history_tweet_ids":["1866569957309603946"],"id":"1866569957309603946","text":"@shaunmmaguire Yes, please. This is gone on for too long. Enough.","tweet_url":"https://x.com/x/status/1866569957309603946"},{"author_id":"44196397","edit_history_tweet_ids":["1866569078539948491"],"id":"1866569078539948491","text":"@FatEmperor 😂","tweet_url":"https://x.com/x/status/1866569078539948491"},{"author_id":"44196397","edit_history_tweet_ids":["1866554579925577793"],"id":"1866554579925577793","text":"@cb_doge I’m not buying or building a house anywhere","tweet_url":"https://x.com/x/status/1866554579925577793"},{"author_id":"44196397","edit_history_tweet_ids":["1866536009833361915"],"id":"1866536009833361915","text":"RT @amuse: http://x.com/i/article/1866500805211123713","tweet_url":"https://x.com/x/status/1866536009833361915"},{"author_id":"44196397","edit_history_tweet_ids":["1866535704924483739"],"id":"1866535704924483739","text":"@benshapiro 😂","tweet_url":"https://x.com/x/status/1866535704924483739"},{"author_id":"44196397","edit_history_tweet_ids":["1866535550632550854"],"id":"1866535550632550854","text":"@AutismCapital 😂","tweet_url":"https://x.com/x/status/1866535550632550854"},{"author_id":"44196397","edit_history_tweet_ids":["1866535352024043804"],"id":"1866535352024043804","text":"@JDVance Yes","tweet_url":"https://x.com/x/status/1866535352024043804"}],"includes":{"users":[{"id":"44196397","name":"Elon Musk","username":"elonmusk"}]},"meta":{"newest_id":"1866572304320466985","next_token":"b26v89c19zqg8o3frr3tekall7a7ooom3sctaw30rz62l","oldest_id":"1866535352024043804","result_count":10}}', # noqa: RUF001
|
||
"tool_call_id": "call_kineaPbYCAof3n6qCwnYSKBb",
|
||
"name": "X_SearchRecentTweetsByUsername",
|
||
},
|
||
{
|
||
"role": "assistant",
|
||
"content": 'Here is a recent tweet from Elon Musk: \n\n"This is awesome 🚀🇺🇸" - [Tweet link](https://x.com/x/status/1866571568266219998)',
|
||
},
|
||
]
|
||
|
||
|
||
@tool_eval()
|
||
def x_eval_suite() -> EvalSuite:
|
||
"""Evaluation suite for X (Twitter) tools."""
|
||
|
||
suite = EvalSuite(
|
||
name="X Tools Evaluation Suite",
|
||
system_message=(
|
||
"You are an AI assistant with access to the X (Twitter) tools. Use them to "
|
||
"help answer the user's X-related tasks/questions."
|
||
),
|
||
catalog=catalog,
|
||
rubric=rubric,
|
||
)
|
||
|
||
# Add cases
|
||
suite.add_case(
|
||
name="Post a tweet",
|
||
user_message=(
|
||
"Send out a tweet that says 'Hello World! Exciting stuff is happening over "
|
||
"at Arcade AI!'"
|
||
),
|
||
expected_tool_calls=[
|
||
ExpectedToolCall(
|
||
func=post_tweet,
|
||
args={"tweet_text": "Hello World! Exciting stuff is happening over at Arcade AI!"},
|
||
)
|
||
],
|
||
critics=[
|
||
BinaryCritic(
|
||
critic_field="tweet_text",
|
||
weight=1.0,
|
||
),
|
||
],
|
||
)
|
||
|
||
suite.add_case(
|
||
name="Delete a tweet by ID",
|
||
user_message="Please delete the tweet with ID '148975632'.",
|
||
expected_tool_calls=[
|
||
ExpectedToolCall(
|
||
func=delete_tweet_by_id,
|
||
args={"tweet_id": "148975632"},
|
||
)
|
||
],
|
||
critics=[
|
||
BinaryCritic(
|
||
critic_field="tweet_id",
|
||
weight=1.0,
|
||
),
|
||
],
|
||
)
|
||
|
||
suite.add_case(
|
||
name="Search recent tweets by username",
|
||
user_message="Show me the recent tweets from 'elonmusk'.",
|
||
expected_tool_calls=[
|
||
ExpectedToolCall(
|
||
func=search_recent_tweets_by_username,
|
||
args={"username": "elonmusk", "max_results": 10},
|
||
)
|
||
],
|
||
critics=[
|
||
BinaryCritic(
|
||
critic_field="username",
|
||
weight=1.0,
|
||
),
|
||
],
|
||
)
|
||
|
||
suite.add_case(
|
||
name="Search recent tweets by username with history",
|
||
user_message="Get the next 42",
|
||
additional_messages=search_recent_tweets_by_username_history,
|
||
expected_tool_calls=[
|
||
ExpectedToolCall(
|
||
func=search_recent_tweets_by_username,
|
||
args={
|
||
"username": "elonmusk",
|
||
"max_results": 42,
|
||
"next_token": "b26v89c19zqg8o3frr3tekall7a7ooom3sctaw30rz62l",
|
||
},
|
||
),
|
||
],
|
||
critics=[
|
||
BinaryCritic(
|
||
critic_field="username",
|
||
weight=0.2,
|
||
),
|
||
BinaryCritic(
|
||
critic_field="max_results",
|
||
weight=0.2,
|
||
),
|
||
BinaryCritic(
|
||
critic_field="next_token",
|
||
weight=0.6,
|
||
),
|
||
],
|
||
)
|
||
|
||
suite.add_case(
|
||
name="Lookup user by username",
|
||
user_message="Can you get information about the user '@jack'?",
|
||
expected_tool_calls=[
|
||
ExpectedToolCall(
|
||
func=lookup_single_user_by_username,
|
||
args={"username": "jack"},
|
||
),
|
||
],
|
||
critics=[
|
||
BinaryCritic(
|
||
critic_field="username",
|
||
weight=1.0,
|
||
),
|
||
],
|
||
)
|
||
|
||
# Add a case for searching recent tweets by keywords
|
||
suite.add_case(
|
||
name="Search recent tweets by keywords",
|
||
user_message="Find recent tweets containing 'Arcade AI'.",
|
||
expected_tool_calls=[
|
||
ExpectedToolCall(
|
||
func=search_recent_tweets_by_keywords,
|
||
args={
|
||
"keywords": None,
|
||
"phrases": ["Arcade AI"],
|
||
"max_results": 10,
|
||
},
|
||
),
|
||
],
|
||
critics=[
|
||
BinaryCritic(
|
||
critic_field="keywords",
|
||
weight=0.1,
|
||
),
|
||
BinaryCritic(
|
||
critic_field="phrases",
|
||
weight=0.9,
|
||
),
|
||
],
|
||
)
|
||
|
||
# Extend the case to test lookup_tweet_by_id
|
||
suite.extend_case(
|
||
name="Lookup tweet by ID",
|
||
user_message="Can you provide details about the tweet with ID '123456789'?",
|
||
expected_tool_calls=[
|
||
ExpectedToolCall(
|
||
func=lookup_tweet_by_id,
|
||
args={"tweet_id": "123456789"},
|
||
),
|
||
],
|
||
critics=[
|
||
BinaryCritic(
|
||
critic_field="tweet_id",
|
||
weight=1.0,
|
||
),
|
||
],
|
||
)
|
||
|
||
return suite
|