Update Evals SDK (#175)
# PR Description
This PR renames `ExpectedToolCall` to `NamedExpectedToolCall` and then
creates a new dataclass called `ExpectedToolCall`. `ExpectedToolCall`
can be passed to the `EvalSuite.add_case` and `EvalSuite.extend_case`
methods.
1. Enhance `EvalSuite.add_case` and `EvalSuite.extend_case` by accepting
a list of `ExpectedToolCall` as their `expected_tool_calls` input
parameter. This helps create a scaffolding for developers. Previously,
the expected type was `list[tuple[Callable, dict[str, Any]]]`, which is
still valid for backward compatibility.
```python
# Before (still valid for backward compatibility)
expected_tool_calls=[
(
adjust_playback_position,
{
"absolute_position_ms": 10000,
},
)
]
# After
expected_tool_calls=[
ExpectedToolCall(
func=adjust_playback_position,
args={"absolute_position_ms": 10000},
)
]
```
2. Removed any references to arcade.core in toolkits directory.
3. Some linting for import organization.
This commit is contained in:
Eric Gustin
GitHub
parent
a4b58d9749
commit
7c228a59d5
22 changed files with 461 additions and 306 deletions
|
|
@ -5,9 +5,9 @@ from arcade.core.toolkit import Toolkit
|
|||
from .tool import tool
|
||||
|
||||
__all__ = [
|
||||
"tool",
|
||||
"ToolAuthorizationContext",
|
||||
"ToolContext",
|
||||
"ToolCatalog",
|
||||
"ToolContext",
|
||||
"Toolkit",
|
||||
"tool",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -1,13 +1,14 @@
|
|||
from .critic import BinaryCritic, DatetimeCritic, NumericCritic, SimilarityCritic
|
||||
from .eval import EvalRubric, EvalSuite, ExpectedToolCall, tool_eval
|
||||
from .eval import EvalRubric, EvalSuite, ExpectedToolCall, NamedExpectedToolCall, tool_eval
|
||||
|
||||
__all__ = [
|
||||
"BinaryCritic",
|
||||
"SimilarityCritic",
|
||||
"NumericCritic",
|
||||
"DatetimeCritic",
|
||||
"EvalRubric",
|
||||
"EvalSuite",
|
||||
"ExpectedToolCall",
|
||||
"NamedExpectedToolCall",
|
||||
"NumericCritic",
|
||||
"SimilarityCritic",
|
||||
"tool_eval",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -28,7 +28,21 @@ if TYPE_CHECKING:
|
|||
@dataclass
|
||||
class ExpectedToolCall:
|
||||
"""
|
||||
Represents an expected tool call with its name and arguments.
|
||||
Represents an expected tool call with the function itself and arguments.
|
||||
|
||||
Attributes:
|
||||
func: The function itself.
|
||||
args: A dictionary containing the expected arguments for the tool.
|
||||
"""
|
||||
|
||||
func: Callable
|
||||
args: dict[str, Any]
|
||||
|
||||
|
||||
@dataclass
|
||||
class NamedExpectedToolCall:
|
||||
"""
|
||||
Represents a tool call with its name and arguments.
|
||||
|
||||
Attributes:
|
||||
name: The name of the tool.
|
||||
|
|
@ -151,7 +165,7 @@ class EvalCase:
|
|||
name: A descriptive name for this evaluation case.
|
||||
system_message: The system message to be sent to the AI model.
|
||||
user_message: The user input to be sent to the AI model.
|
||||
expected_tool_calls: A list of ExpectedToolCall objects representing the expected tool calls.
|
||||
expected_tool_calls: A list of NamedExpectedToolCall objects representing the expected tool calls.
|
||||
critics: A list of Critic objects used to evaluate tool arguments.
|
||||
additional_messages: Optional list of additional context messages.
|
||||
rubric: An EvalRubric object defining pass/fail criteria and tool selection behavior.
|
||||
|
|
@ -160,7 +174,7 @@ class EvalCase:
|
|||
name: str
|
||||
system_message: str
|
||||
user_message: str
|
||||
expected_tool_calls: list[ExpectedToolCall]
|
||||
expected_tool_calls: list[NamedExpectedToolCall]
|
||||
critics: list["Critic"] | None = None
|
||||
additional_messages: list[dict[str, str]] = field(default_factory=list)
|
||||
rubric: EvalRubric = field(default_factory=EvalRubric)
|
||||
|
|
@ -331,14 +345,14 @@ class EvalCase:
|
|||
def _create_cost_matrix(
|
||||
self,
|
||||
actual_tool_calls: list[tuple[str, dict[str, Any]]],
|
||||
expected_tool_calls: list[ExpectedToolCall],
|
||||
expected_tool_calls: list[NamedExpectedToolCall],
|
||||
) -> np.ndarray:
|
||||
"""
|
||||
Create a cost matrix for the assignment problem.
|
||||
|
||||
Args:
|
||||
actual_tool_calls: A list of tuples of actual tool calls.
|
||||
expected_tool_calls: A list of ExpectedToolCall instances.
|
||||
expected_tool_calls: A list of NamedExpectedToolCall instances.
|
||||
|
||||
Returns:
|
||||
A numpy array representing the cost matrix.
|
||||
|
|
@ -401,11 +415,33 @@ class EvalSuite:
|
|||
rubric: EvalRubric = field(default_factory=EvalRubric)
|
||||
max_concurrent: int = 1
|
||||
|
||||
def _convert_to_named_expected_tool_call(
|
||||
self, tc: ExpectedToolCall | tuple[Callable, dict[str, Any]]
|
||||
) -> NamedExpectedToolCall:
|
||||
"""
|
||||
Convert an ExpectedToolCall or a tuple to a NamedExpectedToolCall
|
||||
with default arguments populated.
|
||||
|
||||
Args:
|
||||
tc: The tool call, either as an ExpectedToolCall or a tuple.
|
||||
|
||||
Returns:
|
||||
A NamedExpectedToolCall instance.
|
||||
"""
|
||||
if isinstance(tc, tuple):
|
||||
func, args = tc
|
||||
else:
|
||||
func = tc.func
|
||||
args = tc.args
|
||||
args_with_defaults = self._fill_args_with_defaults(func, args)
|
||||
tool_name = str(self.catalog.find_tool_by_func(func).get_fully_qualified_name())
|
||||
return NamedExpectedToolCall(name=tool_name, args=args_with_defaults)
|
||||
|
||||
def add_case(
|
||||
self,
|
||||
name: str,
|
||||
user_message: str,
|
||||
expected_tool_calls: list[tuple[Callable, dict[str, Any]]],
|
||||
expected_tool_calls: list[ExpectedToolCall] | list[tuple[Callable, dict[str, Any]]],
|
||||
critics: list["Critic"] | None = None,
|
||||
system_message: str | None = None,
|
||||
rubric: EvalRubric | None = None,
|
||||
|
|
@ -417,24 +453,21 @@ class EvalSuite:
|
|||
Args:
|
||||
name: The name of the evaluation case.
|
||||
user_message: The user's input message.
|
||||
expected_tool_calls: A list of expected tool calls as tuples of (function, args).
|
||||
expected_tool_calls: A list of expected tool calls as ExpectedToolCall instances.
|
||||
critics: List of critics to evaluate the tool arguments.
|
||||
system_message: The system message to be used.
|
||||
rubric: The evaluation rubric for this case.
|
||||
additional_messages: Optional list of additional messages for context.
|
||||
"""
|
||||
expected = []
|
||||
for func, args in expected_tool_calls:
|
||||
# Fill in default arguments here
|
||||
args_with_defaults = self._fill_args_with_defaults(func, args)
|
||||
tool_name = str(self.catalog.find_tool_by_func(func).get_fully_qualified_name())
|
||||
expected.append(ExpectedToolCall(name=tool_name, args=args_with_defaults))
|
||||
expected_tool_calls_with_defaults = [
|
||||
self._convert_to_named_expected_tool_call(tc) for tc in expected_tool_calls
|
||||
]
|
||||
|
||||
case = EvalCase(
|
||||
name=name,
|
||||
system_message=system_message or self.system_message,
|
||||
user_message=user_message,
|
||||
expected_tool_calls=expected,
|
||||
expected_tool_calls=expected_tool_calls_with_defaults,
|
||||
rubric=rubric or self.rubric,
|
||||
critics=critics,
|
||||
additional_messages=additional_messages or [],
|
||||
|
|
@ -470,7 +503,9 @@ class EvalSuite:
|
|||
name: str,
|
||||
user_message: str,
|
||||
system_message: str | None = None,
|
||||
expected_tool_calls: list[tuple[Callable, dict[str, Any]]] | None = None,
|
||||
expected_tool_calls: list[ExpectedToolCall]
|
||||
| list[tuple[Callable, dict[str, Any]]]
|
||||
| None = None,
|
||||
rubric: EvalRubric | None = None,
|
||||
critics: list["Critic"] | None = None,
|
||||
additional_messages: list[dict[str, str]] | None = None,
|
||||
|
|
@ -502,12 +537,7 @@ class EvalSuite:
|
|||
|
||||
expected = last_case.expected_tool_calls
|
||||
if expected_tool_calls:
|
||||
expected = []
|
||||
for func, args in expected_tool_calls:
|
||||
# Fill in default arguments here
|
||||
args_with_defaults = self._fill_args_with_defaults(func, args)
|
||||
tool_name = str(self.catalog.find_tool_by_func(func).get_fully_qualified_name())
|
||||
expected.append(ExpectedToolCall(name=tool_name, args=args_with_defaults))
|
||||
expected = [self._convert_to_named_expected_tool_call(tc) for tc in expected_tool_calls]
|
||||
|
||||
# Create a new case, copying from the last one and updating fields
|
||||
new_case = EvalCase(
|
||||
|
|
|
|||
|
|
@ -1,19 +1,22 @@
|
|||
from datetime import timedelta
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import pytz
|
||||
from dateutil import parser
|
||||
|
||||
from arcade.sdk import tool
|
||||
from arcade.sdk.errors import WeightError
|
||||
from arcade.sdk.eval import (
|
||||
BinaryCritic,
|
||||
DatetimeCritic,
|
||||
EvalRubric,
|
||||
ExpectedToolCall,
|
||||
NamedExpectedToolCall,
|
||||
NumericCritic,
|
||||
SimilarityCritic,
|
||||
)
|
||||
from arcade.sdk.eval.eval import EvalCase, EvaluationResult
|
||||
from arcade.sdk.eval.eval import EvalCase, EvalSuite, EvaluationResult
|
||||
|
||||
# Test BinaryCritic.evaluate()
|
||||
|
||||
|
|
@ -142,8 +145,8 @@ def test_eval_case_evaluate():
|
|||
"""
|
||||
# Define expected tool calls and actual tool calls
|
||||
expected_tool_calls = [
|
||||
ExpectedToolCall(name="ToolA", args={"param": "value1"}),
|
||||
ExpectedToolCall(name="ToolB", args={"param": "value2"}),
|
||||
NamedExpectedToolCall(name="ToolA", args={"param": "value1"}),
|
||||
NamedExpectedToolCall(name="ToolB", args={"param": "value2"}),
|
||||
]
|
||||
actual_tool_calls = [
|
||||
("ToolA", {"param": "value1"}),
|
||||
|
|
@ -189,7 +192,7 @@ def test_eval_case_evaluate_mismatched_tools():
|
|||
the expected tool calls to ensure tool selection scoring is correct.
|
||||
"""
|
||||
expected_tool_calls = [
|
||||
ExpectedToolCall(name="ToolA", args={"param": "value"}),
|
||||
NamedExpectedToolCall(name="ToolA", args={"param": "value"}),
|
||||
]
|
||||
actual_tool_calls = [
|
||||
("ToolB", {"param": "value"}),
|
||||
|
|
@ -225,7 +228,7 @@ def test_eval_case_multiple_critics():
|
|||
to ensure individual critic scores are correctly combined into the total score.
|
||||
"""
|
||||
expected_tool_calls = [
|
||||
ExpectedToolCall(name="ToolA", args={"param1": "value1", "param2": "value2"}),
|
||||
NamedExpectedToolCall(name="ToolA", args={"param1": "value1", "param2": "value2"}),
|
||||
]
|
||||
actual_tool_calls = [
|
||||
("ToolA", {"param1": "value1", "param2": "wrong_value"}),
|
||||
|
|
@ -267,7 +270,7 @@ def test_eval_case_with_none_values():
|
|||
expected_args = {"param": None}
|
||||
actual_args = {"param": None}
|
||||
|
||||
expected_tool_calls = [ExpectedToolCall(name="ToolA", args=expected_args)]
|
||||
expected_tool_calls = [NamedExpectedToolCall(name="ToolA", args=expected_args)]
|
||||
actual_tool_calls = [("ToolA", actual_args)]
|
||||
|
||||
critics = [BinaryCritic(critic_field="param", weight=1.0)]
|
||||
|
|
@ -465,3 +468,104 @@ def test_datetime_critic_naive_and_timezone_aware():
|
|||
expected_score = critic.weight * ratio
|
||||
|
||||
assert pytest.approx(result["score"], abs=1e-6) == expected_score
|
||||
|
||||
|
||||
# Test EvalSuite.add_case()
|
||||
def test_eval_suite_add_case():
|
||||
"""
|
||||
Test that add_case correctly adds a new evaluation case to the suite.
|
||||
"""
|
||||
|
||||
@tool
|
||||
def mock_tool(param: str):
|
||||
pass
|
||||
|
||||
mock_catalog = Mock()
|
||||
mock_catalog.find_tool_by_func.return_value.get_fully_qualified_name.return_value = "MockTool"
|
||||
|
||||
suite = EvalSuite(name="TestSuite", system_message="System message", catalog=mock_catalog)
|
||||
|
||||
expected_tool_calls = [
|
||||
ExpectedToolCall(
|
||||
func=mock_tool,
|
||||
args={"param": "value"},
|
||||
),
|
||||
(
|
||||
mock_tool,
|
||||
{"param": "value"},
|
||||
),
|
||||
]
|
||||
|
||||
suite.add_case(
|
||||
name="TestCase",
|
||||
user_message="User message",
|
||||
expected_tool_calls=expected_tool_calls,
|
||||
)
|
||||
|
||||
assert len(suite.cases) == 1
|
||||
case = suite.cases[0]
|
||||
assert len(case.expected_tool_calls) == 2
|
||||
assert case.name == "TestCase"
|
||||
assert case.user_message == "User message"
|
||||
assert case.system_message == "System message"
|
||||
assert case.expected_tool_calls[0] == NamedExpectedToolCall(
|
||||
name="MockTool", args={"param": "value"}
|
||||
)
|
||||
assert case.expected_tool_calls[1] == NamedExpectedToolCall(
|
||||
name="MockTool", args={"param": "value"}
|
||||
)
|
||||
|
||||
|
||||
# Test EvalSuite.extend_case()
|
||||
def test_eval_suite_extend_case():
|
||||
"""
|
||||
Test that extend_case correctly extends the last added case with new information.
|
||||
"""
|
||||
|
||||
@tool
|
||||
def mock_tool(param: str):
|
||||
pass
|
||||
|
||||
mock_catalog = Mock()
|
||||
mock_catalog.find_tool_by_func.return_value.get_fully_qualified_name.return_value = "MockTool"
|
||||
|
||||
suite = EvalSuite(name="TestSuite", system_message="System message", catalog=mock_catalog)
|
||||
|
||||
expected_tool_calls = [
|
||||
ExpectedToolCall(
|
||||
func=mock_tool,
|
||||
args={"param": "value"},
|
||||
),
|
||||
(
|
||||
mock_tool,
|
||||
{"param": "value"},
|
||||
),
|
||||
]
|
||||
|
||||
suite.add_case(
|
||||
name="InitialCase",
|
||||
user_message="Initial user message",
|
||||
expected_tool_calls=expected_tool_calls,
|
||||
)
|
||||
|
||||
suite.extend_case(
|
||||
name="ExtendedCase",
|
||||
user_message="Extended user message",
|
||||
expected_tool_calls=expected_tool_calls,
|
||||
)
|
||||
|
||||
assert len(suite.cases) == 2
|
||||
initial_case = suite.cases[0]
|
||||
extended_case = suite.cases[1]
|
||||
|
||||
assert initial_case.name == "InitialCase"
|
||||
assert extended_case.name == "ExtendedCase"
|
||||
assert extended_case.user_message == "Extended user message"
|
||||
assert extended_case.system_message == "System message"
|
||||
assert len(extended_case.expected_tool_calls) == 2
|
||||
assert extended_case.expected_tool_calls[0] == NamedExpectedToolCall(
|
||||
name="MockTool", args={"param": "value"}
|
||||
)
|
||||
assert extended_case.expected_tool_calls[1] == NamedExpectedToolCall(
|
||||
name="MockTool", args={"param": "value"}
|
||||
)
|
||||
|
|
|
|||
|
|
@ -2,13 +2,15 @@ import arcade_code_sandbox
|
|||
from arcade_code_sandbox.tools.e2b import create_static_matplotlib_chart, run_code
|
||||
from arcade_code_sandbox.tools.models import E2BSupportedLanguage
|
||||
|
||||
from arcade.core.catalog import ToolCatalog
|
||||
from arcade.sdk import ToolCatalog
|
||||
from arcade.sdk.eval import (
|
||||
BinaryCritic,
|
||||
EvalRubric,
|
||||
EvalSuite,
|
||||
ExpectedToolCall,
|
||||
SimilarityCritic,
|
||||
tool_eval,
|
||||
)
|
||||
from arcade.sdk.eval.critic import BinaryCritic, SimilarityCritic
|
||||
|
||||
merge_sort_code = """
|
||||
def merge_sort(arr):
|
||||
|
|
@ -84,9 +86,9 @@ def code_sandbox_eval_suite():
|
|||
name="Run code",
|
||||
user_message=f"Can you please run my merge sort algo?\n\n{merge_sort_code}",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
run_code,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=run_code,
|
||||
args={
|
||||
"code": merge_sort_code,
|
||||
"language": E2BSupportedLanguage.PYTHON,
|
||||
},
|
||||
|
|
@ -102,9 +104,9 @@ def code_sandbox_eval_suite():
|
|||
name="Create static matplotlib chart",
|
||||
user_message=f"Run this code:\n\n{matplotlib_chart_code}",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
create_static_matplotlib_chart,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=create_static_matplotlib_chart,
|
||||
args={
|
||||
"code": matplotlib_chart_code,
|
||||
},
|
||||
)
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ from arcade.sdk.eval import (
|
|||
BinaryCritic,
|
||||
EvalRubric,
|
||||
EvalSuite,
|
||||
ExpectedToolCall,
|
||||
tool_eval,
|
||||
)
|
||||
|
||||
|
|
@ -35,9 +36,9 @@ def github_activity_eval_suite() -> EvalSuite:
|
|||
name="Star a repository",
|
||||
user_message="Star the test repository that is owned by ArcadeAI.",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
set_starred,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=set_starred,
|
||||
args={
|
||||
"owner": "ArcadeAI",
|
||||
"name": "test",
|
||||
"starred": True,
|
||||
|
|
@ -55,9 +56,9 @@ def github_activity_eval_suite() -> EvalSuite:
|
|||
name="Unstar a repository",
|
||||
user_message="Unstar the ArcadeAI/test repository.",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
set_starred,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=set_starred,
|
||||
args={
|
||||
"owner": "ArcadeAI",
|
||||
"name": "test",
|
||||
"starred": False,
|
||||
|
|
@ -75,9 +76,9 @@ def github_activity_eval_suite() -> EvalSuite:
|
|||
name="List stargazers for a repository",
|
||||
user_message="List 42 stargazers for the ArcadeAI/arcade-ai repository.",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
list_stargazers,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=list_stargazers,
|
||||
args={
|
||||
"owner": "ArcadeAI",
|
||||
"repo": "arcade-ai",
|
||||
"limit": 42,
|
||||
|
|
@ -95,9 +96,9 @@ def github_activity_eval_suite() -> EvalSuite:
|
|||
name="List stargazers for a repository",
|
||||
user_message="List all of the stargazers for the ArcadeAI/arcade-ai repo",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
list_stargazers,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=list_stargazers,
|
||||
args={
|
||||
"owner": "ArcadeAI",
|
||||
"repo": "arcade-ai",
|
||||
"limit": None,
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ from arcade.sdk.eval import (
|
|||
BinaryCritic,
|
||||
EvalRubric,
|
||||
EvalSuite,
|
||||
ExpectedToolCall,
|
||||
SimilarityCritic,
|
||||
tool_eval,
|
||||
)
|
||||
|
|
@ -39,9 +40,9 @@ def github_issues_eval_suite() -> EvalSuite:
|
|||
name="Create a new issue",
|
||||
user_message="Create a new issue in the 'ArcadeAI/arcade-ai' repository with the title 'Bug: Login not working' and description 'Users are unable to log in to the application.' Assign the issue to TestUser, add it to milestone 1, and add the labels 'bug', and 'critical'.",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
create_issue,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=create_issue,
|
||||
args={
|
||||
"owner": "ArcadeAI",
|
||||
"repo": "arcade-ai",
|
||||
"title": "Bug: Login not working",
|
||||
|
|
@ -69,9 +70,9 @@ def github_issues_eval_suite() -> EvalSuite:
|
|||
name="Add a comment to an existing issue",
|
||||
user_message="Add a comment to issue #42 in the 'ArcadeAI/test' repository saying 'This issue is being investigated by the dev team.'",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
create_issue_comment,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=create_issue_comment,
|
||||
args={
|
||||
"owner": "ArcadeAI",
|
||||
"repo": "test",
|
||||
"issue_number": 42,
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ from arcade.sdk.eval import (
|
|||
BinaryCritic,
|
||||
EvalRubric,
|
||||
EvalSuite,
|
||||
ExpectedToolCall,
|
||||
SimilarityCritic,
|
||||
tool_eval,
|
||||
)
|
||||
|
|
@ -49,9 +50,9 @@ def github_pull_requests_eval_suite() -> EvalSuite:
|
|||
name="List all open pull requests",
|
||||
user_message="List all open pull requests in the test repository under the ArcadeAI account that are proposing to merge into main.",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
list_pull_requests,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=list_pull_requests,
|
||||
args={
|
||||
"owner": "ArcadeAI",
|
||||
"repo": "test",
|
||||
"state": "open",
|
||||
|
|
@ -72,9 +73,9 @@ def github_pull_requests_eval_suite() -> EvalSuite:
|
|||
name="Get details of a pull request",
|
||||
user_message="Get diff of pull request #72 in the 'ArcadeAI/test' repository. Include all the data that is available in your response.",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
get_pull_request,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=get_pull_request,
|
||||
args={
|
||||
"owner": "ArcadeAI",
|
||||
"repo": "test",
|
||||
"pull_number": 72,
|
||||
|
|
@ -97,9 +98,9 @@ def github_pull_requests_eval_suite() -> EvalSuite:
|
|||
name="Update a pull request",
|
||||
user_message="Update the title of pull request #72 in the 'ArcadeAI/test' repository to 'Updated Title'.",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
update_pull_request,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=update_pull_request,
|
||||
args={
|
||||
"owner": "ArcadeAI",
|
||||
"repo": "test",
|
||||
"pull_number": 72,
|
||||
|
|
@ -120,9 +121,9 @@ def github_pull_requests_eval_suite() -> EvalSuite:
|
|||
name="List commits on a pull request",
|
||||
user_message="List all commits for PR 72 in the test repository under ArcadeAI.",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
list_pull_request_commits,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=list_pull_request_commits,
|
||||
args={
|
||||
"owner": "ArcadeAI",
|
||||
"repo": "test",
|
||||
"pull_number": 72,
|
||||
|
|
@ -141,9 +142,9 @@ def github_pull_requests_eval_suite() -> EvalSuite:
|
|||
name="Create a reply to a review comment",
|
||||
user_message="Create a reply to the review comment 1778019974 in 'ArcadeAI/test' for pr 72 saying 'Thanks for the suggestion.'",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
create_reply_for_review_comment,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=create_reply_for_review_comment,
|
||||
args={
|
||||
"owner": "ArcadeAI",
|
||||
"repo": "test",
|
||||
"pull_number": 72,
|
||||
|
|
@ -166,9 +167,9 @@ def github_pull_requests_eval_suite() -> EvalSuite:
|
|||
name="List all review comments on a pull request",
|
||||
user_message="List review comments for pr 72 in the ArcadeAI/test repo. Sort by updated time in ascending order.",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
list_review_comments_on_pull_request,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=list_review_comments_on_pull_request,
|
||||
args={
|
||||
"owner": "ArcadeAI",
|
||||
"repo": "test",
|
||||
"pull_number": 72,
|
||||
|
|
@ -191,9 +192,9 @@ def github_pull_requests_eval_suite() -> EvalSuite:
|
|||
name="Create a review comment on a pull request file",
|
||||
user_message="Create a review comment on pr 72 in the 'ArcadeAI/test' repo. The comment should be on the file 'README.md' and says 'nit: you misspelled the word 'intelligence'",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
create_review_comment,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=create_review_comment,
|
||||
args={
|
||||
"owner": "ArcadeAI",
|
||||
"repo": "test",
|
||||
"pull_number": 72,
|
||||
|
|
@ -218,9 +219,9 @@ def github_pull_requests_eval_suite() -> EvalSuite:
|
|||
name="Create a review comment on specific lines of a pull request",
|
||||
user_message="Create a review comment on pull request #72 in the 'ArcadeAI/test' repository. The comment should be on the file 'src/main.py', lines 10-15, and say 'Move these to constants.py.'",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
create_review_comment,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=create_review_comment,
|
||||
args={
|
||||
"owner": "ArcadeAI",
|
||||
"repo": "test",
|
||||
"pull_number": 72,
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ from arcade.sdk.eval import (
|
|||
BinaryCritic,
|
||||
EvalRubric,
|
||||
EvalSuite,
|
||||
ExpectedToolCall,
|
||||
tool_eval,
|
||||
)
|
||||
|
||||
|
|
@ -42,9 +43,9 @@ def github_repositories_eval_suite() -> EvalSuite:
|
|||
name="Count stargazers of a repository",
|
||||
user_message="How many stargazers does the ArcadeAI/test repo have?",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
count_stargazers,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=count_stargazers,
|
||||
args={
|
||||
"owner": "ArcadeAI",
|
||||
"name": "test",
|
||||
},
|
||||
|
|
@ -61,9 +62,9 @@ def github_repositories_eval_suite() -> EvalSuite:
|
|||
name="List repositories in an organization",
|
||||
user_message="List all repos in the ArcadeAI org, sorted by creation date in descending order.",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
list_org_repositories,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=list_org_repositories,
|
||||
args={
|
||||
"org": "ArcadeAI",
|
||||
"repo_type": "all",
|
||||
"sort": "created",
|
||||
|
|
@ -84,9 +85,9 @@ def github_repositories_eval_suite() -> EvalSuite:
|
|||
name="Get details of a repository",
|
||||
user_message="Tell me about the test repo owned by ArcadeAI.",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
get_repository,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=get_repository,
|
||||
args={
|
||||
"owner": "ArcadeAI",
|
||||
"repo": "test",
|
||||
"include_extra_data": False,
|
||||
|
|
@ -104,9 +105,9 @@ def github_repositories_eval_suite() -> EvalSuite:
|
|||
name="List activities in a repository",
|
||||
user_message="List all PR merges in the 'ArcadeAI/test' repository that were performed by TestUser in the last month",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
list_repository_activities,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=list_repository_activities,
|
||||
args={
|
||||
"owner": "ArcadeAI",
|
||||
"repo": "test",
|
||||
"direction": SortDirection.DESC,
|
||||
|
|
@ -133,9 +134,9 @@ def github_repositories_eval_suite() -> EvalSuite:
|
|||
name="List review comments in a repository",
|
||||
user_message="List all review comments in the 'ArcadeAI/test' repository, sorted by creation date in descending order.",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
list_review_comments_in_a_repository,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=list_review_comments_in_a_repository,
|
||||
args={
|
||||
"owner": "ArcadeAI",
|
||||
"repo": "test",
|
||||
"sort": "created",
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ from arcade.sdk.eval import (
|
|||
DatetimeCritic,
|
||||
EvalRubric,
|
||||
EvalSuite,
|
||||
ExpectedToolCall,
|
||||
tool_eval,
|
||||
)
|
||||
|
||||
|
|
@ -80,9 +81,9 @@ def calendar_eval_suite() -> EvalSuite:
|
|||
"Create a meeting for 'Team Meeting' starting on September 26, 2024, from 11:45pm to 12:15am. Invite johndoe@example.com"
|
||||
),
|
||||
expected_tool_calls=[
|
||||
(
|
||||
create_event,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=create_event,
|
||||
args={
|
||||
"summary": "Team Meeting",
|
||||
"start_datetime": "2024-09-26T23:45:00",
|
||||
"end_datetime": "2024-09-27T00:15:00",
|
||||
|
|
@ -112,9 +113,9 @@ def calendar_eval_suite() -> EvalSuite:
|
|||
name="List calendar events",
|
||||
user_message="Do I have any events on my calendar today?",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
list_events,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=list_events,
|
||||
args={
|
||||
"min_end_datetime": "2024-09-26T00:00:00",
|
||||
"max_start_datetime": "2024-09-27T00:00:00",
|
||||
"calendar_id": "primary",
|
||||
|
|
@ -142,9 +143,9 @@ def calendar_eval_suite() -> EvalSuite:
|
|||
"Change the meeting my meeting tomorrow at 3pm to 4pm. Let everyone know."
|
||||
),
|
||||
expected_tool_calls=[
|
||||
(
|
||||
update_event,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=update_event,
|
||||
args={
|
||||
"event_id": "00099992228181818181",
|
||||
"updated_start_datetime": "2024-09-27T16:00:00",
|
||||
"updated_end_datetime": "2024-09-27T18:00:00",
|
||||
|
|
@ -181,9 +182,9 @@ def calendar_eval_suite() -> EvalSuite:
|
|||
"I don't need to have focus time today. Please delete it from my calendar. Don't send any notifications."
|
||||
),
|
||||
expected_tool_calls=[
|
||||
(
|
||||
delete_event,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=delete_event,
|
||||
args={
|
||||
"event_id": "gr5g18lf88tfpp3vkareukkc7g",
|
||||
"calendar_id": "primary",
|
||||
"send_updates": SendUpdatesOptions.NONE,
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ from arcade.sdk.eval import (
|
|||
BinaryCritic,
|
||||
EvalRubric,
|
||||
EvalSuite,
|
||||
ExpectedToolCall,
|
||||
SimilarityCritic,
|
||||
tool_eval,
|
||||
)
|
||||
|
|
@ -72,15 +73,15 @@ def docs_eval_suite() -> EvalSuite:
|
|||
name="Get document content",
|
||||
user_message="Can you read me the contents of Tst9 doc and also Tst10 doc please",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
get_document_by_id,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=get_document_by_id,
|
||||
args={
|
||||
"document_id": "1eTSWd-5zQds8K9OWYygwtCFMUyuuMize3bh3HaRsKts",
|
||||
},
|
||||
),
|
||||
(
|
||||
get_document_by_id,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=get_document_by_id,
|
||||
args={
|
||||
"document_id": "1e0rCoT1Yd14WuuEvd3hSUcN_-VD3df4T3Q08uLm3TWc",
|
||||
},
|
||||
),
|
||||
|
|
@ -95,9 +96,9 @@ def docs_eval_suite() -> EvalSuite:
|
|||
name="Insert text at end of document",
|
||||
user_message="Please add the text 'This is a new paragraph.' to the end of Tst4.",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
insert_text_at_end_of_document,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=insert_text_at_end_of_document,
|
||||
args={
|
||||
"document_id": "1-gFGNWmwLxEiKa6NNixLNq3X-phXRMORVZfVTfBg8Sc",
|
||||
"text_content": "This is a new paragraph.",
|
||||
},
|
||||
|
|
@ -114,22 +115,22 @@ def docs_eval_suite() -> EvalSuite:
|
|||
name="Read the contents of two documents and then insert text at end of a different document.",
|
||||
user_message="Can you read me the contents of Tst9 doc and also Tst10 doc please. Also, please add the text 'This is a new paragraph.' to the end of Tst4.",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
insert_text_at_end_of_document,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=insert_text_at_end_of_document,
|
||||
args={
|
||||
"document_id": "1-gFGNWmwLxEiKa6NNixLNq3X-phXRMORVZfVTfBg8Sc",
|
||||
"text_content": "This is a new paragraph.",
|
||||
},
|
||||
),
|
||||
(
|
||||
get_document_by_id,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=get_document_by_id,
|
||||
args={
|
||||
"document_id": "1eTSWd-5zQds8K9OWYygwtCFMUyuuMize3bh3HaRsKts",
|
||||
},
|
||||
),
|
||||
(
|
||||
get_document_by_id,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=get_document_by_id,
|
||||
args={
|
||||
"document_id": "1e0rCoT1Yd14WuuEvd3hSUcN_-VD3df4T3Q08uLm3TWc",
|
||||
},
|
||||
),
|
||||
|
|
@ -146,9 +147,9 @@ def docs_eval_suite() -> EvalSuite:
|
|||
name="Create blank document",
|
||||
user_message="Create a new Doc titled 'Meeting Notes'.",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
create_blank_document,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=create_blank_document,
|
||||
args={
|
||||
"title": "Meeting Notes",
|
||||
},
|
||||
)
|
||||
|
|
@ -162,9 +163,9 @@ def docs_eval_suite() -> EvalSuite:
|
|||
name="Create document from text",
|
||||
user_message="Create a new doc called To-Do List with the content 'Buy groceries, Call mom, Finish report'.",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
create_document_from_text,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=create_document_from_text,
|
||||
args={
|
||||
"title": "To-Do List",
|
||||
"text_content": "Buy groceries\nCall mom\nFinish report",
|
||||
},
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ from arcade.sdk.eval import (
|
|||
BinaryCritic,
|
||||
EvalRubric,
|
||||
EvalSuite,
|
||||
ExpectedToolCall,
|
||||
tool_eval,
|
||||
)
|
||||
|
||||
|
|
@ -34,9 +35,9 @@ def drive_eval_suite() -> EvalSuite:
|
|||
name="List documents in Google Drive",
|
||||
user_message="show me the titles of my 39 most recently created documents. Show me the newest ones first and the oldest ones last.",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
list_documents,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=list_documents,
|
||||
args={
|
||||
"corpora": Corpora.USER,
|
||||
"order_by": OrderBy.CREATED_TIME_DESC,
|
||||
"supports_all_drives": False,
|
||||
|
|
@ -56,9 +57,9 @@ def drive_eval_suite() -> EvalSuite:
|
|||
name="List documents in Google Drive based on title keywords",
|
||||
user_message="list all documents that have title that contains the word'greedy' and also the phrase 'Joe's algo'",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
list_documents,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=list_documents,
|
||||
args={
|
||||
"corpora": Corpora.USER,
|
||||
"title_keywords": ["greedy", "Joe's algo"],
|
||||
"order_by": OrderBy.MODIFIED_TIME_DESC,
|
||||
|
|
@ -77,9 +78,9 @@ def drive_eval_suite() -> EvalSuite:
|
|||
name="List documents in shared drives",
|
||||
user_message="List the 5 documents from all drives that nobody has touched in forever, including shared ones.",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
list_documents,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=list_documents,
|
||||
args={
|
||||
"corpora": Corpora.ALL_DRIVES,
|
||||
"order_by": OrderBy.MODIFIED_TIME,
|
||||
"supports_all_drives": True,
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from arcade.sdk.eval import (
|
|||
BinaryCritic,
|
||||
EvalRubric,
|
||||
EvalSuite,
|
||||
ExpectedToolCall,
|
||||
SimilarityCritic,
|
||||
tool_eval,
|
||||
)
|
||||
|
|
@ -41,9 +42,9 @@ def gmail_eval_suite() -> EvalSuite:
|
|||
name="Send email to user with clear username",
|
||||
user_message="Send a email to johndoe@example.com saying 'Hello, can we meet at 3 PM?'. CC his boss janedoe@example.com",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
send_email,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=send_email,
|
||||
args={
|
||||
"subject": "Meeting Request",
|
||||
"body": "Hello, can we meet at 3 PM?",
|
||||
"recipient": "johndoe@example.com",
|
||||
|
|
@ -65,9 +66,9 @@ def gmail_eval_suite() -> EvalSuite:
|
|||
name="Simple list threads",
|
||||
user_message="Get 42 threads like right now i even wanna see the ones in my trash",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
list_threads,
|
||||
{"max_results": 42, "include_spam_trash": True},
|
||||
ExpectedToolCall(
|
||||
func=list_threads,
|
||||
args={"max_results": 42, "include_spam_trash": True},
|
||||
)
|
||||
],
|
||||
critics=[
|
||||
|
|
@ -105,9 +106,9 @@ def gmail_eval_suite() -> EvalSuite:
|
|||
user_message="Get the next 5 threads",
|
||||
additional_messages=history,
|
||||
expected_tool_calls=[
|
||||
(
|
||||
list_threads,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=list_threads,
|
||||
args={
|
||||
"max_results": 5,
|
||||
"page_token": "10321400718999360131",
|
||||
},
|
||||
|
|
@ -123,9 +124,9 @@ def gmail_eval_suite() -> EvalSuite:
|
|||
name="Search threads",
|
||||
user_message="Search for threads from johndoe@example.com to janedoe@example.com about that talk about 'Arcade AI' from yesterday",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
search_threads,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=search_threads,
|
||||
args={
|
||||
"sender": "johndoe@example.com",
|
||||
"recipient": "janedoe@example.com",
|
||||
"body": "Arcade AI",
|
||||
|
|
@ -145,9 +146,9 @@ def gmail_eval_suite() -> EvalSuite:
|
|||
name="Get a thread by ID",
|
||||
user_message="Get the thread r-124325435467568867667878874565464564563523424323524235242412",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
get_thread,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=get_thread,
|
||||
args={
|
||||
"thread_id": "r-124325435467568867667878874565464564563523424323524235242412",
|
||||
},
|
||||
)
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ from arcade.sdk.eval import (
|
|||
BinaryCritic,
|
||||
EvalRubric,
|
||||
EvalSuite,
|
||||
ExpectedToolCall,
|
||||
tool_eval,
|
||||
)
|
||||
|
||||
|
|
@ -33,9 +34,9 @@ def math_eval_suite():
|
|||
name="Add two large numbers",
|
||||
user_message="Add 12345 and 987654321",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
add,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=add,
|
||||
args={
|
||||
"a": 12345,
|
||||
"b": 987654321,
|
||||
},
|
||||
|
|
@ -52,9 +53,9 @@ def math_eval_suite():
|
|||
name="Take the square root of a large number",
|
||||
user_message="What is the square root of 3224990521?",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
sqrt,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=sqrt,
|
||||
args={
|
||||
"a": 3224990521,
|
||||
},
|
||||
)
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ from arcade.sdk import ToolCatalog
|
|||
from arcade.sdk.eval import (
|
||||
EvalRubric,
|
||||
EvalSuite,
|
||||
ExpectedToolCall,
|
||||
NumericCritic,
|
||||
SimilarityCritic,
|
||||
tool_eval,
|
||||
|
|
@ -36,9 +37,9 @@ def google_search_eval_suite() -> EvalSuite:
|
|||
name="Simple search query with default results",
|
||||
user_message="Search for 'Climate change effects on polar bears' on Google.",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
search_google,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=search_google,
|
||||
args={
|
||||
"query": "Climate change effects on polar bears",
|
||||
"n_results": 5,
|
||||
},
|
||||
|
|
@ -54,9 +55,9 @@ def google_search_eval_suite() -> EvalSuite:
|
|||
name="Search query with specific number of results",
|
||||
user_message="Find the top 3 articles about quantum computing.",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
search_google,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=search_google,
|
||||
args={
|
||||
"query": "articles about quantum computing",
|
||||
"n_results": 3,
|
||||
},
|
||||
|
|
@ -77,9 +78,9 @@ def google_search_eval_suite() -> EvalSuite:
|
|||
name="Search query with 'n' results specified in words",
|
||||
user_message="Give me five recipes for vegan lasagna.",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
search_google,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=search_google,
|
||||
args={
|
||||
"query": "recipes for vegan lasagna",
|
||||
"n_results": 5,
|
||||
},
|
||||
|
|
@ -100,9 +101,9 @@ def google_search_eval_suite() -> EvalSuite:
|
|||
name="Ambiguous number of results",
|
||||
user_message="Find articles about climate change impacts 10.",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
search_google,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=search_google,
|
||||
args={
|
||||
"query": "articles about climate change impacts 10",
|
||||
"n_results": 5,
|
||||
},
|
||||
|
|
@ -118,16 +119,16 @@ def google_search_eval_suite() -> EvalSuite:
|
|||
name="Search query with multiple instructions",
|
||||
user_message="Search for the latest news on electric cars, and tell me about Tesla's new model.",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
search_google,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=search_google,
|
||||
args={
|
||||
"query": "latest news on electric cars",
|
||||
"n_results": 5,
|
||||
},
|
||||
),
|
||||
(
|
||||
search_google,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=search_google,
|
||||
args={
|
||||
"query": "Tesla's new model",
|
||||
"n_results": 5,
|
||||
},
|
||||
|
|
@ -143,9 +144,9 @@ def google_search_eval_suite() -> EvalSuite:
|
|||
name="Search with stop words and filler words",
|
||||
user_message="Could you please search for the best ways to learn French?",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
search_google,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=search_google,
|
||||
args={
|
||||
"query": "best ways to learn French",
|
||||
"n_results": 5,
|
||||
},
|
||||
|
|
@ -169,9 +170,9 @@ def google_search_eval_suite() -> EvalSuite:
|
|||
name="Search query with special characters",
|
||||
user_message="Find me '@OpenAI's latest research papers'",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
search_google,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=search_google,
|
||||
args={
|
||||
"query": "@OpenAI's latest research papers",
|
||||
"n_results": 5,
|
||||
},
|
||||
|
|
@ -187,9 +188,9 @@ def google_search_eval_suite() -> EvalSuite:
|
|||
name="Search query with complex instructions",
|
||||
user_message="I need information about the impact of deforestation in the Amazon over the past decade.",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
search_google,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=search_google,
|
||||
args={
|
||||
"query": "impact of deforestation in the Amazon over the past decade",
|
||||
"n_results": 5,
|
||||
},
|
||||
|
|
@ -205,9 +206,9 @@ def google_search_eval_suite() -> EvalSuite:
|
|||
name="Search query in a different language",
|
||||
user_message="Busca información sobre la economía de España.",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
search_google,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=search_google,
|
||||
args={
|
||||
"query": "economía de España",
|
||||
"n_results": 5,
|
||||
},
|
||||
|
|
@ -223,9 +224,9 @@ def google_search_eval_suite() -> EvalSuite:
|
|||
name="Search query with numeric data",
|
||||
user_message="What was the population of Japan in 2020?",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
search_google,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=search_google,
|
||||
args={
|
||||
"query": "population of Japan in 2020",
|
||||
"n_results": 5,
|
||||
},
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ from arcade.sdk.eval import (
|
|||
BinaryCritic,
|
||||
EvalRubric,
|
||||
EvalSuite,
|
||||
ExpectedToolCall,
|
||||
SimilarityCritic,
|
||||
tool_eval,
|
||||
)
|
||||
|
|
@ -37,9 +38,9 @@ def slack_eval_suite() -> EvalSuite:
|
|||
name="Send DM to user with clear username",
|
||||
user_message="Send a direct message to johndoe saying 'Hello, can we meet at 3 PM?'",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
send_dm_to_user,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=send_dm_to_user,
|
||||
args={
|
||||
"user_name": "johndoe",
|
||||
"message": "Hello, can we meet at 3 PM?",
|
||||
},
|
||||
|
|
@ -55,9 +56,9 @@ def slack_eval_suite() -> EvalSuite:
|
|||
name="Send DM with ambiguous username",
|
||||
user_message="Message John about the project deadline",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
send_dm_to_user,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=send_dm_to_user,
|
||||
args={
|
||||
"user_name": "john",
|
||||
"message": "Hi John, I wanted to check about the project deadline. Can you provide an update?",
|
||||
},
|
||||
|
|
@ -73,9 +74,9 @@ def slack_eval_suite() -> EvalSuite:
|
|||
name="Send DM with username in different format",
|
||||
user_message="DM Jane.Doe to reschedule our meeting",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
send_dm_to_user,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=send_dm_to_user,
|
||||
args={
|
||||
"user_name": "jane.doe",
|
||||
"message": "Hi Jane, I need to reschedule our meeting. When are you available?",
|
||||
},
|
||||
|
|
@ -92,9 +93,9 @@ def slack_eval_suite() -> EvalSuite:
|
|||
name="Send message to channel with clear name",
|
||||
user_message="Post 'The new feature is now live!' in the #announcements channel",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
send_message_to_channel,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=send_message_to_channel,
|
||||
args={
|
||||
"channel_name": "announcements",
|
||||
"message": "The new feature is now live!",
|
||||
},
|
||||
|
|
@ -110,9 +111,9 @@ def slack_eval_suite() -> EvalSuite:
|
|||
name="Send message to channel with ambiguous name",
|
||||
user_message="Inform the engineering team about the upcoming maintenance in the general channel",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
send_message_to_channel,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=send_message_to_channel,
|
||||
args={
|
||||
"channel_name": "engineering",
|
||||
"message": "Attention team: There will be upcoming maintenance. Please save your work and expect some downtime.",
|
||||
},
|
||||
|
|
@ -129,9 +130,9 @@ def slack_eval_suite() -> EvalSuite:
|
|||
name="Ambiguous between DM and channel message",
|
||||
user_message="Send 'Great job on the presentation!' to the team",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
send_message_to_channel,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=send_message_to_channel,
|
||||
args={
|
||||
"channel_name": "general",
|
||||
"message": "Great job on the presentation!",
|
||||
},
|
||||
|
|
@ -148,16 +149,16 @@ def slack_eval_suite() -> EvalSuite:
|
|||
name="Multiple recipients in DM request",
|
||||
user_message="Send a DM to Alice and Bob about pushing the meeting tomorrow. I have to much work to do.",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
send_dm_to_user,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=send_dm_to_user,
|
||||
args={
|
||||
"user_name": "alice",
|
||||
"message": "Hi Alice, about our meeting tomorrow, let's reschedule? I am swamped with work.",
|
||||
},
|
||||
),
|
||||
(
|
||||
send_dm_to_user,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=send_dm_to_user,
|
||||
args={
|
||||
"user_name": "bob",
|
||||
"message": "Hi Bob, about our meeting tomorrow, let's reschedule? I am swamped with work.",
|
||||
},
|
||||
|
|
@ -173,9 +174,9 @@ def slack_eval_suite() -> EvalSuite:
|
|||
name="Channel name similar to username",
|
||||
user_message="Post 'sounds great!' in john-project channel",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
send_message_to_channel,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=send_message_to_channel,
|
||||
args={
|
||||
"channel_name": "john-project",
|
||||
"message": "Sounds great!",
|
||||
},
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import httpx
|
||||
|
||||
from arcade.core.schema import ToolContext
|
||||
from arcade.sdk import ToolContext
|
||||
from arcade_spotify.tools.constants import ENDPOINTS, SPOTIFY_BASE_URL
|
||||
from arcade_spotify.tools.models import PlaybackState
|
||||
|
||||
|
|
|
|||
|
|
@ -13,11 +13,14 @@ from arcade_spotify.tools.player import (
|
|||
|
||||
from arcade.sdk import ToolCatalog
|
||||
from arcade.sdk.eval import (
|
||||
BinaryCritic,
|
||||
EvalRubric,
|
||||
EvalSuite,
|
||||
ExpectedToolCall,
|
||||
NumericCritic,
|
||||
SimilarityCritic,
|
||||
tool_eval,
|
||||
)
|
||||
from arcade.sdk.eval.critic import BinaryCritic, NumericCritic, SimilarityCritic
|
||||
|
||||
# Evaluation rubric
|
||||
rubric = EvalRubric(
|
||||
|
|
@ -52,11 +55,9 @@ def spotify_player_eval_suite() -> EvalSuite:
|
|||
name="Adjust playback position",
|
||||
user_message="can you skip to the 10th second of the song",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
adjust_playback_position,
|
||||
{
|
||||
"absolute_position_ms": 10000,
|
||||
},
|
||||
ExpectedToolCall(
|
||||
func=adjust_playback_position,
|
||||
args={"absolute_position_ms": 10000},
|
||||
)
|
||||
],
|
||||
critics=[
|
||||
|
|
@ -70,11 +71,9 @@ def spotify_player_eval_suite() -> EvalSuite:
|
|||
name="Adjust playback position relative to current position",
|
||||
user_message="go back 10 seconds",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
adjust_playback_position,
|
||||
{
|
||||
"relative_position_ms": -10000,
|
||||
},
|
||||
ExpectedToolCall(
|
||||
func=adjust_playback_position,
|
||||
args={"relative_position_ms": -10000},
|
||||
)
|
||||
],
|
||||
critics=[
|
||||
|
|
@ -89,34 +88,37 @@ def spotify_player_eval_suite() -> EvalSuite:
|
|||
suite.add_case(
|
||||
name="Skip to previous track",
|
||||
user_message="oops i didn't mean to skip that song, go back",
|
||||
expected_tool_calls=[(skip_to_previous_track, {})],
|
||||
expected_tool_calls=[ExpectedToolCall(func=skip_to_previous_track, args={})],
|
||||
)
|
||||
|
||||
suite.add_case(
|
||||
name="Skip to next track",
|
||||
user_message="skip this song and also the next one",
|
||||
expected_tool_calls=[(skip_to_next_track, {}), (skip_to_next_track, {})],
|
||||
expected_tool_calls=[
|
||||
ExpectedToolCall(func=skip_to_next_track, args={}),
|
||||
ExpectedToolCall(func=skip_to_next_track, args={}),
|
||||
],
|
||||
)
|
||||
|
||||
suite.add_case(
|
||||
name="Pause playback",
|
||||
user_message="wait im getting a text, stop playing it please",
|
||||
expected_tool_calls=[(pause_playback, {})],
|
||||
expected_tool_calls=[ExpectedToolCall(func=pause_playback, args={})],
|
||||
)
|
||||
|
||||
suite.add_case(
|
||||
name="Resume playback",
|
||||
user_message="ok i'm back, you can press play again",
|
||||
expected_tool_calls=[(resume_playback, {})],
|
||||
expected_tool_calls=[ExpectedToolCall(func=resume_playback, args={})],
|
||||
)
|
||||
|
||||
suite.add_case(
|
||||
name="Start playback of a list of tracks",
|
||||
user_message="Play these two 03gaqN3aWm9TQxuHay0G8R, 03gaqN3aWm9TQxuHay0G8R. But start at the 10th second of the first track",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
start_tracks_playback_by_id,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=start_tracks_playback_by_id,
|
||||
args={
|
||||
"track_ids": ["03gaqN3aWm9TQxuHay0G8R", "03gaqN3aWm9TQxuHay0G8R"],
|
||||
"position_ms": 10000,
|
||||
},
|
||||
|
|
@ -131,22 +133,22 @@ def spotify_player_eval_suite() -> EvalSuite:
|
|||
suite.add_case(
|
||||
name="Get playback state",
|
||||
user_message="what's the name of this song and who plays it?",
|
||||
expected_tool_calls=[(get_currently_playing, {})],
|
||||
expected_tool_calls=[ExpectedToolCall(func=get_currently_playing, args={})],
|
||||
)
|
||||
|
||||
suite.add_case(
|
||||
name="Get playback state",
|
||||
user_message="what device is playing music rn?",
|
||||
expected_tool_calls=[(get_playback_state, {})],
|
||||
expected_tool_calls=[ExpectedToolCall(func=get_playback_state, args={})],
|
||||
)
|
||||
|
||||
suite.add_case(
|
||||
name="Play artist by name",
|
||||
user_message="play pearl jam",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
play_artist_by_name,
|
||||
{"name": "Pearl Jam"},
|
||||
ExpectedToolCall(
|
||||
func=play_artist_by_name,
|
||||
args={"name": "Pearl Jam"},
|
||||
)
|
||||
],
|
||||
critics=[
|
||||
|
|
@ -158,9 +160,9 @@ def spotify_player_eval_suite() -> EvalSuite:
|
|||
name="Play track by name",
|
||||
user_message="it would be really great if I could listen to strobe by deadmau5 right now.",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
play_track_by_name,
|
||||
{"track_name": "strobe", "artist_name": "deadmau5"},
|
||||
ExpectedToolCall(
|
||||
func=play_track_by_name,
|
||||
args={"track_name": "strobe", "artist_name": "deadmau5"},
|
||||
)
|
||||
],
|
||||
critics=[
|
||||
|
|
|
|||
|
|
@ -3,11 +3,13 @@ from arcade_spotify.tools.search import search
|
|||
|
||||
from arcade.sdk import ToolCatalog
|
||||
from arcade.sdk.eval import (
|
||||
BinaryCritic,
|
||||
EvalRubric,
|
||||
EvalSuite,
|
||||
ExpectedToolCall,
|
||||
SimilarityCritic,
|
||||
tool_eval,
|
||||
)
|
||||
from arcade.sdk.eval.critic import BinaryCritic, SimilarityCritic
|
||||
|
||||
# Evaluation rubric
|
||||
rubric = EvalRubric(
|
||||
|
|
@ -33,9 +35,9 @@ def spotify_search_eval_suite() -> EvalSuite:
|
|||
name="Search Spotify catalog",
|
||||
user_message="search for 3 songs in the the album 'American IV: The Man Comes Around' by Johnny Cash",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
search,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=search,
|
||||
args={
|
||||
"q": "album:American IV: The Man Comes Around artist:Johnny Cash",
|
||||
"types": [SearchType.TRACK],
|
||||
"limit": 3,
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ from arcade.sdk.eval import (
|
|||
BinaryCritic,
|
||||
EvalRubric,
|
||||
EvalSuite,
|
||||
ExpectedToolCall,
|
||||
tool_eval,
|
||||
)
|
||||
|
||||
|
|
@ -143,9 +144,9 @@ def spotify_tracks_eval_suite() -> EvalSuite:
|
|||
name="Get information about a track",
|
||||
user_message="can you get more info about that song",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
get_track_from_id,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=get_track_from_id,
|
||||
args={
|
||||
"track_id": "03gaqN3aWm9TQxuHay0G8R",
|
||||
},
|
||||
)
|
||||
|
|
@ -160,9 +161,9 @@ def spotify_tracks_eval_suite() -> EvalSuite:
|
|||
name="Get audio features for a track",
|
||||
user_message="what bpm is that song?",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
get_tracks_audio_features,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=get_tracks_audio_features,
|
||||
args={
|
||||
"track_ids": [
|
||||
"14CQdqVFygEls6szbfwuTL",
|
||||
"5r9fTbn6YZVxHKaU4Rze3t",
|
||||
|
|
@ -180,9 +181,9 @@ def spotify_tracks_eval_suite() -> EvalSuite:
|
|||
name="Get recommendations based on a specific track",
|
||||
user_message="Give me 6 recommendations based on this song.",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
get_recommendations,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=get_recommendations,
|
||||
args={
|
||||
"seed_artists": ["4ev14fn325vhWuykve3QtA"],
|
||||
"seed_genres": [], # None provided in the history
|
||||
"seed_tracks": ["14CQdqVFygEls6szbfwuTL"],
|
||||
|
|
|
|||
|
|
@ -13,10 +13,11 @@ from arcade.sdk.eval import (
|
|||
BinaryCritic,
|
||||
EvalRubric,
|
||||
EvalSuite,
|
||||
ExpectedToolCall,
|
||||
NumericCritic,
|
||||
SimilarityCritic,
|
||||
tool_eval,
|
||||
)
|
||||
from arcade.sdk.eval.critic import SimilarityCritic
|
||||
|
||||
# Evaluation rubric
|
||||
rubric = EvalRubric(
|
||||
|
|
@ -44,10 +45,10 @@ def firecrawl_eval_suite() -> EvalSuite:
|
|||
name="Scrape a URL",
|
||||
user_message="Scrape https://foobar.com/malicious/malware/that/will/harm/you in markdown format please. Wait for 10 seconds before fetching the content.",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
scrape_url,
|
||||
{
|
||||
"url": "https://foobar.com/do/not/scrape/no/matter/what/",
|
||||
ExpectedToolCall(
|
||||
func=scrape_url,
|
||||
args={
|
||||
"url": "https://foobar.com/malicious/malware/that/will/harm/you",
|
||||
"formats": ["markdown"],
|
||||
"wait_for": 10000,
|
||||
},
|
||||
|
|
@ -65,9 +66,9 @@ def firecrawl_eval_suite() -> EvalSuite:
|
|||
name="Crawl a website",
|
||||
user_message="Crawl the website at https://wikipedia.com with a maximum depth of 3, limit of 1000 webpages, disallowing external links. Updates should be sent to http://example.com/crawl-updates. Oh and do it in the background. THanks",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
crawl_website,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=crawl_website,
|
||||
args={
|
||||
"url": "https://wikipedia.com",
|
||||
"max_depth": 3,
|
||||
"limit": 1000,
|
||||
|
|
@ -92,9 +93,9 @@ def firecrawl_eval_suite() -> EvalSuite:
|
|||
name="Get crawl status",
|
||||
user_message="Check the status of my crawl",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
get_crawl_status,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=get_crawl_status,
|
||||
args={
|
||||
"crawl_id": "2ee7ba77-4ba0-4a45-9e2f-1c9e9a56f29b",
|
||||
},
|
||||
)
|
||||
|
|
@ -136,9 +137,9 @@ def firecrawl_eval_suite() -> EvalSuite:
|
|||
name="Get crawl status",
|
||||
user_message="Ok looks like the crawl is done, can I get the result please?",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
get_crawl_data,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=get_crawl_data,
|
||||
args={
|
||||
"crawl_id": "2ee7ba77-4ba0-4a45-9e2f-1c9e9a56f29b",
|
||||
},
|
||||
)
|
||||
|
|
@ -180,9 +181,9 @@ def firecrawl_eval_suite() -> EvalSuite:
|
|||
name="Get crawl status",
|
||||
user_message="Actually cancel it.",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
cancel_crawl,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=cancel_crawl,
|
||||
args={
|
||||
"crawl_id": "2ee7ba77-4ba0-4a45-9e2f-1c9e9a56f29b",
|
||||
},
|
||||
)
|
||||
|
|
@ -224,9 +225,9 @@ def firecrawl_eval_suite() -> EvalSuite:
|
|||
name="Map a website",
|
||||
user_message="Map the website at https://wikipedia.com with a limit of 100000 links. Only the links that are about the topic of AI",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
map_website,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=map_website,
|
||||
args={
|
||||
"url": "https://wikipedia.com",
|
||||
"search": "AI",
|
||||
"limit": 100000,
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ from arcade.sdk.eval import (
|
|||
BinaryCritic,
|
||||
EvalRubric,
|
||||
EvalSuite,
|
||||
ExpectedToolCall,
|
||||
tool_eval,
|
||||
)
|
||||
|
||||
|
|
@ -77,9 +78,11 @@ def x_eval_suite() -> EvalSuite:
|
|||
"at Arcade AI!'"
|
||||
),
|
||||
expected_tool_calls=[
|
||||
(
|
||||
post_tweet,
|
||||
{"tweet_text": "Hello World! Exciting stuff is happening over at Arcade AI!"},
|
||||
ExpectedToolCall(
|
||||
func=post_tweet,
|
||||
args={
|
||||
"tweet_text": "Hello World! Exciting stuff is happening over at Arcade AI!",
|
||||
},
|
||||
)
|
||||
],
|
||||
critics=[
|
||||
|
|
@ -94,9 +97,9 @@ def x_eval_suite() -> EvalSuite:
|
|||
name="Delete a tweet by ID",
|
||||
user_message="Please delete the tweet with ID '148975632'.",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
delete_tweet_by_id,
|
||||
{"tweet_id": "148975632"},
|
||||
ExpectedToolCall(
|
||||
func=delete_tweet_by_id,
|
||||
args={"tweet_id": "148975632"},
|
||||
)
|
||||
],
|
||||
critics=[
|
||||
|
|
@ -111,9 +114,9 @@ def x_eval_suite() -> EvalSuite:
|
|||
name="Search recent tweets by username",
|
||||
user_message="Show me the recent tweets from 'elonmusk'.",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
search_recent_tweets_by_username,
|
||||
{"username": "elonmusk", "max_results": 10},
|
||||
ExpectedToolCall(
|
||||
func=search_recent_tweets_by_username,
|
||||
args={"username": "elonmusk", "max_results": 10},
|
||||
)
|
||||
],
|
||||
critics=[
|
||||
|
|
@ -129,9 +132,9 @@ def x_eval_suite() -> EvalSuite:
|
|||
user_message="Get the next 42",
|
||||
additional_messages=search_recent_tweets_by_username_history,
|
||||
expected_tool_calls=[
|
||||
(
|
||||
search_recent_tweets_by_username,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=search_recent_tweets_by_username,
|
||||
args={
|
||||
"username": "elonmusk",
|
||||
"max_results": 42,
|
||||
"next_token": "b26v89c19zqg8o3frr3tekall7a7ooom3sctaw30rz62l",
|
||||
|
|
@ -158,9 +161,9 @@ def x_eval_suite() -> EvalSuite:
|
|||
name="Lookup user by username",
|
||||
user_message="Can you get information about the user '@jack'?",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
lookup_single_user_by_username,
|
||||
{"username": "jack"},
|
||||
ExpectedToolCall(
|
||||
func=lookup_single_user_by_username,
|
||||
args={"username": "jack"},
|
||||
)
|
||||
],
|
||||
critics=[
|
||||
|
|
@ -176,9 +179,9 @@ def x_eval_suite() -> EvalSuite:
|
|||
name="Search recent tweets by keywords",
|
||||
user_message="Find recent tweets containing 'Arcade AI'.",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
search_recent_tweets_by_keywords,
|
||||
{
|
||||
ExpectedToolCall(
|
||||
func=search_recent_tweets_by_keywords,
|
||||
args={
|
||||
"keywords": None,
|
||||
"phrases": ["Arcade AI"],
|
||||
"max_results": 10,
|
||||
|
|
@ -202,11 +205,9 @@ def x_eval_suite() -> EvalSuite:
|
|||
name="Lookup tweet by ID",
|
||||
user_message="Can you provide details about the tweet with ID '123456789'?",
|
||||
expected_tool_calls=[
|
||||
(
|
||||
lookup_tweet_by_id,
|
||||
{
|
||||
"tweet_id": "123456789",
|
||||
},
|
||||
ExpectedToolCall(
|
||||
func=lookup_tweet_by_id,
|
||||
args={"tweet_id": "123456789"},
|
||||
)
|
||||
],
|
||||
critics=[
|
||||
|
|
|
|||
Loading…
Reference in a new issue