admin-valentin/arcade-mcp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Eval Framework Improvements (#86)

Browse source 
This PR introduces enhancements and fixes to the Evaluation Framework to
improve accuracy, robustness, and performance.

## Key Improvements

- **Refactored Evaluation Loading**: Extracted evaluation file loading
and suite loading logic into utility functions `get_eval_files` and
`load_eval_suites` in `arcade/cli/utils.py`.
  - **Benefit**: Enhances code modularity and maintainability.

- **Asynchronous Execution of Evaluations**: Modified the evaluations to
run asynchronously using `asyncio`.
- **Benefit**: Significantly reduces total execution time when running
multiple evaluations.

- **Improved Error Handling**: Wrapped critic evaluations in try-except
blocks to handle exceptions gracefully.
- **Benefit**: Ensures that a single failing critic doesn't halt the
entire evaluation process.

## Other Changes

- **Case-Insensitive Tool Name Comparison**: Made tool name comparisons
case-insensitive to improve robustness against casing differences.

- **Refactored Cost Matrix Creation**: Revised cost matrix creation to
handle varying numbers of expected and actual tool calls properly,
ensuring accurate assignment and scoring.

- **Type Casting in `BinaryCritic`**: Added type casting for actual
values to match the expected value's type before comparison, improving
accuracy in evaluations.

- **Removed Synchronous Code Paths**: Simplified the codebase by
removing synchronous evaluation methods, focusing on asynchronous
execution.

- **General Code Cleanup**: Removed unused imports and performed general
code cleanup to enhance readability and maintainability.


## Example

For the google calendar and gmail tools eval set you can run

```go
> arcade evals . -c 8 --models gpt-4o,gpt-4o-mini,gpt-4,gpt-4-turbo

Running evaluations in calendar_eval_suite
Running evaluations in gmail_eval_suite
Model: gpt-4o
PASSED Create calendar event -- Score: 1.00
FAILED List calendar events -- Score: 0.86
PASSED Update a calendar event -- Score: 1.00
PASSED Delete a calendar event -- Score: 1.00
Model: gpt-4o-mini
FAILED Create calendar event -- Score: 0.84
FAILED List calendar events -- Score: 0.86
PASSED Update a calendar event -- Score: 1.00
PASSED Delete a calendar event -- Score: 1.00
Model: gpt-4
PASSED Create calendar event -- Score: 1.00
FAILED List calendar events -- Score: 0.86
PASSED Update a calendar event -- Score: 1.00
PASSED Delete a calendar event -- Score: 1.00
Model: gpt-4-turbo
PASSED Create calendar event -- Score: 1.00
FAILED List calendar events -- Score: 0.87
PASSED Update a calendar event -- Score: 1.00
PASSED Delete a calendar event -- Score: 1.00
Model: gpt-4o
PASSED Send email to user with clear username -- Score: 1.00
Model: gpt-4o-mini
PASSED Send email to user with clear username -- Score: 1.00
Model: gpt-4
PASSED Send email to user with clear username -- Score: 1.00
Model: gpt-4-turbo
PASSED Send email to user with clear username -- Score: 1.00
Summary -- Total: 20 -- Passed: 15 -- Failed: 5
```
This commit is contained in:
Sam Partee 2024-10-04 12:09:58 -07:00 committed by GitHub
parent f3feb85239
commit b52f6daa6f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 228 additions and 200 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

89
arcade/arcade/cli/main.py
View file

@ -1,10 +1,9 @@
import importlib.util
import asyncio
import os
import readline
import threading
import uuid
import webbrowser
from pathlib import Path
from typing import Any, Optional
from urllib.parse import urlencode
@ -23,8 +22,10 @@ from arcade.cli.utils import (
create_cli_catalog,
display_eval_results,
display_tool_messages,
get_eval_files,
handle_chat_interaction,
is_authorization_pending,
load_eval_suites, # Import the new function
validate_and_get_config,
wait_for_authorization_completion,
)
@ -429,30 +430,10 @@ def evals(
"""
config = _get_config_with_overrides(force_tls, force_no_tls, host, port)
models = models.split(",") # type: ignore[assignment]
directory_path = Path(directory).resolve()
if directory_path.is_dir():
eval_files = [
f
for f in directory_path.iterdir()
if f.is_file() and f.name.startswith("eval_") and f.name.endswith(".py")
]
elif directory_path.is_file():
eval_files = (
[directory_path]
if directory_path.name.startswith("eval_") and directory_path.name.endswith(".py")
else []
)
else:
console.print(f"Path not found: {directory_path}", style="bold red")
return
models_list = models.split(",") # Use 'models_list' to avoid shadowing
eval_files = get_eval_files(directory)
if not eval_files:
console.print(
"No evaluation files found. Filenames must start with 'eval_' and end with '.py'.",
style="bold yellow",
)
return
if show_details:
@ -464,42 +445,25 @@ def evals(
)
# Try to hit /health endpoint on engine and warn if it is down
client = Arcade(api_key=config.api.key, base_url=config.engine_url)
log_engine_health(client)
with Arcade(api_key=config.api.key, base_url=config.engine_url) as client:
log_engine_health(client) # type: ignore[arg-type]
for eval_file_path in eval_files:
module_name = eval_file_path.stem # filename without extension
# Use the new function to load eval suites
eval_suites = load_eval_suites(eval_files)
# Now we need to load the module from eval_file_path
file_path_str = str(eval_file_path)
module_name_str = module_name
if not eval_suites:
console.print("No evaluation suites to run.", style="bold yellow")
return
# Load using importlib
spec = importlib.util.spec_from_file_location(module_name_str, file_path_str)
if spec is None:
console.print(f"Failed to load {eval_file_path}", style="bold red")
continue
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module) # type: ignore[union-attr]
eval_suites = [
obj
for name, obj in module.__dict__.items()
if callable(obj) and hasattr(obj, "__tool_eval__")
]
if not eval_suites:
console.print(f"No @tool_eval functions found in {eval_file_path}", style="bold yellow")
continue
if show_details:
suite_label = "suite" if len(eval_suites) == 1 else "suites"
console.print(
f"\nFound {len(eval_suites)} {suite_label} in {eval_file_path}", style="bold"
)
if show_details:
suite_label = "suite" if len(eval_suites) == 1 else "suites"
console.print(
f"\nFound {len(eval_suites)} {suite_label} in the evaluation files.", style="bold"
)
async def run_evaluations() -> None:
all_evaluations = []
tasks = []
for suite_func in eval_suites:
console.print(
Text.assemble(
@ -507,10 +471,21 @@ def evals(
(suite_func.__name__, "bold blue"),
)
)
results = suite_func(config=config, models=models, max_concurrency=max_concurrent)
all_evaluations.append(results)
for model in models_list:
task = asyncio.create_task(
suite_func(config=config, model=model, max_concurrency=max_concurrent)
)
tasks.append(task)
# TODO add a progress bar here
# TODO error handling on each eval
# Wait for all suite functions to complete
results = await asyncio.gather(*tasks)
all_evaluations.extend(results)
display_eval_results(all_evaluations, show_details=show_details)
asyncio.run(run_evaluations())
@cli.command(help="Launch Arcade AI locally for tool dev", rich_help_panel="Launch")
def dev(

89
arcade/arcade/cli/utils.py
View file

@ -1,6 +1,8 @@
import importlib.util
import time
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Union
from pathlib import Path
from typing import TYPE_CHECKING, Any, Callable, Union
import typer
from openai.resources.chat.completions import ChatCompletionChunk, Stream
@ -386,3 +388,88 @@ def is_authorization_pending(tool_authorization: dict | None) -> bool:
tool_authorization is not None and tool_authorization.get("status", "") == "pending"
)
return is_auth_pending
def get_eval_files(directory: str) -> list[Path]:
"""
Get a list of evaluation files starting with 'eval_' and ending with '.py' in the given directory.
Args:
directory: The directory to search for evaluation files.
Returns:
A list of Paths to the evaluation files. Returns an empty list if no files are found.
"""
directory_path = Path(directory).resolve()
if directory_path.is_dir():
eval_files = [
f
for f in directory_path.iterdir()
if f.is_file() and f.name.startswith("eval_") and f.name.endswith(".py")
]
elif directory_path.is_file():
eval_files = (
[directory_path]
if directory_path.name.startswith("eval_") and directory_path.name.endswith(".py")
else []
)
else:
console.print(f"Path not found: {directory_path}", style="bold red")
return []
if not eval_files:
console.print(
"No evaluation files found. Filenames must start with 'eval_' and end with '.py'.",
style="bold yellow",
)
return []
return eval_files
def load_eval_suites(eval_files: list[Path]) -> list[Callable]:
"""
Load evaluation suites from the given eval_files by importing the modules
and extracting functions decorated with `@tool_eval`.
Args:
eval_files: A list of Paths to evaluation files.
Returns:
A list of callable evaluation suite functions.
"""
eval_suites = []
for eval_file_path in eval_files:
module_name = eval_file_path.stem # filename without extension
# Now we need to load the module from eval_file_path
file_path_str = str(eval_file_path)
module_name_str = module_name
# Load using importlib
spec = importlib.util.spec_from_file_location(module_name_str, file_path_str)
if spec is None:
console.print(f"Failed to load {eval_file_path}", style="bold red")
continue
module = importlib.util.module_from_spec(spec)
if spec.loader is not None:
spec.loader.exec_module(module)
else:
console.print(f"Failed to load module: {module_name}", style="bold red")
continue
eval_suite_funcs = [
obj
for name, obj in module.__dict__.items()
if callable(obj) and hasattr(obj, "__tool_eval__")
]
if not eval_suite_funcs:
console.print(f"No @tool_eval functions found in {eval_file_path}", style="bold yellow")
continue
eval_suites.extend(eval_suite_funcs)
return eval_suites

37
arcade/arcade/sdk/eval/critic.py
View file

@ -33,8 +33,43 @@ class BinaryCritic(Critic):
- "score": The full weight if there's a match, otherwise 0.0.
"""
def cast_actual(self, expected: Any, actual: Any) -> Any:
"""
Casts the actual value to the type of the expected value.
Args:
expected (Any): The expected value whose type will be used for casting.
actual (Any): The actual value to be cast.
Returns:
Any: The actual value cast to the type of the expected value.
Raises:
TypeError: If the casting is not possible.
"""
expected_type = type(expected)
try:
return expected_type(actual)
except (ValueError, TypeError) as e:
raise TypeError(
f"Cannot cast actual value '{actual}' to type {expected_type.__name__}: {e}"
) from e
def evaluate(self, expected: Any, actual: Any) -> dict[str, float | bool]:
match = expected == actual
"""
Evaluates whether the expected and actual values are exactly equal after casting.
Args:
expected (Any): The expected value.
actual (Any): The actual value to compare, cast to the type of expected.
Returns:
dict[str, float | bool]: A dictionary containing the match status and score.
"""
# Cast actual to the type of expected
actual_casted = self.cast_actual(expected, actual)
match = expected == actual_casted
return {"match": match, "score": self.weight if match else 0.0}

213
arcade/arcade/sdk/eval/eval.py
View file

@ -15,8 +15,7 @@ except ImportError:
"Use `pip install arcade-ai[evals]` to install the required dependencies for evaluation."
)
from arcade.client.client import Arcade, AsyncArcade
from arcade.client.client import AsyncArcade
from arcade.sdk.error import WeightError
if TYPE_CHECKING:
@ -199,10 +198,11 @@ class EvalCase:
Returns:
True if tool selection failure should occur, False otherwise.
"""
expected_tools = [tc.name for tc in self.expected_tool_calls]
sorted_expected_tools = sorted([tc.name for tc in self.expected_tool_calls])
sorted_actual_tools = sorted(actual_tools)
return self.rubric.fail_on_tool_selection and not all(
compare_tool_name(expected, actual)
for expected, actual in zip(expected_tools, actual_tools)
for expected, actual in zip(sorted_expected_tools, sorted_actual_tools)
)
def check_tool_call_quantity_failure(self, actual_count: int) -> bool:
@ -270,7 +270,7 @@ class EvalCase:
cost_matrix = self._create_cost_matrix(actual_tool_calls)
# Use the Linear Sum Assignment (LSA) algorithm to find the optimal assignment
# The algorithm minimizes the cost of assigning each expected tool call to an actual tool call
# The algorithm maximizes the total score of the assignment
row_ind, col_ind = linear_sum_assignment(cost_matrix, maximize=True)
total_score = 0.0
@ -292,12 +292,23 @@ class EvalCase:
expected_value = expected.args.get(critic.critic_field)
actual_value = actual_args.get(critic.critic_field)
if expected_value is not None and actual_value is not None:
result = critic.evaluate(expected_value, actual_value)
total_score += result["score"]
total_weight += critic.weight
evaluation_result.add(
critic.critic_field, result, critic.weight, expected_value, actual_value
)
try:
result = critic.evaluate(expected_value, actual_value)
total_score += result["score"]
total_weight += critic.weight
evaluation_result.add(
critic.critic_field,
result,
critic.weight,
expected_value,
actual_value,
)
except Exception as e:
print(
f"Critic evaluation failed for field '{critic.critic_field}': {e}"
)
# Depending on requirements, you might want to continue or handle differently
continue
# Compute the final score using the method from EvaluationResult
evaluation_result.compute_final_score(total_weight)
@ -327,27 +338,49 @@ class EvalCase:
Returns:
A numpy array representing the cost matrix.
"""
n = max(len(self.expected_tool_calls), len(actual_tool_calls))
cost_matrix = np.zeros((n, n))
num_expected = len(self.expected_tool_calls)
num_actual = len(actual_tool_calls)
n = max(num_expected, num_actual)
for i, expected in enumerate(self.expected_tool_calls):
for j, (actual_tool, actual_args) in enumerate(actual_tool_calls):
score = 0.0
if expected.name == actual_tool:
score += self.rubric.tool_selection_weight
# Initialize a score matrix with zeros
score_matrix = np.zeros((n, n))
if self.critics:
for critic in self.critics:
expected_value = expected.args.get(critic.critic_field)
actual_value = actual_args.get(critic.critic_field)
if expected_value is not None and actual_value is not None:
result = critic.evaluate(expected_value, actual_value)
score += result["score"]
cost_matrix[i, j] = score
for i in range(n):
for j in range(n):
if i < num_expected and j < num_actual:
expected = self.expected_tool_calls[i]
expected_tool = expected.name
expected_args = expected.args
actual_tool, actual_args = actual_tool_calls[j]
score = 0.0
return cost_matrix
# Tool selection
if compare_tool_name(expected_tool, actual_tool):
score += self.rubric.tool_selection_weight
async def run_async(
# Critics evaluation
if self.critics:
for critic in self.critics:
expected_value = expected_args.get(critic.critic_field)
actual_value = actual_args.get(critic.critic_field)
if expected_value is not None and actual_value is not None:
try:
result = critic.evaluate(expected_value, actual_value)
score += result.get("score", 0.0)
except Exception as e:
print(
f"Critic evaluation failed for field '{critic.critic_field}': {e}"
)
continue
score_matrix[i, j] = score
else:
# Assign a score of 0 for dummy assignments
score_matrix[i, j] = 0.0
return score_matrix
async def run(
self, client: AsyncArcade, model: str, tool_names: list[FullyQualifiedName]
) -> dict[str, Any]:
"""
@ -389,48 +422,6 @@ class EvalCase:
return result
def run_sync(
self, client: Arcade, model: str, tool_names: list[FullyQualifiedName]
) -> dict[str, Any]:
"""
Run the evaluation case synchronously.
Args:
client: The Arcade client instance.
model: The model to evaluate.
tool_names: The list of tool names to use for the evaluation.
Returns:
A dictionary containing the evaluation result for the case.
"""
messages = [{"role": "system", "content": self.system_message}]
messages.extend(list(self.additional_messages))
messages.append({"role": "user", "content": self.user_message})
response = client.chat.completions.create( # type: ignore[call-overload]
model=model,
messages=messages,
tool_choice="auto",
tools=(str(name) for name in tool_names),
user="eval_user",
stream=False,
)
predicted_args = get_tool_args(response)
evaluation = self.evaluate(predicted_args)
result = {
"name": self.name,
"input": self.user_message,
"expected_tool_calls": [
{"name": tc.name, "args": tc.args} for tc in self.expected_tool_calls
],
"predicted_tool_calls": [{"name": tool, "args": args} for tool, args in predicted_args],
"evaluation": evaluation,
}
return result
@dataclass
class EvalSuite:
@ -445,7 +436,6 @@ class EvalSuite:
system_message: The system message to be used for all cases in this suite.
catalog: A ToolCatalog object containing registered tools.
cases: A list of EvalCase objects representing individual test scenarios.
tool_choice: The tool choice mode for the AI model ("auto" or "function").
rubric: The evaluation rubric for this case.
max_concurrent: Maximum number of concurrent evaluations.
"""
@ -455,23 +445,7 @@ class EvalSuite:
catalog: "ToolCatalog"
cases: list[EvalCase] = field(default_factory=list)
rubric: EvalRubric = field(default_factory=EvalRubric)
max_concurrent: int = 1 # Default to sequential execution
_client: AsyncArcade | Arcade | None = None
def initialize_client(self, config: Config) -> None:
"""
Initialize the client instance for the EvalSuite.
"""
if self.max_concurrent > 1:
self._client = AsyncArcade(
api_key=config.api.key,
base_url=config.engine_url,
)
else:
self._client = Arcade(
api_key=config.api.key,
base_url=config.engine_url,
)
max_concurrent: int = 1
def add_case(
self,
@ -568,10 +542,9 @@ class EvalSuite:
critics=critics or (last_case.critics.copy() if last_case.critics else None),
additional_messages=new_additional_messages,
)
self.cases.append(new_case)
async def run_async(self, model: str) -> dict[str, Any]:
async def run(self, client: AsyncArcade, model: str) -> dict[str, Any]:
"""
Run the evaluation suite asynchronously.
@ -581,9 +554,6 @@ class EvalSuite:
Returns:
A dictionary containing the evaluation results.
"""
if not self._client:
raise ValueError("Client not initialized. Call initialize_client() first.")
results: dict[str, Any] = {"model": model, "rubric": self.rubric, "cases": []}
semaphore = asyncio.Semaphore(self.max_concurrent)
@ -591,7 +561,7 @@ class EvalSuite:
async def sem_task(case: EvalCase) -> dict[str, Any]:
async with semaphore:
return await case.run_async(self._client, model, tool_names) # type: ignore[arg-type]
return await case.run(client, model, tool_names)
tasks = [sem_task(case) for case in self.cases]
case_results = await asyncio.gather(*tasks)
@ -599,48 +569,6 @@ class EvalSuite:
results["cases"] = case_results
return results
def run_sync(self, model: str) -> dict[str, Any]:
"""
Run the evaluation suite synchronously.
Args:
model: The model to evaluate.
Returns:
A dictionary containing the evaluation results.
"""
if not self._client:
raise ValueError("Client not initialized. Call initialize_client() first.")
cases: list[dict[str, Any]] = []
results = {"model": model, "rubric": self.rubric, "cases": cases}
tool_names = list(self.catalog.get_tool_names())
for case in self.cases:
result = case.run_sync(self._client, model, tool_names) # type: ignore[arg-type]
cases.append(result)
return results
def run(self, config: Config, model: str) -> dict[str, Any]:
"""
Run the evaluation suite.
Args:
model: The model to evaluate.
Returns:
A dictionary containing the evaluation results.
"""
if not self._client:
self.initialize_client(config)
if self.max_concurrent > 1:
# Run asynchronously with concurrency
return asyncio.run(self.run_async(model))
else:
# Run synchronously
return self.run_sync(model)
def get_tool_args(chat_completion: Any) -> list[tuple[str, dict[str, Any]]]:
"""
@ -675,15 +603,15 @@ def compare_tool_name(expected: str, actual: str) -> bool:
actual_clean = "".join(char for char in actual if char not in separators)
# Compare the cleaned names
return expected_clean == actual_clean
return expected_clean.lower() == actual_clean.lower()
def tool_eval() -> Callable[[Callable], Callable]:
def decorator(func: Callable) -> Callable:
@functools.wraps(func)
def wrapper(
async def wrapper(
config: Config,
models: list[str],
model: str,
max_concurrency: int = 1,
) -> list[dict[str, Any]]:
suite = func()
@ -691,8 +619,11 @@ def tool_eval() -> Callable[[Callable], Callable]:
raise TypeError("Eval function must return an EvalSuite")
suite.max_concurrent = max_concurrency
results = []
for model in models:
result = suite.run(config, model)
async with AsyncArcade(
api_key=config.api.key,
base_url=config.engine_url,
) as client:
result = await suite.run(client, model) # type: ignore[arg-type]
results.append(result)
return results