From b52f6daa6f45d932d6a449d820e4390f3ed9c197 Mon Sep 17 00:00:00 2001 From: Sam Partee Date: Fri, 4 Oct 2024 12:09:58 -0700 Subject: [PATCH] Eval Framework Improvements (#86) This PR introduces enhancements and fixes to the Evaluation Framework to improve accuracy, robustness, and performance. ## Key Improvements - **Refactored Evaluation Loading**: Extracted evaluation file loading and suite loading logic into utility functions `get_eval_files` and `load_eval_suites` in `arcade/cli/utils.py`. - **Benefit**: Enhances code modularity and maintainability. - **Asynchronous Execution of Evaluations**: Modified the evaluations to run asynchronously using `asyncio`. - **Benefit**: Significantly reduces total execution time when running multiple evaluations. - **Improved Error Handling**: Wrapped critic evaluations in try-except blocks to handle exceptions gracefully. - **Benefit**: Ensures that a single failing critic doesn't halt the entire evaluation process. ## Other Changes - **Case-Insensitive Tool Name Comparison**: Made tool name comparisons case-insensitive to improve robustness against casing differences. - **Refactored Cost Matrix Creation**: Revised cost matrix creation to handle varying numbers of expected and actual tool calls properly, ensuring accurate assignment and scoring. - **Type Casting in `BinaryCritic`**: Added type casting for actual values to match the expected value's type before comparison, improving accuracy in evaluations. - **Removed Synchronous Code Paths**: Simplified the codebase by removing synchronous evaluation methods, focusing on asynchronous execution. - **General Code Cleanup**: Removed unused imports and performed general code cleanup to enhance readability and maintainability. ## Example For the google calendar and gmail tools eval set you can run ```go > arcade evals . -c 8 --models gpt-4o,gpt-4o-mini,gpt-4,gpt-4-turbo Running evaluations in calendar_eval_suite Running evaluations in gmail_eval_suite Model: gpt-4o PASSED Create calendar event -- Score: 1.00 FAILED List calendar events -- Score: 0.86 PASSED Update a calendar event -- Score: 1.00 PASSED Delete a calendar event -- Score: 1.00 Model: gpt-4o-mini FAILED Create calendar event -- Score: 0.84 FAILED List calendar events -- Score: 0.86 PASSED Update a calendar event -- Score: 1.00 PASSED Delete a calendar event -- Score: 1.00 Model: gpt-4 PASSED Create calendar event -- Score: 1.00 FAILED List calendar events -- Score: 0.86 PASSED Update a calendar event -- Score: 1.00 PASSED Delete a calendar event -- Score: 1.00 Model: gpt-4-turbo PASSED Create calendar event -- Score: 1.00 FAILED List calendar events -- Score: 0.87 PASSED Update a calendar event -- Score: 1.00 PASSED Delete a calendar event -- Score: 1.00 Model: gpt-4o PASSED Send email to user with clear username -- Score: 1.00 Model: gpt-4o-mini PASSED Send email to user with clear username -- Score: 1.00 Model: gpt-4 PASSED Send email to user with clear username -- Score: 1.00 Model: gpt-4-turbo PASSED Send email to user with clear username -- Score: 1.00 Summary -- Total: 20 -- Passed: 15 -- Failed: 5 ``` --- arcade/arcade/cli/main.py | 89 +++++-------- arcade/arcade/cli/utils.py | 89 ++++++++++++- arcade/arcade/sdk/eval/critic.py | 37 +++++- arcade/arcade/sdk/eval/eval.py | 213 +++++++++++-------------------- 4 files changed, 228 insertions(+), 200 deletions(-) diff --git a/arcade/arcade/cli/main.py b/arcade/arcade/cli/main.py index 27bf2f28..d6d4778d 100644 --- a/arcade/arcade/cli/main.py +++ b/arcade/arcade/cli/main.py @@ -1,10 +1,9 @@ -import importlib.util +import asyncio import os import readline import threading import uuid import webbrowser -from pathlib import Path from typing import Any, Optional from urllib.parse import urlencode @@ -23,8 +22,10 @@ from arcade.cli.utils import ( create_cli_catalog, display_eval_results, display_tool_messages, + get_eval_files, handle_chat_interaction, is_authorization_pending, + load_eval_suites, # Import the new function validate_and_get_config, wait_for_authorization_completion, ) @@ -429,30 +430,10 @@ def evals( """ config = _get_config_with_overrides(force_tls, force_no_tls, host, port) - models = models.split(",") # type: ignore[assignment] - directory_path = Path(directory).resolve() - - if directory_path.is_dir(): - eval_files = [ - f - for f in directory_path.iterdir() - if f.is_file() and f.name.startswith("eval_") and f.name.endswith(".py") - ] - elif directory_path.is_file(): - eval_files = ( - [directory_path] - if directory_path.name.startswith("eval_") and directory_path.name.endswith(".py") - else [] - ) - else: - console.print(f"Path not found: {directory_path}", style="bold red") - return + models_list = models.split(",") # Use 'models_list' to avoid shadowing + eval_files = get_eval_files(directory) if not eval_files: - console.print( - "No evaluation files found. Filenames must start with 'eval_' and end with '.py'.", - style="bold yellow", - ) return if show_details: @@ -464,42 +445,25 @@ def evals( ) # Try to hit /health endpoint on engine and warn if it is down - client = Arcade(api_key=config.api.key, base_url=config.engine_url) - log_engine_health(client) + with Arcade(api_key=config.api.key, base_url=config.engine_url) as client: + log_engine_health(client) # type: ignore[arg-type] - for eval_file_path in eval_files: - module_name = eval_file_path.stem # filename without extension + # Use the new function to load eval suites + eval_suites = load_eval_suites(eval_files) - # Now we need to load the module from eval_file_path - file_path_str = str(eval_file_path) - module_name_str = module_name + if not eval_suites: + console.print("No evaluation suites to run.", style="bold yellow") + return - # Load using importlib - spec = importlib.util.spec_from_file_location(module_name_str, file_path_str) - if spec is None: - console.print(f"Failed to load {eval_file_path}", style="bold red") - continue - - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) # type: ignore[union-attr] - - eval_suites = [ - obj - for name, obj in module.__dict__.items() - if callable(obj) and hasattr(obj, "__tool_eval__") - ] - - if not eval_suites: - console.print(f"No @tool_eval functions found in {eval_file_path}", style="bold yellow") - continue - - if show_details: - suite_label = "suite" if len(eval_suites) == 1 else "suites" - console.print( - f"\nFound {len(eval_suites)} {suite_label} in {eval_file_path}", style="bold" - ) + if show_details: + suite_label = "suite" if len(eval_suites) == 1 else "suites" + console.print( + f"\nFound {len(eval_suites)} {suite_label} in the evaluation files.", style="bold" + ) + async def run_evaluations() -> None: all_evaluations = [] + tasks = [] for suite_func in eval_suites: console.print( Text.assemble( @@ -507,10 +471,21 @@ def evals( (suite_func.__name__, "bold blue"), ) ) - results = suite_func(config=config, models=models, max_concurrency=max_concurrent) - all_evaluations.append(results) + for model in models_list: + task = asyncio.create_task( + suite_func(config=config, model=model, max_concurrency=max_concurrent) + ) + tasks.append(task) + + # TODO add a progress bar here + # TODO error handling on each eval + # Wait for all suite functions to complete + results = await asyncio.gather(*tasks) + all_evaluations.extend(results) display_eval_results(all_evaluations, show_details=show_details) + asyncio.run(run_evaluations()) + @cli.command(help="Launch Arcade AI locally for tool dev", rich_help_panel="Launch") def dev( diff --git a/arcade/arcade/cli/utils.py b/arcade/arcade/cli/utils.py index b332c27d..a9fd983f 100644 --- a/arcade/arcade/cli/utils.py +++ b/arcade/arcade/cli/utils.py @@ -1,6 +1,8 @@ +import importlib.util import time from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Union +from pathlib import Path +from typing import TYPE_CHECKING, Any, Callable, Union import typer from openai.resources.chat.completions import ChatCompletionChunk, Stream @@ -386,3 +388,88 @@ def is_authorization_pending(tool_authorization: dict | None) -> bool: tool_authorization is not None and tool_authorization.get("status", "") == "pending" ) return is_auth_pending + + +def get_eval_files(directory: str) -> list[Path]: + """ + Get a list of evaluation files starting with 'eval_' and ending with '.py' in the given directory. + + Args: + directory: The directory to search for evaluation files. + + Returns: + A list of Paths to the evaluation files. Returns an empty list if no files are found. + """ + directory_path = Path(directory).resolve() + + if directory_path.is_dir(): + eval_files = [ + f + for f in directory_path.iterdir() + if f.is_file() and f.name.startswith("eval_") and f.name.endswith(".py") + ] + elif directory_path.is_file(): + eval_files = ( + [directory_path] + if directory_path.name.startswith("eval_") and directory_path.name.endswith(".py") + else [] + ) + else: + console.print(f"Path not found: {directory_path}", style="bold red") + return [] + + if not eval_files: + console.print( + "No evaluation files found. Filenames must start with 'eval_' and end with '.py'.", + style="bold yellow", + ) + return [] + + return eval_files + + +def load_eval_suites(eval_files: list[Path]) -> list[Callable]: + """ + Load evaluation suites from the given eval_files by importing the modules + and extracting functions decorated with `@tool_eval`. + + Args: + eval_files: A list of Paths to evaluation files. + + Returns: + A list of callable evaluation suite functions. + """ + eval_suites = [] + for eval_file_path in eval_files: + module_name = eval_file_path.stem # filename without extension + + # Now we need to load the module from eval_file_path + file_path_str = str(eval_file_path) + module_name_str = module_name + + # Load using importlib + spec = importlib.util.spec_from_file_location(module_name_str, file_path_str) + if spec is None: + console.print(f"Failed to load {eval_file_path}", style="bold red") + continue + + module = importlib.util.module_from_spec(spec) + if spec.loader is not None: + spec.loader.exec_module(module) + else: + console.print(f"Failed to load module: {module_name}", style="bold red") + continue + + eval_suite_funcs = [ + obj + for name, obj in module.__dict__.items() + if callable(obj) and hasattr(obj, "__tool_eval__") + ] + + if not eval_suite_funcs: + console.print(f"No @tool_eval functions found in {eval_file_path}", style="bold yellow") + continue + + eval_suites.extend(eval_suite_funcs) + + return eval_suites diff --git a/arcade/arcade/sdk/eval/critic.py b/arcade/arcade/sdk/eval/critic.py index 0ed3f847..8546b4eb 100644 --- a/arcade/arcade/sdk/eval/critic.py +++ b/arcade/arcade/sdk/eval/critic.py @@ -33,8 +33,43 @@ class BinaryCritic(Critic): - "score": The full weight if there's a match, otherwise 0.0. """ + def cast_actual(self, expected: Any, actual: Any) -> Any: + """ + Casts the actual value to the type of the expected value. + + Args: + expected (Any): The expected value whose type will be used for casting. + actual (Any): The actual value to be cast. + + Returns: + Any: The actual value cast to the type of the expected value. + + Raises: + TypeError: If the casting is not possible. + """ + expected_type = type(expected) + try: + return expected_type(actual) + except (ValueError, TypeError) as e: + raise TypeError( + f"Cannot cast actual value '{actual}' to type {expected_type.__name__}: {e}" + ) from e + def evaluate(self, expected: Any, actual: Any) -> dict[str, float | bool]: - match = expected == actual + """ + Evaluates whether the expected and actual values are exactly equal after casting. + + Args: + expected (Any): The expected value. + actual (Any): The actual value to compare, cast to the type of expected. + + Returns: + dict[str, float | bool]: A dictionary containing the match status and score. + """ + # Cast actual to the type of expected + actual_casted = self.cast_actual(expected, actual) + + match = expected == actual_casted return {"match": match, "score": self.weight if match else 0.0} diff --git a/arcade/arcade/sdk/eval/eval.py b/arcade/arcade/sdk/eval/eval.py index a88f2189..e11039fb 100644 --- a/arcade/arcade/sdk/eval/eval.py +++ b/arcade/arcade/sdk/eval/eval.py @@ -15,8 +15,7 @@ except ImportError: "Use `pip install arcade-ai[evals]` to install the required dependencies for evaluation." ) - -from arcade.client.client import Arcade, AsyncArcade +from arcade.client.client import AsyncArcade from arcade.sdk.error import WeightError if TYPE_CHECKING: @@ -199,10 +198,11 @@ class EvalCase: Returns: True if tool selection failure should occur, False otherwise. """ - expected_tools = [tc.name for tc in self.expected_tool_calls] + sorted_expected_tools = sorted([tc.name for tc in self.expected_tool_calls]) + sorted_actual_tools = sorted(actual_tools) return self.rubric.fail_on_tool_selection and not all( compare_tool_name(expected, actual) - for expected, actual in zip(expected_tools, actual_tools) + for expected, actual in zip(sorted_expected_tools, sorted_actual_tools) ) def check_tool_call_quantity_failure(self, actual_count: int) -> bool: @@ -270,7 +270,7 @@ class EvalCase: cost_matrix = self._create_cost_matrix(actual_tool_calls) # Use the Linear Sum Assignment (LSA) algorithm to find the optimal assignment - # The algorithm minimizes the cost of assigning each expected tool call to an actual tool call + # The algorithm maximizes the total score of the assignment row_ind, col_ind = linear_sum_assignment(cost_matrix, maximize=True) total_score = 0.0 @@ -292,12 +292,23 @@ class EvalCase: expected_value = expected.args.get(critic.critic_field) actual_value = actual_args.get(critic.critic_field) if expected_value is not None and actual_value is not None: - result = critic.evaluate(expected_value, actual_value) - total_score += result["score"] - total_weight += critic.weight - evaluation_result.add( - critic.critic_field, result, critic.weight, expected_value, actual_value - ) + try: + result = critic.evaluate(expected_value, actual_value) + total_score += result["score"] + total_weight += critic.weight + evaluation_result.add( + critic.critic_field, + result, + critic.weight, + expected_value, + actual_value, + ) + except Exception as e: + print( + f"Critic evaluation failed for field '{critic.critic_field}': {e}" + ) + # Depending on requirements, you might want to continue or handle differently + continue # Compute the final score using the method from EvaluationResult evaluation_result.compute_final_score(total_weight) @@ -327,27 +338,49 @@ class EvalCase: Returns: A numpy array representing the cost matrix. """ - n = max(len(self.expected_tool_calls), len(actual_tool_calls)) - cost_matrix = np.zeros((n, n)) + num_expected = len(self.expected_tool_calls) + num_actual = len(actual_tool_calls) + n = max(num_expected, num_actual) - for i, expected in enumerate(self.expected_tool_calls): - for j, (actual_tool, actual_args) in enumerate(actual_tool_calls): - score = 0.0 - if expected.name == actual_tool: - score += self.rubric.tool_selection_weight + # Initialize a score matrix with zeros + score_matrix = np.zeros((n, n)) - if self.critics: - for critic in self.critics: - expected_value = expected.args.get(critic.critic_field) - actual_value = actual_args.get(critic.critic_field) - if expected_value is not None and actual_value is not None: - result = critic.evaluate(expected_value, actual_value) - score += result["score"] - cost_matrix[i, j] = score + for i in range(n): + for j in range(n): + if i < num_expected and j < num_actual: + expected = self.expected_tool_calls[i] + expected_tool = expected.name + expected_args = expected.args + actual_tool, actual_args = actual_tool_calls[j] + score = 0.0 - return cost_matrix + # Tool selection + if compare_tool_name(expected_tool, actual_tool): + score += self.rubric.tool_selection_weight - async def run_async( + # Critics evaluation + if self.critics: + for critic in self.critics: + expected_value = expected_args.get(critic.critic_field) + actual_value = actual_args.get(critic.critic_field) + if expected_value is not None and actual_value is not None: + try: + result = critic.evaluate(expected_value, actual_value) + score += result.get("score", 0.0) + except Exception as e: + print( + f"Critic evaluation failed for field '{critic.critic_field}': {e}" + ) + continue + + score_matrix[i, j] = score + else: + # Assign a score of 0 for dummy assignments + score_matrix[i, j] = 0.0 + + return score_matrix + + async def run( self, client: AsyncArcade, model: str, tool_names: list[FullyQualifiedName] ) -> dict[str, Any]: """ @@ -389,48 +422,6 @@ class EvalCase: return result - def run_sync( - self, client: Arcade, model: str, tool_names: list[FullyQualifiedName] - ) -> dict[str, Any]: - """ - Run the evaluation case synchronously. - - Args: - client: The Arcade client instance. - model: The model to evaluate. - tool_names: The list of tool names to use for the evaluation. - Returns: - A dictionary containing the evaluation result for the case. - """ - messages = [{"role": "system", "content": self.system_message}] - messages.extend(list(self.additional_messages)) - messages.append({"role": "user", "content": self.user_message}) - - response = client.chat.completions.create( # type: ignore[call-overload] - model=model, - messages=messages, - tool_choice="auto", - tools=(str(name) for name in tool_names), - user="eval_user", - stream=False, - ) - - predicted_args = get_tool_args(response) - - evaluation = self.evaluate(predicted_args) - - result = { - "name": self.name, - "input": self.user_message, - "expected_tool_calls": [ - {"name": tc.name, "args": tc.args} for tc in self.expected_tool_calls - ], - "predicted_tool_calls": [{"name": tool, "args": args} for tool, args in predicted_args], - "evaluation": evaluation, - } - - return result - @dataclass class EvalSuite: @@ -445,7 +436,6 @@ class EvalSuite: system_message: The system message to be used for all cases in this suite. catalog: A ToolCatalog object containing registered tools. cases: A list of EvalCase objects representing individual test scenarios. - tool_choice: The tool choice mode for the AI model ("auto" or "function"). rubric: The evaluation rubric for this case. max_concurrent: Maximum number of concurrent evaluations. """ @@ -455,23 +445,7 @@ class EvalSuite: catalog: "ToolCatalog" cases: list[EvalCase] = field(default_factory=list) rubric: EvalRubric = field(default_factory=EvalRubric) - max_concurrent: int = 1 # Default to sequential execution - _client: AsyncArcade | Arcade | None = None - - def initialize_client(self, config: Config) -> None: - """ - Initialize the client instance for the EvalSuite. - """ - if self.max_concurrent > 1: - self._client = AsyncArcade( - api_key=config.api.key, - base_url=config.engine_url, - ) - else: - self._client = Arcade( - api_key=config.api.key, - base_url=config.engine_url, - ) + max_concurrent: int = 1 def add_case( self, @@ -568,10 +542,9 @@ class EvalSuite: critics=critics or (last_case.critics.copy() if last_case.critics else None), additional_messages=new_additional_messages, ) - self.cases.append(new_case) - async def run_async(self, model: str) -> dict[str, Any]: + async def run(self, client: AsyncArcade, model: str) -> dict[str, Any]: """ Run the evaluation suite asynchronously. @@ -581,9 +554,6 @@ class EvalSuite: Returns: A dictionary containing the evaluation results. """ - if not self._client: - raise ValueError("Client not initialized. Call initialize_client() first.") - results: dict[str, Any] = {"model": model, "rubric": self.rubric, "cases": []} semaphore = asyncio.Semaphore(self.max_concurrent) @@ -591,7 +561,7 @@ class EvalSuite: async def sem_task(case: EvalCase) -> dict[str, Any]: async with semaphore: - return await case.run_async(self._client, model, tool_names) # type: ignore[arg-type] + return await case.run(client, model, tool_names) tasks = [sem_task(case) for case in self.cases] case_results = await asyncio.gather(*tasks) @@ -599,48 +569,6 @@ class EvalSuite: results["cases"] = case_results return results - def run_sync(self, model: str) -> dict[str, Any]: - """ - Run the evaluation suite synchronously. - - Args: - model: The model to evaluate. - - Returns: - A dictionary containing the evaluation results. - """ - if not self._client: - raise ValueError("Client not initialized. Call initialize_client() first.") - - cases: list[dict[str, Any]] = [] - results = {"model": model, "rubric": self.rubric, "cases": cases} - tool_names = list(self.catalog.get_tool_names()) - for case in self.cases: - result = case.run_sync(self._client, model, tool_names) # type: ignore[arg-type] - cases.append(result) - - return results - - def run(self, config: Config, model: str) -> dict[str, Any]: - """ - Run the evaluation suite. - - Args: - model: The model to evaluate. - - Returns: - A dictionary containing the evaluation results. - """ - if not self._client: - self.initialize_client(config) - - if self.max_concurrent > 1: - # Run asynchronously with concurrency - return asyncio.run(self.run_async(model)) - else: - # Run synchronously - return self.run_sync(model) - def get_tool_args(chat_completion: Any) -> list[tuple[str, dict[str, Any]]]: """ @@ -675,15 +603,15 @@ def compare_tool_name(expected: str, actual: str) -> bool: actual_clean = "".join(char for char in actual if char not in separators) # Compare the cleaned names - return expected_clean == actual_clean + return expected_clean.lower() == actual_clean.lower() def tool_eval() -> Callable[[Callable], Callable]: def decorator(func: Callable) -> Callable: @functools.wraps(func) - def wrapper( + async def wrapper( config: Config, - models: list[str], + model: str, max_concurrency: int = 1, ) -> list[dict[str, Any]]: suite = func() @@ -691,8 +619,11 @@ def tool_eval() -> Callable[[Callable], Callable]: raise TypeError("Eval function must return an EvalSuite") suite.max_concurrent = max_concurrency results = [] - for model in models: - result = suite.run(config, model) + async with AsyncArcade( + api_key=config.api.key, + base_url=config.engine_url, + ) as client: + result = await suite.run(client, model) # type: ignore[arg-type] results.append(result) return results