Eval Framework Improvements (#86)

This PR introduces enhancements and fixes to the Evaluation Framework to
improve accuracy, robustness, and performance.

## Key Improvements

- **Refactored Evaluation Loading**: Extracted evaluation file loading
and suite loading logic into utility functions `get_eval_files` and
`load_eval_suites` in `arcade/cli/utils.py`.
  - **Benefit**: Enhances code modularity and maintainability.

- **Asynchronous Execution of Evaluations**: Modified the evaluations to
run asynchronously using `asyncio`.
- **Benefit**: Significantly reduces total execution time when running
multiple evaluations.

- **Improved Error Handling**: Wrapped critic evaluations in try-except
blocks to handle exceptions gracefully.
- **Benefit**: Ensures that a single failing critic doesn't halt the
entire evaluation process.

## Other Changes

- **Case-Insensitive Tool Name Comparison**: Made tool name comparisons
case-insensitive to improve robustness against casing differences.

- **Refactored Cost Matrix Creation**: Revised cost matrix creation to
handle varying numbers of expected and actual tool calls properly,
ensuring accurate assignment and scoring.

- **Type Casting in `BinaryCritic`**: Added type casting for actual
values to match the expected value's type before comparison, improving
accuracy in evaluations.

- **Removed Synchronous Code Paths**: Simplified the codebase by
removing synchronous evaluation methods, focusing on asynchronous
execution.

- **General Code Cleanup**: Removed unused imports and performed general
code cleanup to enhance readability and maintainability.


## Example

For the google calendar and gmail tools eval set you can run

```go
> arcade evals . -c 8 --models gpt-4o,gpt-4o-mini,gpt-4,gpt-4-turbo

Running evaluations in calendar_eval_suite
Running evaluations in gmail_eval_suite
Model: gpt-4o
PASSED Create calendar event -- Score: 1.00
FAILED List calendar events -- Score: 0.86
PASSED Update a calendar event -- Score: 1.00
PASSED Delete a calendar event -- Score: 1.00
Model: gpt-4o-mini
FAILED Create calendar event -- Score: 0.84
FAILED List calendar events -- Score: 0.86
PASSED Update a calendar event -- Score: 1.00
PASSED Delete a calendar event -- Score: 1.00
Model: gpt-4
PASSED Create calendar event -- Score: 1.00
FAILED List calendar events -- Score: 0.86
PASSED Update a calendar event -- Score: 1.00
PASSED Delete a calendar event -- Score: 1.00
Model: gpt-4-turbo
PASSED Create calendar event -- Score: 1.00
FAILED List calendar events -- Score: 0.87
PASSED Update a calendar event -- Score: 1.00
PASSED Delete a calendar event -- Score: 1.00
Model: gpt-4o
PASSED Send email to user with clear username -- Score: 1.00
Model: gpt-4o-mini
PASSED Send email to user with clear username -- Score: 1.00
Model: gpt-4
PASSED Send email to user with clear username -- Score: 1.00
Model: gpt-4-turbo
PASSED Send email to user with clear username -- Score: 1.00
Summary -- Total: 20 -- Passed: 15 -- Failed: 5
```

arcade/arcade/cli/main.py

@@ -1,10 +1,9 @@
-import importlib.util
+import asyncio
 import os
 import readline
 import threading
 import uuid
 import webbrowser
-from pathlib import Path
 from typing import Any, Optional
 from urllib.parse import urlencode
 
@@ -23,8 +22,10 @@ from arcade.cli.utils import (
     create_cli_catalog,
     display_eval_results,
     display_tool_messages,
+    get_eval_files,
     handle_chat_interaction,
     is_authorization_pending,
+    load_eval_suites,  # Import the new function
     validate_and_get_config,
     wait_for_authorization_completion,
 )
@@ -429,30 +430,10 @@ def evals(
     """
     config = _get_config_with_overrides(force_tls, force_no_tls, host, port)
 
-    models = models.split(",")  # type: ignore[assignment]
+    models_list = models.split(",")  # Use 'models_list' to avoid shadowing
-    directory_path = Path(directory).resolve()
-
-    if directory_path.is_dir():
-        eval_files = [
-            f
-            for f in directory_path.iterdir()
-            if f.is_file() and f.name.startswith("eval_") and f.name.endswith(".py")
-        ]
-    elif directory_path.is_file():
-        eval_files = (
-            [directory_path]
-            if directory_path.name.startswith("eval_") and directory_path.name.endswith(".py")
-            else []
-        )
-    else:
-        console.print(f"Path not found: {directory_path}", style="bold red")
-        return
 
+    eval_files = get_eval_files(directory)
     if not eval_files:
-        console.print(
-            "No evaluation files found. Filenames must start with 'eval_' and end with '.py'.",
-            style="bold yellow",
-        )
         return
 
     if show_details:
@@ -464,42 +445,25 @@ def evals(
         )
 
     # Try to hit /health endpoint on engine and warn if it is down
-    client = Arcade(api_key=config.api.key, base_url=config.engine_url)
+    with Arcade(api_key=config.api.key, base_url=config.engine_url) as client:
-    log_engine_health(client)
+        log_engine_health(client)  # type: ignore[arg-type]
 
-    for eval_file_path in eval_files:
+    # Use the new function to load eval suites
-        module_name = eval_file_path.stem  # filename without extension
+    eval_suites = load_eval_suites(eval_files)
 
-        # Now we need to load the module from eval_file_path
+    if not eval_suites:
-        file_path_str = str(eval_file_path)
+        console.print("No evaluation suites to run.", style="bold yellow")
-        module_name_str = module_name
+        return
 
-        # Load using importlib
+    if show_details:
-        spec = importlib.util.spec_from_file_location(module_name_str, file_path_str)
+        suite_label = "suite" if len(eval_suites) == 1 else "suites"
-        if spec is None:
+        console.print(
-            console.print(f"Failed to load {eval_file_path}", style="bold red")
+            f"\nFound {len(eval_suites)} {suite_label} in the evaluation files.", style="bold"
-            continue
+        )
-
-        module = importlib.util.module_from_spec(spec)
-        spec.loader.exec_module(module)  # type: ignore[union-attr]
-
-        eval_suites = [
-            obj
-            for name, obj in module.__dict__.items()
-            if callable(obj) and hasattr(obj, "__tool_eval__")
-        ]
-
-        if not eval_suites:
-            console.print(f"No @tool_eval functions found in {eval_file_path}", style="bold yellow")
-            continue
-
-        if show_details:
-            suite_label = "suite" if len(eval_suites) == 1 else "suites"
-            console.print(
-                f"\nFound {len(eval_suites)} {suite_label} in {eval_file_path}", style="bold"
-            )
 
+    async def run_evaluations() -> None:
         all_evaluations = []
+        tasks = []
         for suite_func in eval_suites:
             console.print(
                 Text.assemble(
@@ -507,10 +471,21 @@ def evals(
                     (suite_func.__name__, "bold blue"),
                 )
             )
-            results = suite_func(config=config, models=models, max_concurrency=max_concurrent)
+            for model in models_list:
-            all_evaluations.append(results)
+                task = asyncio.create_task(
+                    suite_func(config=config, model=model, max_concurrency=max_concurrent)
+                )
+                tasks.append(task)
+
+        # TODO add a progress bar here
+        # TODO error handling on each eval
+        # Wait for all suite functions to complete
+        results = await asyncio.gather(*tasks)
+        all_evaluations.extend(results)
         display_eval_results(all_evaluations, show_details=show_details)
 
+    asyncio.run(run_evaluations())
+
 
 @cli.command(help="Launch Arcade AI locally for tool dev", rich_help_panel="Launch")
 def dev(

arcade/arcade/cli/utils.py

@@ -1,6 +1,8 @@
+import importlib.util
 import time
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, Any, Union
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Callable, Union
 
 import typer
 from openai.resources.chat.completions import ChatCompletionChunk, Stream
@@ -386,3 +388,88 @@ def is_authorization_pending(tool_authorization: dict | None) -> bool:
         tool_authorization is not None and tool_authorization.get("status", "") == "pending"
     )
     return is_auth_pending
+
+
+def get_eval_files(directory: str) -> list[Path]:
+    """
+    Get a list of evaluation files starting with 'eval_' and ending with '.py' in the given directory.
+
+    Args:
+        directory: The directory to search for evaluation files.
+
+    Returns:
+        A list of Paths to the evaluation files. Returns an empty list if no files are found.
+    """
+    directory_path = Path(directory).resolve()
+
+    if directory_path.is_dir():
+        eval_files = [
+            f
+            for f in directory_path.iterdir()
+            if f.is_file() and f.name.startswith("eval_") and f.name.endswith(".py")
+        ]
+    elif directory_path.is_file():
+        eval_files = (
+            [directory_path]
+            if directory_path.name.startswith("eval_") and directory_path.name.endswith(".py")
+            else []
+        )
+    else:
+        console.print(f"Path not found: {directory_path}", style="bold red")
+        return []
+
+    if not eval_files:
+        console.print(
+            "No evaluation files found. Filenames must start with 'eval_' and end with '.py'.",
+            style="bold yellow",
+        )
+        return []
+
+    return eval_files
+
+
+def load_eval_suites(eval_files: list[Path]) -> list[Callable]:
+    """
+    Load evaluation suites from the given eval_files by importing the modules
+    and extracting functions decorated with `@tool_eval`.
+
+    Args:
+        eval_files: A list of Paths to evaluation files.
+
+    Returns:
+        A list of callable evaluation suite functions.
+    """
+    eval_suites = []
+    for eval_file_path in eval_files:
+        module_name = eval_file_path.stem  # filename without extension
+
+        # Now we need to load the module from eval_file_path
+        file_path_str = str(eval_file_path)
+        module_name_str = module_name
+
+        # Load using importlib
+        spec = importlib.util.spec_from_file_location(module_name_str, file_path_str)
+        if spec is None:
+            console.print(f"Failed to load {eval_file_path}", style="bold red")
+            continue
+
+        module = importlib.util.module_from_spec(spec)
+        if spec.loader is not None:
+            spec.loader.exec_module(module)
+        else:
+            console.print(f"Failed to load module: {module_name}", style="bold red")
+            continue
+
+        eval_suite_funcs = [
+            obj
+            for name, obj in module.__dict__.items()
+            if callable(obj) and hasattr(obj, "__tool_eval__")
+        ]
+
+        if not eval_suite_funcs:
+            console.print(f"No @tool_eval functions found in {eval_file_path}", style="bold yellow")
+            continue
+
+        eval_suites.extend(eval_suite_funcs)
+
+    return eval_suites

arcade/arcade/sdk/eval/critic.py

@@ -33,8 +33,43 @@ class BinaryCritic(Critic):
             - "score": The full weight if there's a match, otherwise 0.0.
     """
 
+    def cast_actual(self, expected: Any, actual: Any) -> Any:
+        """
+        Casts the actual value to the type of the expected value.
+
+        Args:
+            expected (Any): The expected value whose type will be used for casting.
+            actual (Any): The actual value to be cast.
+
+        Returns:
+            Any: The actual value cast to the type of the expected value.
+
+        Raises:
+            TypeError: If the casting is not possible.
+        """
+        expected_type = type(expected)
+        try:
+            return expected_type(actual)
+        except (ValueError, TypeError) as e:
+            raise TypeError(
+                f"Cannot cast actual value '{actual}' to type {expected_type.__name__}: {e}"
+            ) from e
+
     def evaluate(self, expected: Any, actual: Any) -> dict[str, float | bool]:
-        match = expected == actual
+        """
+        Evaluates whether the expected and actual values are exactly equal after casting.
+
+        Args:
+            expected (Any): The expected value.
+            actual (Any): The actual value to compare, cast to the type of expected.
+
+        Returns:
+            dict[str, float | bool]: A dictionary containing the match status and score.
+        """
+        # Cast actual to the type of expected
+        actual_casted = self.cast_actual(expected, actual)
+
+        match = expected == actual_casted
         return {"match": match, "score": self.weight if match else 0.0}
 
 

arcade/arcade/sdk/eval/eval.py

@@ -15,8 +15,7 @@ except ImportError:
         "Use `pip install arcade-ai[evals]` to install the required dependencies for evaluation."
     )
 
-
+from arcade.client.client import AsyncArcade
-from arcade.client.client import Arcade, AsyncArcade
 from arcade.sdk.error import WeightError
 
 if TYPE_CHECKING:
@@ -199,10 +198,11 @@ class EvalCase:
         Returns:
             True if tool selection failure should occur, False otherwise.
         """
-        expected_tools = [tc.name for tc in self.expected_tool_calls]
+        sorted_expected_tools = sorted([tc.name for tc in self.expected_tool_calls])
+        sorted_actual_tools = sorted(actual_tools)
         return self.rubric.fail_on_tool_selection and not all(
             compare_tool_name(expected, actual)
-            for expected, actual in zip(expected_tools, actual_tools)
+            for expected, actual in zip(sorted_expected_tools, sorted_actual_tools)
         )
 
     def check_tool_call_quantity_failure(self, actual_count: int) -> bool:
@@ -270,7 +270,7 @@ class EvalCase:
         cost_matrix = self._create_cost_matrix(actual_tool_calls)
 
         # Use the Linear Sum Assignment (LSA) algorithm to find the optimal assignment
-        # The algorithm minimizes the cost of assigning each expected tool call to an actual tool call
+        # The algorithm maximizes the total score of the assignment
         row_ind, col_ind = linear_sum_assignment(cost_matrix, maximize=True)
 
         total_score = 0.0
@@ -292,12 +292,23 @@ class EvalCase:
                     expected_value = expected.args.get(critic.critic_field)
                     actual_value = actual_args.get(critic.critic_field)
                     if expected_value is not None and actual_value is not None:
-                        result = critic.evaluate(expected_value, actual_value)
+                        try:
-                        total_score += result["score"]
+                            result = critic.evaluate(expected_value, actual_value)
-                        total_weight += critic.weight
+                            total_score += result["score"]
-                        evaluation_result.add(
+                            total_weight += critic.weight
-                            critic.critic_field, result, critic.weight, expected_value, actual_value
+                            evaluation_result.add(
-                        )
+                                critic.critic_field,
+                                result,
+                                critic.weight,
+                                expected_value,
+                                actual_value,
+                            )
+                        except Exception as e:
+                            print(
+                                f"Critic evaluation failed for field '{critic.critic_field}': {e}"
+                            )
+                            # Depending on requirements, you might want to continue or handle differently
+                            continue
 
         # Compute the final score using the method from EvaluationResult
         evaluation_result.compute_final_score(total_weight)
@@ -327,27 +338,49 @@ class EvalCase:
         Returns:
             A numpy array representing the cost matrix.
         """
-        n = max(len(self.expected_tool_calls), len(actual_tool_calls))
+        num_expected = len(self.expected_tool_calls)
-        cost_matrix = np.zeros((n, n))
+        num_actual = len(actual_tool_calls)
+        n = max(num_expected, num_actual)
 
-        for i, expected in enumerate(self.expected_tool_calls):
+        # Initialize a score matrix with zeros
-            for j, (actual_tool, actual_args) in enumerate(actual_tool_calls):
+        score_matrix = np.zeros((n, n))
-                score = 0.0
-                if expected.name == actual_tool:
-                    score += self.rubric.tool_selection_weight
 
-                if self.critics:
+        for i in range(n):
-                    for critic in self.critics:
+            for j in range(n):
-                        expected_value = expected.args.get(critic.critic_field)
+                if i < num_expected and j < num_actual:
-                        actual_value = actual_args.get(critic.critic_field)
+                    expected = self.expected_tool_calls[i]
-                        if expected_value is not None and actual_value is not None:
+                    expected_tool = expected.name
-                            result = critic.evaluate(expected_value, actual_value)
+                    expected_args = expected.args
-                            score += result["score"]
+                    actual_tool, actual_args = actual_tool_calls[j]
-                cost_matrix[i, j] = score
+                    score = 0.0
 
-        return cost_matrix
+                    # Tool selection
+                    if compare_tool_name(expected_tool, actual_tool):
+                        score += self.rubric.tool_selection_weight
 
-    async def run_async(
+                    # Critics evaluation
+                    if self.critics:
+                        for critic in self.critics:
+                            expected_value = expected_args.get(critic.critic_field)
+                            actual_value = actual_args.get(critic.critic_field)
+                            if expected_value is not None and actual_value is not None:
+                                try:
+                                    result = critic.evaluate(expected_value, actual_value)
+                                    score += result.get("score", 0.0)
+                                except Exception as e:
+                                    print(
+                                        f"Critic evaluation failed for field '{critic.critic_field}': {e}"
+                                    )
+                                    continue
+
+                    score_matrix[i, j] = score
+                else:
+                    # Assign a score of 0 for dummy assignments
+                    score_matrix[i, j] = 0.0
+
+        return score_matrix
+
+    async def run(
         self, client: AsyncArcade, model: str, tool_names: list[FullyQualifiedName]
     ) -> dict[str, Any]:
         """
@@ -389,48 +422,6 @@ class EvalCase:
 
         return result
 
-    def run_sync(
-        self, client: Arcade, model: str, tool_names: list[FullyQualifiedName]
-    ) -> dict[str, Any]:
-        """
-        Run the evaluation case synchronously.
-
-        Args:
-            client: The Arcade client instance.
-            model: The model to evaluate.
-            tool_names: The list of tool names to use for the evaluation.
-        Returns:
-            A dictionary containing the evaluation result for the case.
-        """
-        messages = [{"role": "system", "content": self.system_message}]
-        messages.extend(list(self.additional_messages))
-        messages.append({"role": "user", "content": self.user_message})
-
-        response = client.chat.completions.create(  # type: ignore[call-overload]
-            model=model,
-            messages=messages,
-            tool_choice="auto",
-            tools=(str(name) for name in tool_names),
-            user="eval_user",
-            stream=False,
-        )
-
-        predicted_args = get_tool_args(response)
-
-        evaluation = self.evaluate(predicted_args)
-
-        result = {
-            "name": self.name,
-            "input": self.user_message,
-            "expected_tool_calls": [
-                {"name": tc.name, "args": tc.args} for tc in self.expected_tool_calls
-            ],
-            "predicted_tool_calls": [{"name": tool, "args": args} for tool, args in predicted_args],
-            "evaluation": evaluation,
-        }
-
-        return result
-
 
 @dataclass
 class EvalSuite:
@@ -445,7 +436,6 @@ class EvalSuite:
         system_message: The system message to be used for all cases in this suite.
         catalog: A ToolCatalog object containing registered tools.
         cases: A list of EvalCase objects representing individual test scenarios.
-        tool_choice: The tool choice mode for the AI model ("auto" or "function").
         rubric: The evaluation rubric for this case.
         max_concurrent: Maximum number of concurrent evaluations.
     """
@@ -455,23 +445,7 @@ class EvalSuite:
     catalog: "ToolCatalog"
     cases: list[EvalCase] = field(default_factory=list)
     rubric: EvalRubric = field(default_factory=EvalRubric)
-    max_concurrent: int = 1  # Default to sequential execution
+    max_concurrent: int = 1
-    _client: AsyncArcade | Arcade | None = None
-
-    def initialize_client(self, config: Config) -> None:
-        """
-        Initialize the client instance for the EvalSuite.
-        """
-        if self.max_concurrent > 1:
-            self._client = AsyncArcade(
-                api_key=config.api.key,
-                base_url=config.engine_url,
-            )
-        else:
-            self._client = Arcade(
-                api_key=config.api.key,
-                base_url=config.engine_url,
-            )
 
     def add_case(
         self,
@@ -568,10 +542,9 @@ class EvalSuite:
             critics=critics or (last_case.critics.copy() if last_case.critics else None),
             additional_messages=new_additional_messages,
         )
-
         self.cases.append(new_case)
 
-    async def run_async(self, model: str) -> dict[str, Any]:
+    async def run(self, client: AsyncArcade, model: str) -> dict[str, Any]:
         """
         Run the evaluation suite asynchronously.
 
@@ -581,9 +554,6 @@ class EvalSuite:
         Returns:
             A dictionary containing the evaluation results.
         """
-        if not self._client:
-            raise ValueError("Client not initialized. Call initialize_client() first.")
-
         results: dict[str, Any] = {"model": model, "rubric": self.rubric, "cases": []}
 
         semaphore = asyncio.Semaphore(self.max_concurrent)
@@ -591,7 +561,7 @@ class EvalSuite:
 
         async def sem_task(case: EvalCase) -> dict[str, Any]:
             async with semaphore:
-                return await case.run_async(self._client, model, tool_names)  # type: ignore[arg-type]
+                return await case.run(client, model, tool_names)
 
         tasks = [sem_task(case) for case in self.cases]
         case_results = await asyncio.gather(*tasks)
@@ -599,48 +569,6 @@ class EvalSuite:
         results["cases"] = case_results
         return results
 
-    def run_sync(self, model: str) -> dict[str, Any]:
-        """
-        Run the evaluation suite synchronously.
-
-        Args:
-            model: The model to evaluate.
-
-        Returns:
-            A dictionary containing the evaluation results.
-        """
-        if not self._client:
-            raise ValueError("Client not initialized. Call initialize_client() first.")
-
-        cases: list[dict[str, Any]] = []
-        results = {"model": model, "rubric": self.rubric, "cases": cases}
-        tool_names = list(self.catalog.get_tool_names())
-        for case in self.cases:
-            result = case.run_sync(self._client, model, tool_names)  # type: ignore[arg-type]
-            cases.append(result)
-
-        return results
-
-    def run(self, config: Config, model: str) -> dict[str, Any]:
-        """
-        Run the evaluation suite.
-
-        Args:
-            model: The model to evaluate.
-
-        Returns:
-            A dictionary containing the evaluation results.
-        """
-        if not self._client:
-            self.initialize_client(config)
-
-        if self.max_concurrent > 1:
-            # Run asynchronously with concurrency
-            return asyncio.run(self.run_async(model))
-        else:
-            # Run synchronously
-            return self.run_sync(model)
-
 
 def get_tool_args(chat_completion: Any) -> list[tuple[str, dict[str, Any]]]:
     """
@@ -675,15 +603,15 @@ def compare_tool_name(expected: str, actual: str) -> bool:
     actual_clean = "".join(char for char in actual if char not in separators)
 
     # Compare the cleaned names
-    return expected_clean == actual_clean
+    return expected_clean.lower() == actual_clean.lower()
 
 
 def tool_eval() -> Callable[[Callable], Callable]:
     def decorator(func: Callable) -> Callable:
         @functools.wraps(func)
-        def wrapper(
+        async def wrapper(
             config: Config,
-            models: list[str],
+            model: str,
             max_concurrency: int = 1,
         ) -> list[dict[str, Any]]:
             suite = func()
@@ -691,8 +619,11 @@ def tool_eval() -> Callable[[Callable], Callable]:
                 raise TypeError("Eval function must return an EvalSuite")
             suite.max_concurrent = max_concurrency
             results = []
-            for model in models:
+            async with AsyncArcade(
-                result = suite.run(config, model)
+                api_key=config.api.key,
+                base_url=config.engine_url,
+            ) as client:
+                result = await suite.run(client, model)  # type: ignore[arg-type]
                 results.append(result)
             return results