arcade-mcp/libs/arcade-evals/arcade_evals/_evalsuite/_comparative_execution.py

"""Comparative evaluation execution mixin for EvalSuite.

This module provides the execution logic for comparative evaluations,
allowing the same cases to be run against multiple tool tracks.
"""

from __future__ import annotations

import asyncio
import logging
import time
from typing import TYPE_CHECKING, Any

from arcade_evals._evalsuite._comparative import ComparativeCaseBuilder
from arcade_evals._evalsuite._types import (
    _VALID_PASS_RULES,
    PASS_RULE_LAST,
    ComparativeCase,
    EvalRubric,
)

if TYPE_CHECKING:
    from arcade_evals._evalsuite._providers import ProviderName
    from arcade_evals._evalsuite._tool_registry import EvalSuiteToolRegistry
    from arcade_evals._evalsuite._tracks import TrackManager

logger = logging.getLogger(__name__)


class _EvalSuiteComparativeMixin:
    """Mixin providing comparative evaluation execution methods."""

    # Type hints for attributes from EvalSuite
    name: str
    system_message: str
    rubric: EvalRubric  # EvalSuite always has a rubric (default_factory)
    max_concurrent: int
    _comparative_case_builders: list[ComparativeCaseBuilder]
    _track_manager: TrackManager
    _create_eval_case: Any  # Method from EvalSuite to create EvalCase
    _convert_to_named_expected_tool_call: Any  # Method from EvalSuite
    _add_none_critics: Any  # Method from EvalSuite
    _process_tool_calls: Any  # Method from EvalSuite
    _run_openai: Any  # Method from EvalSuite
    _run_anthropic: Any  # Method from EvalSuite

    async def _run_case_with_stats(
        self,
        case: Any,
        client: Any,
        model: str,
        provider: ProviderName,
        *,
        num_runs: int,
        seed: str | int | None,
        pass_rule: str,
        registry: EvalSuiteToolRegistry | None = None,
    ) -> dict[str, Any]:
        raise NotImplementedError  # Implemented in EvalSuite

    def add_comparative_case(
        self,
        name: str,
        user_message: str,
        system_message: str | None = None,
        additional_messages: list[dict[str, Any]] | None = None,
        rubric: EvalRubric | None = None,
    ) -> ComparativeCaseBuilder:
        """Create a comparative case that runs against multiple tool tracks.

        Use .for_track() on the returned builder to configure track-specific
        expected tool calls and critics.

        Args:
            name: Unique case name.
            user_message: User message (shared across all tracks).
            system_message: System message (shared, defaults to suite's system_message).
            additional_messages: Additional context messages (shared).
            rubric: Evaluation rubric (shared, defaults to suite's rubric).

        Returns:
            A ComparativeCaseBuilder for fluent track configuration.

        Example:
            suite.add_comparative_case(
                name="weather_query",
                user_message="What's the weather in NYC?",
            ).for_track(
                "Google Weather",
                expected_tool_calls=[ExpectedMCPToolCall("Google_GetWeather", city="NYC")],
                critics=[RangeCritic(field="temperature", min_val=0, max_val=100)],
            ).for_track(
                "OpenWeather",
                expected_tool_calls=[ExpectedMCPToolCall("get_current", location="NYC")],
                critics=[RangeCritic(field="main.temp", min_val=273, max_val=373)],
            )
        """
        builder = ComparativeCaseBuilder(
            suite=self,
            name=name,
            user_message=user_message,
            system_message=system_message or self.system_message,
            additional_messages=additional_messages,
            rubric=rubric or self.rubric,
        )
        # Store the builder (validated at execution time to allow fluent configuration)
        self._comparative_case_builders.append(builder)
        return builder

    async def run_comparative(
        self,
        client: Any,
        model: str,
        provider: ProviderName = "openai",
        num_runs: int = 1,
        seed: str | int | None = "constant",
        multi_run_pass_rule: str = PASS_RULE_LAST,
    ) -> dict[str, dict[str, Any]]:
        """Run comparative cases across all configured tracks.

        Args:
            client: The LLM client instance.
            model: The model to evaluate.
            provider: The provider name.
            num_runs: Number of runs per case.
            seed: Seed policy ("constant", "random", or an integer seed).
            multi_run_pass_rule: How to determine pass/warn for multi-run cases.

        Returns:
            Dictionary mapping track names to their results.
            Each track result contains:
                - model: The model name
                - suite_name: The suite name
                - track_name: The track name
                - cases: List of case results

        Example:
            results = await suite.run_comparative(client, "gpt-4o")
            # results["Google Weather"]["cases"][0] -> first case result
            # results["OpenWeather"]["cases"][0] -> same case, different track
        """
        if not self._comparative_case_builders:
            raise ValueError(
                "No comparative cases defined. Use add_comparative_case() to add cases."
            )

        # Validate upfront before making any API calls
        if num_runs < 1:
            raise ValueError("num_runs must be >= 1")
        if multi_run_pass_rule not in _VALID_PASS_RULES:
            raise ValueError(
                f"Invalid multi-run pass rule '{multi_run_pass_rule}'. "
                f"Valid values: {', '.join(sorted(_VALID_PASS_RULES))}"
            )

        # Build and validate all cases upfront
        comparative_cases: list[ComparativeCase] = []
        all_required_tracks: set[str] = set()
        for builder in self._comparative_case_builders:
            comp_case = builder.build()  # Validates that tracks are configured
            comparative_cases.append(comp_case)
            all_required_tracks.update(comp_case.track_configs.keys())

        # Validate all required tracks exist upfront (fail fast)
        missing_tracks = [t for t in all_required_tracks if not self._track_manager.has_track(t)]
        if missing_tracks:
            available = self._track_manager.get_track_names()
            raise ValueError(
                f"Missing track registries: {missing_tracks}. "
                f"Available tracks: {available}. "
                f"Ensure you registered tools with track='<track_name>'."
            )

        # Initialize track results structure
        track_results: dict[str, dict[str, Any]] = {}
        for track_name in all_required_tracks:
            track_results[track_name] = {
                "model": model,
                "suite_name": self.name,
                "track_name": track_name,
                "rubric": self.rubric,
                "cases": [],
            }

        # Prepare all async tasks for parallel execution
        semaphore = asyncio.Semaphore(self.max_concurrent)
        tasks: list[tuple[str, Any]] = []  # (track_name, task)

        for comp_case in comparative_cases:
            for track_name, track_config in comp_case.track_configs.items():
                registry = self._track_manager.get_registry(track_name)
                # We validated above that all registries exist, so this should never be None
                if registry is None:
                    raise RuntimeError(
                        f"Registry for '{track_name}' unexpectedly None after validation"
                    )

                # Create EvalCase from comparative case + track config
                expected_tool_calls = [
                    self._convert_to_named_expected_tool_call(tc)
                    for tc in track_config.expected_tool_calls
                ]
                critics = self._add_none_critics(expected_tool_calls, track_config.critics or [])

                eval_case = self._create_eval_case(
                    name=comp_case.name,
                    system_message=comp_case.system_message,
                    user_message=comp_case.user_message,
                    expected_tool_calls=expected_tool_calls,
                    rubric=comp_case.rubric or self.rubric,
                    critics=critics,
                    additional_messages=comp_case.additional_messages,
                )

                # Create task for this case+track combination
                async def run_track_case(
                    _case: Any,  # EvalCase
                    _reg: EvalSuiteToolRegistry,
                    _t_name: str,
                ) -> dict[str, Any]:
                    async with semaphore:
                        start = time.time()
                        logger.debug("[TASK START] %s @ %s", _case.name, _t_name)
                        case_result = await self._run_case_with_stats(
                            _case,
                            client,
                            model,
                            provider,
                            num_runs=num_runs,
                            seed=seed,
                            pass_rule=multi_run_pass_rule,
                            registry=_reg,
                        )
                        elapsed = time.time() - start
                        logger.debug("[TASK DONE] %s @ %s (%.1fs)", _case.name, _t_name, elapsed)

                        result = {
                            "name": _case.name,
                            "track": _t_name,
                            "input": _case.user_message,
                            "system_message": _case.system_message,
                            "additional_messages": _case.additional_messages,
                            "expected_tool_calls": [
                                {"name": tc.name, "args": tc.args}
                                for tc in _case.expected_tool_calls
                            ],
                            "predicted_tool_calls": [
                                {"name": name, "args": args}
                                for name, args in case_result["predicted_tool_calls"]
                            ],
                            "evaluation": case_result["evaluation"],
                        }
                        if num_runs > 1:
                            result["run_stats"] = case_result["run_stats"]
                            if case_result["critic_stats"]:
                                result["critic_stats"] = case_result["critic_stats"]
                        return result

                task = run_track_case(eval_case, registry, track_name)
                tasks.append((track_name, task))

        # Execute all tasks in parallel (respecting max_concurrent via semaphore)
        results = await asyncio.gather(*[task for _, task in tasks])

        # Organize results by track
        for (track_name, _), result in zip(tasks, results):
            track_results[track_name]["cases"].append(result)

        return track_results