from abc import ABC, abstractmethod from dataclasses import dataclass from datetime import timedelta from typing import Any, ClassVar import pytz from dateutil import parser from arcade_evals.errors import WeightError from arcade_evals.weights import FuzzyWeight, Weight, resolve_weight @dataclass class Critic(ABC): """ Base class for all critics. Attributes: critic_field: The field name this critic evaluates. weight: The weight for this critic. Can be a float (0.0-1.0) or FuzzyWeight enum. When using FuzzyWeight, weights are auto-normalized to sum to 1.0. """ critic_field: str weight: Weight def __post_init__(self) -> None: if isinstance(self.weight, FuzzyWeight): return if self.weight < 0: raise WeightError(f"Critic weight must be non-negative, got {self.weight}") @property def resolved_weight(self) -> float: """Get the weight as a float value.""" return resolve_weight(self.weight) @abstractmethod def evaluate(self, expected: Any, actual: Any) -> dict[str, Any]: pass @dataclass class NoneCritic(Critic): """ A critic that has no effect on the evaluation results and does not actually evaluate. If a critic is not found for an evaluation case's field, then a NoneCritic is used to indicate that the field was not criticized. """ # Marker attribute to identify placeholder critics without isinstance checks # (avoids circular imports in weights.py) _is_placeholder: ClassVar[bool] = True weight: float = 0.0 def __post_init__(self) -> None: self.weight = 0.0 super().__post_init__() def evaluate(self, expected: Any, actual: Any) -> dict[str, Any]: return {"match": None, "score": self.weight, "is_criticized": False} @dataclass class BinaryCritic(Critic): """ A critic for performing exact equality comparisons between expected and actual values. This critic evaluates whether the expected and actual values are exactly equal. It's useful for scenarios where only an exact match is acceptable. Returns: A dict with: - "match": True if expected == actual, otherwise False. - "score": The full weight if there's a match, otherwise 0.0. """ def cast_actual(self, expected: Any, actual: Any) -> Any: """ Casts the actual value to the type of the expected value. Args: expected (Any): The expected value whose type will be used for casting. actual (Any): The actual value to be cast. Returns: Any: The actual value cast to the type of the expected value. Raises: TypeError: If the casting is not possible. """ # In case both are strings. if actual == "None": actual = None if expected == "None": expected = None if expected is None: # No need to cast; return actual as is return actual if actual is None: # No need to cast; return None return None expected_type = type(expected) try: return expected_type(actual) except (ValueError, TypeError) as e: raise TypeError( f"Cannot cast actual value '{actual}' to type {expected_type.__name__}: {e}" ) from e def evaluate(self, expected: Any, actual: Any) -> dict[str, float | bool]: """ Evaluates whether the expected and actual values are exactly equal after casting. Args: expected: The expected value. actual: The actual value to compare, cast to the type of expected. Returns: dict: A dictionary containing the match status and score. """ # Cast actual to the type of expected try: actual_casted = self.cast_actual(expected, actual) # TODO log or something better here except TypeError: actual_casted = actual match = expected == actual_casted return {"match": match, "score": self.resolved_weight if match else 0.0} @dataclass class NumericCritic(Critic): """ A critic for evaluating numeric values within a specified range. This critic performs a "fuzzy" comparison of numeric values, where values closer to each other (relative to the specified range) result in higher scores. It's useful for scenarios where exact matches aren't necessary, but closeness within a certain tolerance is rewarded. Attributes: value_range: The min and max values of the expected range. match_threshold: The threshold for considering a match (default 0.8). The evaluation process: 1. Normalizes both expected and actual values to a 0-1 scale based on value_range. 2. Calculates the absolute difference between these normalized values. 3. Subtracts this difference from 1 to get a similarity score (closer to 1 is more similar). 4. Multiplies the similarity by the critic's weight for the final score. Returns: A dict with: - "match": True if the score >= match_threshold, otherwise False. - "score": The calculated score (similarity * weight). """ value_range: tuple[float, float] match_threshold: float = 0.8 def __init__( self, critic_field: str, weight: float, value_range: tuple[float, float], match_threshold: float = 0.8, ): super().__init__(critic_field, weight) if value_range[0] >= value_range[1]: raise ValueError("Invalid value_range: minimum must be less than maximum.") self.value_range = value_range self.match_threshold = match_threshold def evaluate(self, expected: Any, actual: Any) -> dict[str, Any]: min_val, max_val = self.value_range normalized_expected = float((float(expected) - min_val) / (max_val - min_val)) normalized_actual = float((float(actual) - min_val) / (max_val - min_val)) score = float(1 - abs(normalized_expected - normalized_actual)) return { "match": bool(score >= self.match_threshold), "score": float(score * self.resolved_weight), } @dataclass class SimilarityCritic(Critic): """ A critic for evaluating the similarity between two strings. This critic uses a specified similarity metric to compare the expected and actual string values. Currently, it supports cosine similarity using TF-IDF vectorization. Args: metric: The similarity metric to use (default is "cosine"). similarity_threshold: The threshold for considering a match (default 0.8). The evaluation process: 1. Converts both expected and actual values to strings. 2. Calculates the similarity score using the specified metric. 3. Determines a match based on the similarity_threshold. 4. Calculates the final score by multiplying the similarity by the critic's weight. Returns: A dict with: - "match": True if similarity >= similarity_threshold, otherwise False. - "score": The calculated score (similarity * weight). Raises: ImportError: If scikit-learn is not installed (required for cosine similarity). ValueError: If an unsupported similarity metric is specified. """ metric: str = "cosine" similarity_threshold: float = 0.8 SUPPORTED_METRICS: ClassVar[list[str]] = ["cosine"] def __init__( self, critic_field: str, weight: float, similarity_threshold: float = 0.8, metric: str = "cosine", ): super().__init__(critic_field, weight) if metric not in self.SUPPORTED_METRICS: raise ValueError(f"Unsupported similarity metric: {metric}") self.similarity_threshold = similarity_threshold self.metric = metric def evaluate(self, expected: Any, actual: Any) -> dict[str, float | bool]: # IMPORTANT: Convert non-string values to strings before TF-IDF comparison. # sklearn's TfidfVectorizer calls .lower() on inputs, which fails on lists/dicts. # This commonly occurs when SimilarityCritic is used for tool arguments that are # lists (e.g., teams_to_add=["Engineering", "Platform"]) instead of strings. # Lists are joined with spaces to create comparable text representations. if not isinstance(expected, str): expected = ( " ".join(str(item) for item in expected) if isinstance(expected, list) else str(expected) ) if not isinstance(actual, str): actual = ( " ".join(str(item) for item in actual) if isinstance(actual, list) else str(actual) ) if self.metric == "cosine": try: from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.metrics.pairwise import cosine_similarity except ImportError: raise ImportError( "Use `pip install 'arcade-evals` to install the required dependencies for similarity metrics." ) # Handle edge case: empty strings or strings with no valid tokens # TfidfVectorizer fails with "empty vocabulary" for such inputs if not expected.strip() or not actual.strip(): # Both empty = match, one empty = no match is_match = expected.strip() == actual.strip() return { "match": is_match, "score": self.resolved_weight if is_match else 0.0, } try: vectorizer = TfidfVectorizer() tfidf_matrix = vectorizer.fit_transform([expected, actual]) similarity = float(cosine_similarity(tfidf_matrix[0], tfidf_matrix[1])[0][0]) except ValueError: # TfidfVectorizer raises ValueError for empty vocabulary # (e.g., only numbers/punctuation which get filtered as stop words) # Fall back to exact string match is_match = expected == actual return { "match": is_match, "score": self.resolved_weight if is_match else 0.0, } else: raise ValueError(f"Unsupported similarity metric: {self.metric}") return { "match": similarity >= self.similarity_threshold, "score": min(similarity * self.resolved_weight, self.resolved_weight), } @dataclass class DatetimeCritic(Critic): """ A critic that evaluates the closeness of datetime values within a specified tolerance. Attributes: tolerance: Acceptable timedelta between expected and actual datetimes. max_difference: Maximum timedelta for a partial score. """ critic_field: str weight: float tolerance: timedelta = timedelta(seconds=500) max_difference: timedelta = timedelta(hours=2) def evaluate(self, expected: str, actual: str) -> dict[str, float | bool]: """Evaluates the closeness of datetime values within a specified tolerance.""" # Attempt to parse expected and actual datetime strings try: expected_dt = parser.parse(expected) actual_dt = parser.parse(actual) except (ValueError, TypeError): # If parsing fails, return score 0 return {"match": False, "score": 0.0} # Handle cases based on presence of tzinfo if expected_dt.tzinfo is None and actual_dt.tzinfo is None: # Both datetimes are naive, compare directly time_diff_seconds = abs((expected_dt - actual_dt).total_seconds()) elif expected_dt.tzinfo is not None and actual_dt.tzinfo is not None: # Both datetimes have tzinfo, compare in UTC expected_utc = expected_dt.astimezone(pytz.utc) actual_utc = actual_dt.astimezone(pytz.utc) time_diff_seconds = abs((expected_utc - actual_utc).total_seconds()) else: # One datetime has tzinfo and the other doesn't # Compare naive datetime with the other's naive equivalent if expected_dt.tzinfo is not None: expected_naive = expected_dt.replace(tzinfo=None) time_diff_seconds = abs((expected_naive - actual_dt).total_seconds()) else: actual_naive = actual_dt.replace(tzinfo=None) time_diff_seconds = abs((expected_dt - actual_naive).total_seconds()) # Convert tolerances to seconds tolerance_seconds = self.tolerance.total_seconds() max_difference_seconds = self.max_difference.total_seconds() if time_diff_seconds <= tolerance_seconds: # Full score if within tolerance return {"match": True, "score": self.resolved_weight} elif time_diff_seconds >= max_difference_seconds: # No score if beyond max_difference return {"match": False, "score": 0.0} else: # Partial score based on time difference ratio = 1 - (time_diff_seconds / max_difference_seconds) # Ensure ratio is not negative ratio = max(ratio, 0) score = self.resolved_weight * ratio return {"match": False, "score": score}