arcade-mcp/libs/arcade-evals/arcade_evals/weights.py

"""
Weight definitions and normalization for arcade-evals.

This module contains:
- FuzzyWeight enum for qualitative weight assignment
- Weight type alias (float | FuzzyWeight)
- Normalization functions for critic weights
- Validation utilities for weight constraints
"""

from enum import Enum
from typing import TYPE_CHECKING

from arcade_evals.errors import WeightError

if TYPE_CHECKING:
    from arcade_evals.critic import Critic


def _is_placeholder_critic(critic: "Critic") -> bool:
    """
    Check if a critic is a placeholder (like NoneCritic).

    Uses duck typing via the _is_placeholder class attribute to avoid
    circular imports between weights.py and critic.py.
    """
    return getattr(critic, "_is_placeholder", False)


class FuzzyWeight(Enum):
    """
    Qualitative weight buckets for critic importance.

    Instead of manually calculating float weights, use these qualitative
    buckets to express relative importance. Weights are auto-normalized
    using Softmax-inspired scaling.

    Example:
        >>> critics = [
        ...     BinaryCritic(critic_field="owner", weight=FuzzyWeight.HIGH),
        ...     BinaryCritic(critic_field="state", weight=FuzzyWeight.LOW),
        ... ]
        # HIGH (5) gets 62.5% weight, LOW (3) gets 37.5% weight

    Weight Buckets (linear scale, uniform increment of 1):
        - MINIMAL: 1 - Almost negligible, rarely affects outcome
        - VERY_LOW: 2 - Rarely important, edge case checking
        - LOW: 3 - Minor importance
        - MEDIUM: 4 - Standard importance (default)
        - HIGH: 5 - Important parameter
        - VERY_HIGH: 6 - Critical, must-match parameter
        - CRITICAL: 7 - Absolutely essential, highest priority
    """

    MINIMAL = 1
    VERY_LOW = 2
    LOW = 3
    MEDIUM = 4
    HIGH = 5
    VERY_HIGH = 6
    CRITICAL = 7


# Type alias for weight parameter
Weight = float | FuzzyWeight


def normalize_fuzzy_weights(critics: list["Critic"]) -> list[float]:
    """
    Normalize a list of critic weights to sum to 1.0.

    Uses Softmax-inspired normalization: each weight is divided by the
    sum of all weights, ensuring:
    1. All weights sum to exactly 1.0
    2. Relative proportions are preserved

    Args:
        critics: List of critics with weight attributes.
                 Weights can be float or FuzzyWeight.

    Returns:
        List of normalized float weights in the same order as input critics.

    Example:
        >>> from arcade_evals.critic import BinaryCritic
        >>> critics = [
        ...     BinaryCritic("a", FuzzyWeight.HIGH),
        ...     BinaryCritic("b", FuzzyWeight.LOW),
        ... ]
        >>> normalize_fuzzy_weights(critics)
        [0.625, 0.375]  # HIGH (5) / (5 + 3), LOW (3) / (5 + 3)
    """
    if not critics:
        return []

    # Extract raw weight values (convert FuzzyWeight to float)
    raw_weights: list[float] = []
    for critic in critics:
        if isinstance(critic.weight, FuzzyWeight):
            raw_weights.append(float(critic.weight.value))
        else:
            raw_weights.append(float(critic.weight))

    # Calculate total for normalization
    total = sum(raw_weights)
    if total <= 0:
        # Edge case: all weights are zero or negative
        # Return zeros to indicate no scoring should occur
        return [0.0] * len(critics)

    # Normalize weights (simple division by sum)
    return [w / total for w in raw_weights]


def resolve_weight(weight: Weight) -> float:
    """
    Resolve a Weight value to a float.

    Used when a single weight needs to be resolved without full normalization.

    Args:
        weight: Either a float or FuzzyWeight enum.

    Returns:
        Float weight value.
    """
    if isinstance(weight, FuzzyWeight):
        return weight.value
    return float(weight)


# =============================================================================
# Critic Weight Validation and Normalization
# =============================================================================


def validate_and_normalize_critic_weights(critics: list["Critic"]) -> None:
    """
    Validate and normalize critic weights in-place.

    If any critic uses FuzzyWeight, all weights are normalized using
    Softmax-inspired scaling to sum to 1.0. Otherwise, validates that
    all float weights are non-negative.

    This function modifies critics in-place, setting their `weight` attribute
    to the normalized float value. The original weight is preserved in
    `_original_weight` for FuzzyWeight critics.

    Args:
        critics: List of critics to validate and normalize.

    Raises:
        WeightError: If any float weight is negative.

    Example:
        >>> critics = [
        ...     BinaryCritic(critic_field="a", weight=FuzzyWeight.HIGH),
        ...     BinaryCritic(critic_field="b", weight=FuzzyWeight.LOW),
        ... ]
        >>> validate_and_normalize_critic_weights(critics)
        >>> critics[0].weight  # Now normalized float
        0.625
    """
    if not critics:
        return

    # Check if any critic uses FuzzyWeight
    has_fuzzy = any(isinstance(c.weight, FuzzyWeight) for c in critics)

    if has_fuzzy:
        _normalize_fuzzy_critic_weights(critics)
    else:
        _validate_float_critic_weights(critics)


def _normalize_fuzzy_critic_weights(critics: list["Critic"]) -> None:
    """
    Normalize critic weights when FuzzyWeight is used.

    Filters out placeholder critics (like NoneCritic, which always has weight=0)
    and normalizes the remaining critics' weights to sum to 1.0.

    Args:
        critics: List of critics to normalize (modified in-place).
    """
    # Filter out placeholder critics for normalization (they keep weight=0)
    non_placeholder_critics = [c for c in critics if not _is_placeholder_critic(c)]

    if not non_placeholder_critics:
        return

    normalized = normalize_fuzzy_weights(non_placeholder_critics)

    for critic, norm_weight in zip(non_placeholder_critics, normalized):
        # Store original weight for reference
        critic._original_weight = critic.weight  # type: ignore[attr-defined]
        # Set normalized weight for evaluation
        critic.weight = norm_weight


def _validate_float_critic_weights(critics: list["Critic"]) -> None:
    """
    Validate that all float critic weights are non-negative.

    This is the legacy validation path used when no FuzzyWeight is present.
    Float weights are allowed to be any non-negative value; normalization
    happens implicitly through the scoring calculation.

    Args:
        critics: List of critics to validate.

    Raises:
        WeightError: If any weight is negative.
    """
    for critic in critics:
        if _is_placeholder_critic(critic):
            continue

        weight = resolve_weight(critic.weight)
        if weight < 0:
            raise WeightError(f"Critic weight must be non-negative, got {weight}")