"""Tests for evals_runner error handling."""

from typing import Any, cast
from unittest.mock import AsyncMock, MagicMock, patch

import pytest
from arcade_cli.evals_runner import (
    ALL_FORMATS,
    CaptureTaskResult,
    EvalTaskResult,
    _run_capture_task,
    _run_eval_task,
    parse_output_formats,
    run_capture,
    run_evaluations,
)
from arcade_cli.utils import ModelSpec, Provider
from arcade_evals import CaptureResult

RUN_RULE_LAST = "last"


class TestEvalTaskResult:
    """Test EvalTaskResult dataclass."""

    def test_from_success(self) -> None:
        """Test creating a successful result."""
        result = EvalTaskResult.from_success("test_suite", "gpt-4o", "openai", {"score": 0.9})
        assert result.success is True
        assert result.suite_name == "test_suite"
        assert result.model == "gpt-4o"
        assert result.provider == "openai"
        assert result.result == {"score": 0.9}
        assert result.error is None
        assert result.error_type is None

    def test_from_error(self) -> None:
        """Test creating a failed result from an exception."""
        error = ValueError("Something went wrong")
        result = EvalTaskResult.from_error("test_suite", "gpt-4o", "openai", error)
        assert result.success is False
        assert result.suite_name == "test_suite"
        assert result.model == "gpt-4o"
        assert result.provider == "openai"
        assert result.error == "Something went wrong"
        assert result.error_type == "ValueError"
        assert result.result is None

    def test_from_error_with_different_exception_types(self) -> None:
        """Test that error_type captures the correct exception class name."""
        errors = [
            (RuntimeError("runtime"), "RuntimeError"),
            (TypeError("type"), "TypeError"),
            (KeyError("key"), "KeyError"),
            (ConnectionError("conn"), "ConnectionError"),
        ]
        for error, expected_type in errors:
            result = EvalTaskResult.from_error("suite", "model", "openai", error)
            assert result.error_type == expected_type

    def test_display_name(self) -> None:
        """Test that display_name shows provider/model format."""
        result = EvalTaskResult.from_success("suite", "gpt-4o", "openai", {})
        assert result.display_name == "openai/gpt-4o"

        result2 = EvalTaskResult.from_success("suite", "claude-3-sonnet", "anthropic", {})
        assert result2.display_name == "anthropic/claude-3-sonnet"


class TestCaptureTaskResult:
    """Test CaptureTaskResult dataclass."""

    def test_from_success(self) -> None:
        """Test creating a successful capture result."""
        mock_captures = cast(
            list[CaptureResult],
            [
                MagicMock(spec=CaptureResult),
                MagicMock(spec=CaptureResult),
            ],
        )
        result = CaptureTaskResult.from_success("test_suite", "gpt-4o", "openai", mock_captures)
        assert result.success is True
        assert result.suite_name == "test_suite"
        assert result.model == "gpt-4o"
        assert result.provider == "openai"
        assert result.result == mock_captures
        assert result.error is None
        assert result.error_type is None

    def test_from_error(self) -> None:
        """Test creating a failed capture result."""
        error = RuntimeError("Capture failed")
        result = CaptureTaskResult.from_error("test_suite", "gpt-4o", "openai", error)
        assert result.success is False
        assert result.error == "Capture failed"
        assert result.error_type == "RuntimeError"
        assert result.result is None

    def test_display_name(self) -> None:
        """Test that display_name shows provider/model format."""
        result = CaptureTaskResult.from_success("suite", "gpt-4o", "openai", [])
        assert result.display_name == "openai/gpt-4o"


class TestRunEvalTask:
    """Test _run_eval_task error handling."""

    @pytest.mark.asyncio
    async def test_successful_task(self) -> None:
        """Test that successful task returns success result."""
        mock_suite = AsyncMock(return_value={"score": 0.95})
        mock_suite.__name__ = "test_suite"

        model_spec = ModelSpec(provider=Provider.OPENAI, model="gpt-4o", api_key="test-key")
        result = await _run_eval_task(
            suite_func=mock_suite,
            model_spec=model_spec,
            max_concurrent=1,
            num_runs=1,
            seed="constant",
            multi_run_pass_rule=RUN_RULE_LAST,
        )

        assert result.success is True
        assert result.result == {"score": 0.95}
        assert result.suite_name == "test_suite"
        assert result.model == "gpt-4o"
        assert result.provider == "openai"

    @pytest.mark.asyncio
    async def test_failed_task_returns_error_result(self) -> None:
        """Test that failed task returns error result instead of raising."""
        mock_suite = AsyncMock(side_effect=ValueError("API error"))
        mock_suite.__name__ = "test_suite"

        model_spec = ModelSpec(provider=Provider.OPENAI, model="gpt-4o", api_key="test-key")
        result = await _run_eval_task(
            suite_func=mock_suite,
            model_spec=model_spec,
            max_concurrent=1,
            num_runs=1,
            seed="constant",
            multi_run_pass_rule=RUN_RULE_LAST,
        )

        assert result.success is False
        assert result.error is not None
        assert "API error" in result.error
        assert result.error_type == "ValueError"
        assert result.result is None

    @pytest.mark.asyncio
    async def test_passes_correct_arguments_to_suite(self) -> None:
        """Test that correct arguments are passed to the suite function."""
        mock_suite = AsyncMock(return_value={"score": 1.0})
        mock_suite.__name__ = "test_suite"

        model_spec = ModelSpec(provider=Provider.ANTHROPIC, model="claude-sonnet", api_key="my-key")
        await _run_eval_task(
            suite_func=mock_suite,
            model_spec=model_spec,
            max_concurrent=5,
            include_context=False,
            num_runs=1,
            seed="constant",
            multi_run_pass_rule=RUN_RULE_LAST,
        )

        mock_suite.assert_called_once_with(
            provider_api_key="my-key",
            model="claude-sonnet",
            max_concurrency=5,
            provider="anthropic",
            include_context=False,
            num_runs=1,
            seed="constant",
            multi_run_pass_rule=RUN_RULE_LAST,
        )


class TestRunCaptureTask:
    """Test _run_capture_task error handling."""

    @pytest.mark.asyncio
    async def test_successful_capture_task(self) -> None:
        """Test that successful capture task returns success result."""
        mock_captures = [MagicMock()]
        mock_suite = AsyncMock(return_value=mock_captures)
        mock_suite.__name__ = "capture_suite"

        model_spec = ModelSpec(provider=Provider.OPENAI, model="gpt-4o", api_key="test-key")
        result = await _run_capture_task(
            suite_func=mock_suite,
            model_spec=model_spec,
            max_concurrent=1,
            include_context=True,
            num_runs=1,
            seed="constant",
        )

        assert result.success is True
        assert result.result == mock_captures

    @pytest.mark.asyncio
    async def test_failed_capture_task_returns_error_result(self) -> None:
        """Test that failed capture task returns error result."""
        mock_suite = AsyncMock(side_effect=ConnectionError("Network failed"))
        mock_suite.__name__ = "capture_suite"

        model_spec = ModelSpec(provider=Provider.OPENAI, model="gpt-4o", api_key="test-key")
        result = await _run_capture_task(
            suite_func=mock_suite,
            model_spec=model_spec,
            max_concurrent=1,
            include_context=False,
            num_runs=1,
            seed="constant",
        )

        assert result.success is False
        assert result.error is not None
        assert "Network failed" in result.error
        assert result.error_type == "ConnectionError"

    @pytest.mark.asyncio
    async def test_capture_mode_passed(self) -> None:
        """Test that capture_mode and include_context are passed."""
        mock_suite = AsyncMock(return_value=[])
        mock_suite.__name__ = "capture_suite"

        model_spec = ModelSpec(provider=Provider.OPENAI, model="gpt-4o", api_key="key")
        await _run_capture_task(
            suite_func=mock_suite,
            model_spec=model_spec,
            max_concurrent=2,
            include_context=True,
            num_runs=1,
            seed="constant",
        )

        mock_suite.assert_called_once_with(
            provider_api_key="key",
            model="gpt-4o",
            max_concurrency=2,
            provider="openai",
            capture_mode=True,
            include_context=True,
            num_runs=1,
            seed="constant",
        )


class TestRunEvaluationsErrorHandling:
    """Test run_evaluations handles partial failures."""

    @pytest.mark.asyncio
    async def test_partial_failure_continues(self) -> None:
        """Test that one failing task doesn't stop others."""
        successful_suite = AsyncMock(return_value=MagicMock())
        successful_suite.__name__ = "success_suite"

        failing_suite = AsyncMock(side_effect=RuntimeError("Oops"))
        failing_suite.__name__ = "failing_suite"

        console = MagicMock()
        model_specs = [ModelSpec(provider=Provider.OPENAI, model="gpt-4o", api_key="test")]

        with (
            patch("arcade_cli.evals_runner.display_eval_results"),
            patch("arcade_cli.evals_runner.Progress") as mock_progress,
        ):
            # Mock Progress context manager
            mock_progress.return_value.__enter__ = MagicMock(return_value=mock_progress)
            mock_progress.return_value.__exit__ = MagicMock(return_value=None)
            mock_progress.add_task = MagicMock(return_value=0)
            mock_progress.update = MagicMock()

            await run_evaluations(
                eval_suites=[successful_suite, failing_suite],
                model_specs=model_specs,
                max_concurrent=1,
                show_details=False,
                output_file=None,
                output_format="txt",
                failed_only=False,
                console=console,
                num_runs=1,
                seed="constant",
                multi_run_pass_rule=RUN_RULE_LAST,
            )

        # Verify both were attempted
        successful_suite.assert_called_once()
        failing_suite.assert_called_once()

    @pytest.mark.asyncio
    async def test_all_failures_reports_none_completed(self) -> None:
        """Test appropriate message when all tasks fail."""
        failing_suite = AsyncMock(side_effect=RuntimeError("Oops"))
        failing_suite.__name__ = "failing_suite"

        console = MagicMock()
        model_specs = [ModelSpec(provider=Provider.OPENAI, model="gpt-4o", api_key="test")]

        await run_evaluations(
            eval_suites=[failing_suite],
            model_specs=model_specs,
            max_concurrent=1,
            show_details=False,
            output_file=None,
            output_format="txt",
            failed_only=False,
            console=console,
            num_runs=1,
            seed="constant",
            multi_run_pass_rule=RUN_RULE_LAST,
        )

        # Should print "No evaluations completed successfully" (with emoji)
        console.print.assert_any_call(
            "\n[bold red]❌ No evaluations completed successfully.[/bold red]"
        )

    @pytest.mark.asyncio
    async def test_failure_warning_displayed(self) -> None:
        """Test that failure warnings are displayed."""
        failing_suite = AsyncMock(side_effect=ValueError("Bad input"))
        failing_suite.__name__ = "bad_suite"

        console = MagicMock()
        model_specs = [ModelSpec(provider=Provider.OPENAI, model="gpt-4o", api_key="test")]

        await run_evaluations(
            eval_suites=[failing_suite],
            model_specs=model_specs,
            max_concurrent=1,
            show_details=False,
            output_file=None,
            output_format="txt",
            failed_only=False,
            console=console,
            num_runs=1,
            seed="constant",
            multi_run_pass_rule=RUN_RULE_LAST,
        )

        # Check that failure count is printed
        calls = [str(c) for c in console.print.call_args_list]
        assert any("1 evaluation(s) failed" in c for c in calls)

    @pytest.mark.asyncio
    async def test_all_success_no_failure_warning(self) -> None:
        """Test that no failure warning when all succeed."""
        successful_suite = AsyncMock(return_value=MagicMock())
        successful_suite.__name__ = "success_suite"

        console = MagicMock()
        model_specs = [ModelSpec(provider=Provider.OPENAI, model="gpt-4o", api_key="test")]

        with patch("arcade_cli.evals_runner.display_eval_results"):
            await run_evaluations(
                eval_suites=[successful_suite],
                model_specs=model_specs,
                max_concurrent=1,
                show_details=False,
                output_file=None,
                output_format="txt",
                failed_only=False,
                console=console,
                num_runs=1,
                seed="constant",
                multi_run_pass_rule=RUN_RULE_LAST,
            )

        # Check that no failure warning is printed
        calls = [str(c) for c in console.print.call_args_list]
        assert not any("failed" in c.lower() for c in calls)

    @pytest.mark.asyncio
    async def test_multiple_models_partial_failure(self) -> None:
        """Test partial failure with multiple models."""

        # Suite that fails on one model but succeeds on another
        async def conditional_suite(**kwargs: Any) -> MagicMock:
            if kwargs["model"] == "bad-model":
                raise RuntimeError("Model not supported")
            return MagicMock()

        mock_suite = AsyncMock(side_effect=conditional_suite)
        mock_suite.__name__ = "conditional_suite"

        console = MagicMock()
        model_specs = [
            ModelSpec(provider=Provider.OPENAI, model="gpt-4o", api_key="test"),
            ModelSpec(provider=Provider.OPENAI, model="bad-model", api_key="test"),
        ]

        with (
            patch("arcade_cli.evals_runner.display_eval_results"),
            patch("arcade_cli.evals_runner.Progress") as mock_progress,
        ):
            # Mock Progress context manager
            mock_progress.return_value.__enter__ = MagicMock(return_value=mock_progress)
            mock_progress.return_value.__exit__ = MagicMock(return_value=None)
            mock_progress.add_task = MagicMock(return_value=0)
            mock_progress.update = MagicMock()

            await run_evaluations(
                eval_suites=[mock_suite],
                model_specs=model_specs,
                max_concurrent=1,
                show_details=False,
                output_file=None,
                output_format="txt",
                failed_only=False,
                console=console,
                num_runs=1,
                seed="constant",
                multi_run_pass_rule=RUN_RULE_LAST,
            )

        # Should have been called twice
        assert mock_suite.call_count == 2


class TestRunCaptureErrorHandling:
    """Test run_capture handles partial failures."""

    @pytest.mark.asyncio
    async def test_all_captures_fail_reports_none_completed(self) -> None:
        """Test appropriate message when all capture tasks fail."""
        failing_suite = AsyncMock(side_effect=RuntimeError("Capture failed"))
        failing_suite.__name__ = "failing_capture"

        console = MagicMock()
        model_specs = [ModelSpec(provider=Provider.OPENAI, model="gpt-4o", api_key="test")]

        await run_capture(
            eval_suites=[failing_suite],
            model_specs=model_specs,
            max_concurrent=1,
            include_context=False,
            output_file=None,
            output_format="json",
            console=console,
            num_runs=1,
            seed="constant",
        )

        # Error message includes emoji
        console.print.assert_any_call(
            "\n[bold red]❌ No captures completed successfully.[/bold red]"
        )

    @pytest.mark.asyncio
    async def test_partial_capture_failure_continues(self) -> None:
        """Test that one failing capture doesn't stop others."""
        # Mock CaptureResult
        mock_capture = MagicMock()
        mock_capture.to_dict.return_value = {"test": "data"}
        mock_capture.captured_cases = []

        successful_suite = AsyncMock(return_value=[mock_capture])
        successful_suite.__name__ = "success_capture"

        failing_suite = AsyncMock(side_effect=RuntimeError("Oops"))
        failing_suite.__name__ = "failing_capture"

        console = MagicMock()
        model_specs = [ModelSpec(provider=Provider.OPENAI, model="gpt-4o", api_key="test")]

        with patch("arcade_cli.evals_runner.Progress") as mock_progress:
            # Mock Progress context manager
            mock_progress.return_value.__enter__ = MagicMock(return_value=mock_progress)
            mock_progress.return_value.__exit__ = MagicMock(return_value=None)
            mock_progress.add_task = MagicMock(return_value=0)
            mock_progress.update = MagicMock()

            await run_capture(
                eval_suites=[successful_suite, failing_suite],
                model_specs=model_specs,
                max_concurrent=1,
                include_context=False,
                output_file=None,
                output_format="json",
                console=console,
                num_runs=1,
                seed="constant",
            )

        # Both should have been attempted
        successful_suite.assert_called_once()
        failing_suite.assert_called_once()

        # Check failure warning was printed
        calls = [str(c) for c in console.print.call_args_list]
        assert any("1 capture(s) failed" in c for c in calls)

    @pytest.mark.asyncio
    async def test_capture_failure_details_displayed(self) -> None:
        """Test that capture failure details are shown."""
        failing_suite = AsyncMock(side_effect=ConnectionError("Network error"))
        failing_suite.__name__ = "network_capture"

        console = MagicMock()
        model_specs = [ModelSpec(provider=Provider.OPENAI, model="gpt-4o", api_key="test")]

        await run_capture(
            eval_suites=[failing_suite],
            model_specs=model_specs,
            max_concurrent=1,
            include_context=False,
            output_file=None,
            output_format="json",
            console=console,
            num_runs=1,
            seed="constant",
        )

        # Check error details are printed
        calls = [str(c) for c in console.print.call_args_list]
        assert any("network_capture" in c for c in calls)
        assert any("ConnectionError" in c for c in calls)


class TestParseOutputFormats:
    """Tests for parse_output_formats function."""

    def test_single_format(self) -> None:
        """Should return a list with a single format."""
        console = MagicMock()
        assert parse_output_formats("md", console) == ["md"]
        assert parse_output_formats("txt", console) == ["txt"]
        assert parse_output_formats("html", console) == ["html"]
        assert parse_output_formats("json", console) == ["json"]

    def test_comma_separated_formats(self) -> None:
        """Should return a list of multiple formats."""
        console = MagicMock()
        assert parse_output_formats("md,html", console) == ["md", "html"]
        assert parse_output_formats("txt,md,html,json", console) == ["txt", "md", "html", "json"]

    def test_comma_separated_with_spaces(self) -> None:
        """Should handle spaces around commas."""
        console = MagicMock()
        assert parse_output_formats("md, html", console) == ["md", "html"]
        assert parse_output_formats(" md , html ", console) == ["md", "html"]

    def test_all_keyword(self) -> None:
        """Should return all formats for 'all' keyword."""
        console = MagicMock()
        assert parse_output_formats("all", console) == ALL_FORMATS
        assert parse_output_formats("ALL", console) == ALL_FORMATS
        assert parse_output_formats("All", console) == ALL_FORMATS

    def test_case_insensitive(self) -> None:
        """Should be case-insensitive."""
        console = MagicMock()
        assert parse_output_formats("MD", console) == ["md"]
        assert parse_output_formats("HTML,JSON", console) == ["html", "json"]

    def test_invalid_formats_raise_error(self) -> None:
        """Should raise ValueError for invalid formats (parse-time validation)."""
        console = MagicMock()

        with pytest.raises(ValueError, match="Invalid format.*invalid"):
            parse_output_formats("md,invalid", console)

        with pytest.raises(ValueError, match="Invalid format.*invalid"):
            parse_output_formats("invalid", console)

    def test_mixed_valid_invalid_raises(self) -> None:
        """Should raise ValueError when any invalid formats present."""
        with pytest.raises(ValueError, match="Invalid format"):
            parse_output_formats("md,foo,html,bar", MagicMock())

    def test_raises_on_invalid_formats(self) -> None:
        """Should raise ValueError when invalid formats are provided."""
        with pytest.raises(ValueError) as exc_info:
            parse_output_formats("xlsx,invalid", MagicMock())

        error_msg = str(exc_info.value)
        assert "Invalid format" in error_msg
        assert "xlsx" in error_msg
        assert "invalid" in error_msg

    def test_raises_on_partially_invalid_formats(self) -> None:
        """Should raise ValueError even when some valid formats exist."""
        with pytest.raises(ValueError) as exc_info:
            parse_output_formats("md,xlsx,html", MagicMock())

        error_msg = str(exc_info.value)
        assert "Invalid format" in error_msg
        assert "xlsx" in error_msg

    def test_no_error_when_all_valid(self) -> None:
        """Should not raise when all formats are valid."""
        console = MagicMock()
        result = parse_output_formats("md,html,json", console)
        assert result == ["md", "html", "json"]
