7472b18106
@EricGustin you can use this cli command:
```
uv run arcade evals mcp_building_evals_results/eval_toolkit_iteration_dict.py \
-p openai:gpt-4o,gpt-4o-mini \
-p anthropic:claude-sonnet-4-20250514 \
-k openai:$OPENAI_API_KEY \
-k anthropic:$ANTHROPIC_API_KEY \
-d \
--num-runs 3 \
--seed random \
--multi-run-pass-rule majority \
--max-concurrent 6 \
-o mcp_building_evals_results/results
```
<!-- CURSOR_SUMMARY -->
---
> [!NOTE]
> **Medium Risk**
> Touches core eval execution and all result formatters while adding new
CLI inputs and output schema (`run_stats`/`critic_stats` and capture
`runs`), so regressions could affect evaluation results and report
compatibility despite being additive and validated.
>
> **Overview**
> Adds **multi-run evaluation support** to `arcade evals` via new flags
`--num-runs`, `--seed`, and `--multi-run-pass-rule`, with upfront
validation and plumbing through the CLI runner into eval/capture suite
execution.
>
> Fixes provider selection UX/bug by making `--use-provider/-p`
**repeatable** (instead of a space-delimited string), updates
docs/examples accordingly, and extends capture mode to optionally record
**per-run tool calls** (`CapturedRun`) when `num_runs > 1`.
>
> Enhances all output formatters (HTML/Markdown/Text/JSON) to
**propagate and display** per-case `run_stats` and `critic_stats`,
including new HTML UI for run tabs/cards and comparative tables showing
mean ± stddev when multi-run data is present.
>
> <sup>Written by [Cursor
Bugbot](https://cursor.com/dashboard?tab=bugbot) for commit
2ee1654b7d1fbb9538373507355636164b16a066. This will update automatically
on new commits. Configure
[here](https://cursor.com/dashboard?tab=bugbot).</sup>
<!-- /CURSOR_SUMMARY -->
601 lines
22 KiB
Python
601 lines
22 KiB
Python
"""Tests for evals_runner error handling."""
|
|
|
|
from typing import Any, cast
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
import pytest
|
|
from arcade_cli.evals_runner import (
|
|
ALL_FORMATS,
|
|
CaptureTaskResult,
|
|
EvalTaskResult,
|
|
_run_capture_task,
|
|
_run_eval_task,
|
|
parse_output_formats,
|
|
run_capture,
|
|
run_evaluations,
|
|
)
|
|
from arcade_cli.utils import ModelSpec, Provider
|
|
from arcade_evals import CaptureResult
|
|
|
|
RUN_RULE_LAST = "last"
|
|
|
|
|
|
class TestEvalTaskResult:
|
|
"""Test EvalTaskResult dataclass."""
|
|
|
|
def test_from_success(self) -> None:
|
|
"""Test creating a successful result."""
|
|
result = EvalTaskResult.from_success("test_suite", "gpt-4o", "openai", {"score": 0.9})
|
|
assert result.success is True
|
|
assert result.suite_name == "test_suite"
|
|
assert result.model == "gpt-4o"
|
|
assert result.provider == "openai"
|
|
assert result.result == {"score": 0.9}
|
|
assert result.error is None
|
|
assert result.error_type is None
|
|
|
|
def test_from_error(self) -> None:
|
|
"""Test creating a failed result from an exception."""
|
|
error = ValueError("Something went wrong")
|
|
result = EvalTaskResult.from_error("test_suite", "gpt-4o", "openai", error)
|
|
assert result.success is False
|
|
assert result.suite_name == "test_suite"
|
|
assert result.model == "gpt-4o"
|
|
assert result.provider == "openai"
|
|
assert result.error == "Something went wrong"
|
|
assert result.error_type == "ValueError"
|
|
assert result.result is None
|
|
|
|
def test_from_error_with_different_exception_types(self) -> None:
|
|
"""Test that error_type captures the correct exception class name."""
|
|
errors = [
|
|
(RuntimeError("runtime"), "RuntimeError"),
|
|
(TypeError("type"), "TypeError"),
|
|
(KeyError("key"), "KeyError"),
|
|
(ConnectionError("conn"), "ConnectionError"),
|
|
]
|
|
for error, expected_type in errors:
|
|
result = EvalTaskResult.from_error("suite", "model", "openai", error)
|
|
assert result.error_type == expected_type
|
|
|
|
def test_display_name(self) -> None:
|
|
"""Test that display_name shows provider/model format."""
|
|
result = EvalTaskResult.from_success("suite", "gpt-4o", "openai", {})
|
|
assert result.display_name == "openai/gpt-4o"
|
|
|
|
result2 = EvalTaskResult.from_success("suite", "claude-3-sonnet", "anthropic", {})
|
|
assert result2.display_name == "anthropic/claude-3-sonnet"
|
|
|
|
|
|
class TestCaptureTaskResult:
|
|
"""Test CaptureTaskResult dataclass."""
|
|
|
|
def test_from_success(self) -> None:
|
|
"""Test creating a successful capture result."""
|
|
mock_captures = cast(
|
|
list[CaptureResult],
|
|
[
|
|
MagicMock(spec=CaptureResult),
|
|
MagicMock(spec=CaptureResult),
|
|
],
|
|
)
|
|
result = CaptureTaskResult.from_success("test_suite", "gpt-4o", "openai", mock_captures)
|
|
assert result.success is True
|
|
assert result.suite_name == "test_suite"
|
|
assert result.model == "gpt-4o"
|
|
assert result.provider == "openai"
|
|
assert result.result == mock_captures
|
|
assert result.error is None
|
|
assert result.error_type is None
|
|
|
|
def test_from_error(self) -> None:
|
|
"""Test creating a failed capture result."""
|
|
error = RuntimeError("Capture failed")
|
|
result = CaptureTaskResult.from_error("test_suite", "gpt-4o", "openai", error)
|
|
assert result.success is False
|
|
assert result.error == "Capture failed"
|
|
assert result.error_type == "RuntimeError"
|
|
assert result.result is None
|
|
|
|
def test_display_name(self) -> None:
|
|
"""Test that display_name shows provider/model format."""
|
|
result = CaptureTaskResult.from_success("suite", "gpt-4o", "openai", [])
|
|
assert result.display_name == "openai/gpt-4o"
|
|
|
|
|
|
class TestRunEvalTask:
|
|
"""Test _run_eval_task error handling."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_successful_task(self) -> None:
|
|
"""Test that successful task returns success result."""
|
|
mock_suite = AsyncMock(return_value={"score": 0.95})
|
|
mock_suite.__name__ = "test_suite"
|
|
|
|
model_spec = ModelSpec(provider=Provider.OPENAI, model="gpt-4o", api_key="test-key")
|
|
result = await _run_eval_task(
|
|
suite_func=mock_suite,
|
|
model_spec=model_spec,
|
|
max_concurrent=1,
|
|
num_runs=1,
|
|
seed="constant",
|
|
multi_run_pass_rule=RUN_RULE_LAST,
|
|
)
|
|
|
|
assert result.success is True
|
|
assert result.result == {"score": 0.95}
|
|
assert result.suite_name == "test_suite"
|
|
assert result.model == "gpt-4o"
|
|
assert result.provider == "openai"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_failed_task_returns_error_result(self) -> None:
|
|
"""Test that failed task returns error result instead of raising."""
|
|
mock_suite = AsyncMock(side_effect=ValueError("API error"))
|
|
mock_suite.__name__ = "test_suite"
|
|
|
|
model_spec = ModelSpec(provider=Provider.OPENAI, model="gpt-4o", api_key="test-key")
|
|
result = await _run_eval_task(
|
|
suite_func=mock_suite,
|
|
model_spec=model_spec,
|
|
max_concurrent=1,
|
|
num_runs=1,
|
|
seed="constant",
|
|
multi_run_pass_rule=RUN_RULE_LAST,
|
|
)
|
|
|
|
assert result.success is False
|
|
assert result.error is not None
|
|
assert "API error" in result.error
|
|
assert result.error_type == "ValueError"
|
|
assert result.result is None
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_passes_correct_arguments_to_suite(self) -> None:
|
|
"""Test that correct arguments are passed to the suite function."""
|
|
mock_suite = AsyncMock(return_value={"score": 1.0})
|
|
mock_suite.__name__ = "test_suite"
|
|
|
|
model_spec = ModelSpec(provider=Provider.ANTHROPIC, model="claude-sonnet", api_key="my-key")
|
|
await _run_eval_task(
|
|
suite_func=mock_suite,
|
|
model_spec=model_spec,
|
|
max_concurrent=5,
|
|
include_context=False,
|
|
num_runs=1,
|
|
seed="constant",
|
|
multi_run_pass_rule=RUN_RULE_LAST,
|
|
)
|
|
|
|
mock_suite.assert_called_once_with(
|
|
provider_api_key="my-key",
|
|
model="claude-sonnet",
|
|
max_concurrency=5,
|
|
provider="anthropic",
|
|
include_context=False,
|
|
num_runs=1,
|
|
seed="constant",
|
|
multi_run_pass_rule=RUN_RULE_LAST,
|
|
)
|
|
|
|
|
|
class TestRunCaptureTask:
|
|
"""Test _run_capture_task error handling."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_successful_capture_task(self) -> None:
|
|
"""Test that successful capture task returns success result."""
|
|
mock_captures = [MagicMock()]
|
|
mock_suite = AsyncMock(return_value=mock_captures)
|
|
mock_suite.__name__ = "capture_suite"
|
|
|
|
model_spec = ModelSpec(provider=Provider.OPENAI, model="gpt-4o", api_key="test-key")
|
|
result = await _run_capture_task(
|
|
suite_func=mock_suite,
|
|
model_spec=model_spec,
|
|
max_concurrent=1,
|
|
include_context=True,
|
|
num_runs=1,
|
|
seed="constant",
|
|
)
|
|
|
|
assert result.success is True
|
|
assert result.result == mock_captures
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_failed_capture_task_returns_error_result(self) -> None:
|
|
"""Test that failed capture task returns error result."""
|
|
mock_suite = AsyncMock(side_effect=ConnectionError("Network failed"))
|
|
mock_suite.__name__ = "capture_suite"
|
|
|
|
model_spec = ModelSpec(provider=Provider.OPENAI, model="gpt-4o", api_key="test-key")
|
|
result = await _run_capture_task(
|
|
suite_func=mock_suite,
|
|
model_spec=model_spec,
|
|
max_concurrent=1,
|
|
include_context=False,
|
|
num_runs=1,
|
|
seed="constant",
|
|
)
|
|
|
|
assert result.success is False
|
|
assert result.error is not None
|
|
assert "Network failed" in result.error
|
|
assert result.error_type == "ConnectionError"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_capture_mode_passed(self) -> None:
|
|
"""Test that capture_mode and include_context are passed."""
|
|
mock_suite = AsyncMock(return_value=[])
|
|
mock_suite.__name__ = "capture_suite"
|
|
|
|
model_spec = ModelSpec(provider=Provider.OPENAI, model="gpt-4o", api_key="key")
|
|
await _run_capture_task(
|
|
suite_func=mock_suite,
|
|
model_spec=model_spec,
|
|
max_concurrent=2,
|
|
include_context=True,
|
|
num_runs=1,
|
|
seed="constant",
|
|
)
|
|
|
|
mock_suite.assert_called_once_with(
|
|
provider_api_key="key",
|
|
model="gpt-4o",
|
|
max_concurrency=2,
|
|
provider="openai",
|
|
capture_mode=True,
|
|
include_context=True,
|
|
num_runs=1,
|
|
seed="constant",
|
|
)
|
|
|
|
|
|
class TestRunEvaluationsErrorHandling:
|
|
"""Test run_evaluations handles partial failures."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_partial_failure_continues(self) -> None:
|
|
"""Test that one failing task doesn't stop others."""
|
|
successful_suite = AsyncMock(return_value=MagicMock())
|
|
successful_suite.__name__ = "success_suite"
|
|
|
|
failing_suite = AsyncMock(side_effect=RuntimeError("Oops"))
|
|
failing_suite.__name__ = "failing_suite"
|
|
|
|
console = MagicMock()
|
|
model_specs = [ModelSpec(provider=Provider.OPENAI, model="gpt-4o", api_key="test")]
|
|
|
|
with (
|
|
patch("arcade_cli.evals_runner.display_eval_results"),
|
|
patch("arcade_cli.evals_runner.Progress") as mock_progress,
|
|
):
|
|
# Mock Progress context manager
|
|
mock_progress.return_value.__enter__ = MagicMock(return_value=mock_progress)
|
|
mock_progress.return_value.__exit__ = MagicMock(return_value=None)
|
|
mock_progress.add_task = MagicMock(return_value=0)
|
|
mock_progress.update = MagicMock()
|
|
|
|
await run_evaluations(
|
|
eval_suites=[successful_suite, failing_suite],
|
|
model_specs=model_specs,
|
|
max_concurrent=1,
|
|
show_details=False,
|
|
output_file=None,
|
|
output_format="txt",
|
|
failed_only=False,
|
|
console=console,
|
|
num_runs=1,
|
|
seed="constant",
|
|
multi_run_pass_rule=RUN_RULE_LAST,
|
|
)
|
|
|
|
# Verify both were attempted
|
|
successful_suite.assert_called_once()
|
|
failing_suite.assert_called_once()
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_all_failures_reports_none_completed(self) -> None:
|
|
"""Test appropriate message when all tasks fail."""
|
|
failing_suite = AsyncMock(side_effect=RuntimeError("Oops"))
|
|
failing_suite.__name__ = "failing_suite"
|
|
|
|
console = MagicMock()
|
|
model_specs = [ModelSpec(provider=Provider.OPENAI, model="gpt-4o", api_key="test")]
|
|
|
|
await run_evaluations(
|
|
eval_suites=[failing_suite],
|
|
model_specs=model_specs,
|
|
max_concurrent=1,
|
|
show_details=False,
|
|
output_file=None,
|
|
output_format="txt",
|
|
failed_only=False,
|
|
console=console,
|
|
num_runs=1,
|
|
seed="constant",
|
|
multi_run_pass_rule=RUN_RULE_LAST,
|
|
)
|
|
|
|
# Should print "No evaluations completed successfully" (with emoji)
|
|
console.print.assert_any_call(
|
|
"\n[bold red]❌ No evaluations completed successfully.[/bold red]"
|
|
)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_failure_warning_displayed(self) -> None:
|
|
"""Test that failure warnings are displayed."""
|
|
failing_suite = AsyncMock(side_effect=ValueError("Bad input"))
|
|
failing_suite.__name__ = "bad_suite"
|
|
|
|
console = MagicMock()
|
|
model_specs = [ModelSpec(provider=Provider.OPENAI, model="gpt-4o", api_key="test")]
|
|
|
|
await run_evaluations(
|
|
eval_suites=[failing_suite],
|
|
model_specs=model_specs,
|
|
max_concurrent=1,
|
|
show_details=False,
|
|
output_file=None,
|
|
output_format="txt",
|
|
failed_only=False,
|
|
console=console,
|
|
num_runs=1,
|
|
seed="constant",
|
|
multi_run_pass_rule=RUN_RULE_LAST,
|
|
)
|
|
|
|
# Check that failure count is printed
|
|
calls = [str(c) for c in console.print.call_args_list]
|
|
assert any("1 evaluation(s) failed" in c for c in calls)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_all_success_no_failure_warning(self) -> None:
|
|
"""Test that no failure warning when all succeed."""
|
|
successful_suite = AsyncMock(return_value=MagicMock())
|
|
successful_suite.__name__ = "success_suite"
|
|
|
|
console = MagicMock()
|
|
model_specs = [ModelSpec(provider=Provider.OPENAI, model="gpt-4o", api_key="test")]
|
|
|
|
with patch("arcade_cli.evals_runner.display_eval_results"):
|
|
await run_evaluations(
|
|
eval_suites=[successful_suite],
|
|
model_specs=model_specs,
|
|
max_concurrent=1,
|
|
show_details=False,
|
|
output_file=None,
|
|
output_format="txt",
|
|
failed_only=False,
|
|
console=console,
|
|
num_runs=1,
|
|
seed="constant",
|
|
multi_run_pass_rule=RUN_RULE_LAST,
|
|
)
|
|
|
|
# Check that no failure warning is printed
|
|
calls = [str(c) for c in console.print.call_args_list]
|
|
assert not any("failed" in c.lower() for c in calls)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_multiple_models_partial_failure(self) -> None:
|
|
"""Test partial failure with multiple models."""
|
|
|
|
# Suite that fails on one model but succeeds on another
|
|
async def conditional_suite(**kwargs: Any) -> MagicMock:
|
|
if kwargs["model"] == "bad-model":
|
|
raise RuntimeError("Model not supported")
|
|
return MagicMock()
|
|
|
|
mock_suite = AsyncMock(side_effect=conditional_suite)
|
|
mock_suite.__name__ = "conditional_suite"
|
|
|
|
console = MagicMock()
|
|
model_specs = [
|
|
ModelSpec(provider=Provider.OPENAI, model="gpt-4o", api_key="test"),
|
|
ModelSpec(provider=Provider.OPENAI, model="bad-model", api_key="test"),
|
|
]
|
|
|
|
with (
|
|
patch("arcade_cli.evals_runner.display_eval_results"),
|
|
patch("arcade_cli.evals_runner.Progress") as mock_progress,
|
|
):
|
|
# Mock Progress context manager
|
|
mock_progress.return_value.__enter__ = MagicMock(return_value=mock_progress)
|
|
mock_progress.return_value.__exit__ = MagicMock(return_value=None)
|
|
mock_progress.add_task = MagicMock(return_value=0)
|
|
mock_progress.update = MagicMock()
|
|
|
|
await run_evaluations(
|
|
eval_suites=[mock_suite],
|
|
model_specs=model_specs,
|
|
max_concurrent=1,
|
|
show_details=False,
|
|
output_file=None,
|
|
output_format="txt",
|
|
failed_only=False,
|
|
console=console,
|
|
num_runs=1,
|
|
seed="constant",
|
|
multi_run_pass_rule=RUN_RULE_LAST,
|
|
)
|
|
|
|
# Should have been called twice
|
|
assert mock_suite.call_count == 2
|
|
|
|
|
|
class TestRunCaptureErrorHandling:
|
|
"""Test run_capture handles partial failures."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_all_captures_fail_reports_none_completed(self) -> None:
|
|
"""Test appropriate message when all capture tasks fail."""
|
|
failing_suite = AsyncMock(side_effect=RuntimeError("Capture failed"))
|
|
failing_suite.__name__ = "failing_capture"
|
|
|
|
console = MagicMock()
|
|
model_specs = [ModelSpec(provider=Provider.OPENAI, model="gpt-4o", api_key="test")]
|
|
|
|
await run_capture(
|
|
eval_suites=[failing_suite],
|
|
model_specs=model_specs,
|
|
max_concurrent=1,
|
|
include_context=False,
|
|
output_file=None,
|
|
output_format="json",
|
|
console=console,
|
|
num_runs=1,
|
|
seed="constant",
|
|
)
|
|
|
|
# Error message includes emoji
|
|
console.print.assert_any_call(
|
|
"\n[bold red]❌ No captures completed successfully.[/bold red]"
|
|
)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_partial_capture_failure_continues(self) -> None:
|
|
"""Test that one failing capture doesn't stop others."""
|
|
# Mock CaptureResult
|
|
mock_capture = MagicMock()
|
|
mock_capture.to_dict.return_value = {"test": "data"}
|
|
mock_capture.captured_cases = []
|
|
|
|
successful_suite = AsyncMock(return_value=[mock_capture])
|
|
successful_suite.__name__ = "success_capture"
|
|
|
|
failing_suite = AsyncMock(side_effect=RuntimeError("Oops"))
|
|
failing_suite.__name__ = "failing_capture"
|
|
|
|
console = MagicMock()
|
|
model_specs = [ModelSpec(provider=Provider.OPENAI, model="gpt-4o", api_key="test")]
|
|
|
|
with patch("arcade_cli.evals_runner.Progress") as mock_progress:
|
|
# Mock Progress context manager
|
|
mock_progress.return_value.__enter__ = MagicMock(return_value=mock_progress)
|
|
mock_progress.return_value.__exit__ = MagicMock(return_value=None)
|
|
mock_progress.add_task = MagicMock(return_value=0)
|
|
mock_progress.update = MagicMock()
|
|
|
|
await run_capture(
|
|
eval_suites=[successful_suite, failing_suite],
|
|
model_specs=model_specs,
|
|
max_concurrent=1,
|
|
include_context=False,
|
|
output_file=None,
|
|
output_format="json",
|
|
console=console,
|
|
num_runs=1,
|
|
seed="constant",
|
|
)
|
|
|
|
# Both should have been attempted
|
|
successful_suite.assert_called_once()
|
|
failing_suite.assert_called_once()
|
|
|
|
# Check failure warning was printed
|
|
calls = [str(c) for c in console.print.call_args_list]
|
|
assert any("1 capture(s) failed" in c for c in calls)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_capture_failure_details_displayed(self) -> None:
|
|
"""Test that capture failure details are shown."""
|
|
failing_suite = AsyncMock(side_effect=ConnectionError("Network error"))
|
|
failing_suite.__name__ = "network_capture"
|
|
|
|
console = MagicMock()
|
|
model_specs = [ModelSpec(provider=Provider.OPENAI, model="gpt-4o", api_key="test")]
|
|
|
|
await run_capture(
|
|
eval_suites=[failing_suite],
|
|
model_specs=model_specs,
|
|
max_concurrent=1,
|
|
include_context=False,
|
|
output_file=None,
|
|
output_format="json",
|
|
console=console,
|
|
num_runs=1,
|
|
seed="constant",
|
|
)
|
|
|
|
# Check error details are printed
|
|
calls = [str(c) for c in console.print.call_args_list]
|
|
assert any("network_capture" in c for c in calls)
|
|
assert any("ConnectionError" in c for c in calls)
|
|
|
|
|
|
class TestParseOutputFormats:
|
|
"""Tests for parse_output_formats function."""
|
|
|
|
def test_single_format(self) -> None:
|
|
"""Should return a list with a single format."""
|
|
console = MagicMock()
|
|
assert parse_output_formats("md", console) == ["md"]
|
|
assert parse_output_formats("txt", console) == ["txt"]
|
|
assert parse_output_formats("html", console) == ["html"]
|
|
assert parse_output_formats("json", console) == ["json"]
|
|
|
|
def test_comma_separated_formats(self) -> None:
|
|
"""Should return a list of multiple formats."""
|
|
console = MagicMock()
|
|
assert parse_output_formats("md,html", console) == ["md", "html"]
|
|
assert parse_output_formats("txt,md,html,json", console) == ["txt", "md", "html", "json"]
|
|
|
|
def test_comma_separated_with_spaces(self) -> None:
|
|
"""Should handle spaces around commas."""
|
|
console = MagicMock()
|
|
assert parse_output_formats("md, html", console) == ["md", "html"]
|
|
assert parse_output_formats(" md , html ", console) == ["md", "html"]
|
|
|
|
def test_all_keyword(self) -> None:
|
|
"""Should return all formats for 'all' keyword."""
|
|
console = MagicMock()
|
|
assert parse_output_formats("all", console) == ALL_FORMATS
|
|
assert parse_output_formats("ALL", console) == ALL_FORMATS
|
|
assert parse_output_formats("All", console) == ALL_FORMATS
|
|
|
|
def test_case_insensitive(self) -> None:
|
|
"""Should be case-insensitive."""
|
|
console = MagicMock()
|
|
assert parse_output_formats("MD", console) == ["md"]
|
|
assert parse_output_formats("HTML,JSON", console) == ["html", "json"]
|
|
|
|
def test_invalid_formats_raise_error(self) -> None:
|
|
"""Should raise ValueError for invalid formats (parse-time validation)."""
|
|
console = MagicMock()
|
|
|
|
with pytest.raises(ValueError, match="Invalid format.*invalid"):
|
|
parse_output_formats("md,invalid", console)
|
|
|
|
with pytest.raises(ValueError, match="Invalid format.*invalid"):
|
|
parse_output_formats("invalid", console)
|
|
|
|
def test_mixed_valid_invalid_raises(self) -> None:
|
|
"""Should raise ValueError when any invalid formats present."""
|
|
with pytest.raises(ValueError, match="Invalid format"):
|
|
parse_output_formats("md,foo,html,bar", MagicMock())
|
|
|
|
def test_raises_on_invalid_formats(self) -> None:
|
|
"""Should raise ValueError when invalid formats are provided."""
|
|
with pytest.raises(ValueError) as exc_info:
|
|
parse_output_formats("xlsx,invalid", MagicMock())
|
|
|
|
error_msg = str(exc_info.value)
|
|
assert "Invalid format" in error_msg
|
|
assert "xlsx" in error_msg
|
|
assert "invalid" in error_msg
|
|
|
|
def test_raises_on_partially_invalid_formats(self) -> None:
|
|
"""Should raise ValueError even when some valid formats exist."""
|
|
with pytest.raises(ValueError) as exc_info:
|
|
parse_output_formats("md,xlsx,html", MagicMock())
|
|
|
|
error_msg = str(exc_info.value)
|
|
assert "Invalid format" in error_msg
|
|
assert "xlsx" in error_msg
|
|
|
|
def test_no_error_when_all_valid(self) -> None:
|
|
"""Should not raise when all formats are valid."""
|
|
console = MagicMock()
|
|
result = parse_output_formats("md,html,json", console)
|
|
assert result == ["md", "html", "json"]
|