arcade-mcp/libs/tests/arcade_evals/test_convenience_async.py

"""Tests for async MCP convenience methods in EvalSuite."""

from unittest.mock import patch

import pytest
from arcade_evals import EvalSuite

# Mark all tests in this module as requiring evals dependencies
pytestmark = pytest.mark.evals


class TestAddMcpServer:
    """Tests for add_mcp_server async convenience method."""

    @pytest.mark.asyncio
    async def test_add_mcp_server_loads_and_registers_tools(self) -> None:
        """Test that add_mcp_server loads tools and adds them to registry."""
        suite = EvalSuite(name="test", system_message="test")

        mock_tools = [
            {"name": "tool1", "description": "Test tool 1", "inputSchema": {}},
            {"name": "tool2", "description": "Test tool 2", "inputSchema": {}},
        ]

        with patch("arcade_evals._evalsuite._convenience.load_mcp_remote_async") as mock_load:
            mock_load.return_value = mock_tools

            result = await suite.add_mcp_server(
                "http://localhost:8000",
                headers={"Authorization": "Bearer token"},
                timeout=15,
                use_sse=True,
            )

            # Verify loader was called with correct args
            mock_load.assert_called_once_with(
                "http://localhost:8000",
                timeout=15,
                headers={"Authorization": "Bearer token"},
                use_sse=True,
            )

            # Verify tools were registered
            tools = suite._internal_registry.list_tools_for_model("openai")
            assert len(tools) == 2

            # Verify returns self for chaining
            assert result is suite

    @pytest.mark.asyncio
    async def test_add_mcp_server_with_track(self) -> None:
        """Test add_mcp_server with track parameter."""
        suite = EvalSuite(name="test", system_message="test")

        mock_tools = [{"name": "tool1", "description": "Test", "inputSchema": {}}]

        with patch("arcade_evals._evalsuite._convenience.load_mcp_remote_async") as mock_load:
            mock_load.return_value = mock_tools

            await suite.add_mcp_server("http://localhost:8000", track="github")

            # Verify track was created
            assert suite._track_manager.has_track("github")

            # Verify tool is in track registry
            track_registry = suite._track_manager.get_registry("github")
            assert track_registry is not None
            track_tools = track_registry.list_tools_for_model("openai")
            assert len(track_tools) == 1
            assert track_tools[0]["function"]["name"] == "tool1"

    @pytest.mark.asyncio
    async def test_add_mcp_server_warns_on_empty_tools(self) -> None:
        """Test that add_mcp_server warns when no tools are loaded."""
        suite = EvalSuite(name="test", system_message="test")

        with patch("arcade_evals._evalsuite._convenience.load_mcp_remote_async") as mock_load:
            mock_load.return_value = []  # Empty tools

            with pytest.warns(UserWarning, match="No tools loaded from"):
                await suite.add_mcp_server("http://localhost:8000")

    @pytest.mark.asyncio
    async def test_add_mcp_server_handles_loader_exception(self) -> None:
        """Test that add_mcp_server propagates loader exceptions."""
        suite = EvalSuite(name="test", system_message="test")

        with patch("arcade_evals._evalsuite._convenience.load_mcp_remote_async") as mock_load:
            mock_load.side_effect = TimeoutError("Connection timeout")

            with pytest.raises(TimeoutError, match="Connection timeout"):
                await suite.add_mcp_server("http://localhost:8000")


class TestAddMcpStdioServer:
    """Tests for add_mcp_stdio_server async convenience method."""

    @pytest.mark.asyncio
    async def test_add_mcp_stdio_server_loads_and_registers_tools(self) -> None:
        """Test that add_mcp_stdio_server loads tools and adds them to registry."""
        suite = EvalSuite(name="test", system_message="test")

        mock_tools = [
            {"name": "linear_search", "description": "Search", "inputSchema": {}},
            {"name": "linear_create", "description": "Create", "inputSchema": {}},
        ]

        with patch("arcade_evals._evalsuite._convenience.load_from_stdio_async") as mock_load:
            mock_load.return_value = mock_tools

            command = ["python", "-m", "arcade_mcp_server", "stdio"]
            env = {"ARCADE_API_KEY": "test_key"}

            result = await suite.add_mcp_stdio_server(command, env=env, timeout=20)

            # Verify loader was called with correct args
            mock_load.assert_called_once_with(command, timeout=20, env=env)

            # Verify tools were registered
            tools = suite._internal_registry.list_tools_for_model("openai")
            assert len(tools) == 2

            # Verify returns self for chaining
            assert result is suite

    @pytest.mark.asyncio
    async def test_add_mcp_stdio_server_with_track(self) -> None:
        """Test add_mcp_stdio_server with track parameter."""
        suite = EvalSuite(name="test", system_message="test")

        mock_tools = [{"name": "tool1", "description": "Test", "inputSchema": {}}]

        with patch("arcade_evals._evalsuite._convenience.load_from_stdio_async") as mock_load:
            mock_load.return_value = mock_tools

            await suite.add_mcp_stdio_server(["python", "server.py"], track="linear")

            # Verify track was created
            assert suite._track_manager.has_track("linear")

    @pytest.mark.asyncio
    async def test_add_mcp_stdio_server_warns_on_empty_tools(self) -> None:
        """Test that add_mcp_stdio_server warns when no tools are loaded."""
        suite = EvalSuite(name="test", system_message="test")

        with patch("arcade_evals._evalsuite._convenience.load_from_stdio_async") as mock_load:
            mock_load.return_value = []

            with pytest.warns(UserWarning, match="No tools loaded from stdio"):
                await suite.add_mcp_stdio_server(["python", "server.py"])

    @pytest.mark.asyncio
    async def test_add_mcp_stdio_server_handles_loader_exception(self) -> None:
        """Test that add_mcp_stdio_server propagates loader exceptions."""
        suite = EvalSuite(name="test", system_message="test")

        with patch("arcade_evals._evalsuite._convenience.load_from_stdio_async") as mock_load:
            mock_load.side_effect = TimeoutError("Stdio timeout")

            with pytest.raises(TimeoutError, match="Stdio timeout"):
                await suite.add_mcp_stdio_server(["python", "server.py"])


class TestAddArcadeGateway:
    """Tests for add_arcade_gateway async convenience method."""

    @pytest.mark.asyncio
    async def test_add_arcade_gateway_loads_and_registers_tools(self) -> None:
        """Test that add_arcade_gateway loads tools and adds them to registry."""
        suite = EvalSuite(name="test", system_message="test")

        mock_tools = [
            {"name": "Github_CreateIssue", "description": "Create issue", "inputSchema": {}},
            {"name": "Github_GetIssue", "description": "Get issue", "inputSchema": {}},
        ]

        with patch(
            "arcade_evals._evalsuite._convenience.load_arcade_mcp_gateway_async"
        ) as mock_load:
            mock_load.return_value = mock_tools

            result = await suite.add_arcade_gateway(
                "my-gateway",
                arcade_api_key="test_key",
                arcade_user_id="test@example.com",
                base_url="https://api.arcade.dev",
                timeout=10,
            )

            # Verify loader was called with correct args
            mock_load.assert_called_once_with(
                "my-gateway",
                arcade_api_key="test_key",
                arcade_user_id="test@example.com",
                base_url="https://api.arcade.dev",
                timeout=10,
            )

            # Verify tools were registered
            tools = suite._internal_registry.list_tools_for_model("openai")
            assert len(tools) == 2

            # Verify returns self for chaining
            assert result is suite

    @pytest.mark.asyncio
    async def test_add_arcade_gateway_with_track(self) -> None:
        """Test add_arcade_gateway with track parameter."""
        suite = EvalSuite(name="test", system_message="test")

        mock_tools = [{"name": "tool1", "description": "Test", "inputSchema": {}}]

        with patch(
            "arcade_evals._evalsuite._convenience.load_arcade_mcp_gateway_async"
        ) as mock_load:
            mock_load.return_value = mock_tools

            await suite.add_arcade_gateway("my-gateway", track="arcade")

            # Verify track was created
            assert suite._track_manager.has_track("arcade")

    @pytest.mark.asyncio
    async def test_add_arcade_gateway_warns_on_empty_tools(self) -> None:
        """Test that add_arcade_gateway warns when no tools are loaded."""
        suite = EvalSuite(name="test", system_message="test")

        with patch(
            "arcade_evals._evalsuite._convenience.load_arcade_mcp_gateway_async"
        ) as mock_load:
            mock_load.return_value = []

            with pytest.warns(UserWarning, match="No tools loaded from Arcade gateway"):
                await suite.add_arcade_gateway("my-gateway")

    @pytest.mark.asyncio
    async def test_add_arcade_gateway_handles_loader_exception(self) -> None:
        """Test that add_arcade_gateway propagates loader exceptions."""
        suite = EvalSuite(name="test", system_message="test")

        with patch(
            "arcade_evals._evalsuite._convenience.load_arcade_mcp_gateway_async"
        ) as mock_load:
            mock_load.side_effect = Exception("Gateway connection failed")

            with pytest.raises(Exception, match="Gateway connection failed"):
                await suite.add_arcade_gateway("my-gateway")


class TestAsyncConvenienceMethodChaining:
    """Tests for method chaining with async MCP methods."""

    @pytest.mark.asyncio
    async def test_chaining_multiple_mcp_sources(self) -> None:
        """Test that async methods can be chained together."""
        suite = EvalSuite(name="test", system_message="test")

        mock_http_tools = [{"name": "http_tool", "description": "HTTP", "inputSchema": {}}]
        mock_stdio_tools = [{"name": "stdio_tool", "description": "Stdio", "inputSchema": {}}]
        mock_gateway_tools = [
            {"name": "gateway_tool", "description": "Gateway", "inputSchema": {}}
        ]

        with (
            patch(
                "arcade_evals._evalsuite._convenience.load_mcp_remote_async"
            ) as mock_http,
            patch(
                "arcade_evals._evalsuite._convenience.load_from_stdio_async"
            ) as mock_stdio,
            patch(
                "arcade_evals._evalsuite._convenience.load_arcade_mcp_gateway_async"
            ) as mock_gateway,
        ):
            mock_http.return_value = mock_http_tools
            mock_stdio.return_value = mock_stdio_tools
            mock_gateway.return_value = mock_gateway_tools

            # Chain all three methods
            result = await suite.add_mcp_server("http://localhost:8000")
            result = await result.add_mcp_stdio_server(["python", "server.py"])
            result = await result.add_arcade_gateway("my-gateway")

            # Verify all tools were registered
            tools = suite._internal_registry.list_tools_for_model("openai")
            assert len(tools) == 3
            tool_names = [t["function"]["name"] for t in tools]
            assert "http_tool" in tool_names
            assert "stdio_tool" in tool_names
            assert "gateway_tool" in tool_names

            # Verify final result is still the suite
            assert result is suite