# MCP Server Tool Evaluation Support
## Overview
Add support for evaluating tools from remote MCP servers without
requiring Python callables. Enables direct evaluation of any
MCP-compatible tool server.
## What's New
### Core Features
- **`MCPToolRegistry`**: Evaluate tools from a single MCP server
- **`CompositeMCPRegistry`**: Evaluate tools from multiple MCP servers
simultaneously
- **Automatic loaders**: `load_from_stdio()` and `load_from_http()` to
fetch tools from running servers
- **Automatic namespacing**: Tools prefixed with server name (e.g.,
`server_tool_name`)
- **Smart name resolution**: Use short names if unique, full names if
ambiguous
- **OpenAI strict mode**: Automatic schema conversion prevents parameter
hallucinations
### Usage
**Automatic Loading:**
```python
from arcade_evals import load_from_stdio, MCPToolRegistry
# Load tools automatically from MCP server
tools = load_from_stdio(["npx", "-y", "@modelcontextprotocol/server-github"])
registry = MCPToolRegistry(tools)
```
**Single MCP Server:**
```python
from arcade_evals import MCPToolRegistry, ExpectedToolCall
registry = MCPToolRegistry(mcp_tools)
suite = EvalSuite(catalog=registry)
suite.add_case(
expected_tool_calls=[
ExpectedToolCall(tool_name="tool_name", args={...})
]
)
```
**Multiple MCP Servers:**
```python
from arcade_evals import CompositeMCPRegistry, load_from_stdio
# Load from multiple servers
github_tools = load_from_stdio(["npx", "-y", "@modelcontextprotocol/server-github"])
slack_tools = load_from_stdio(["npx", "-y", "@modelcontextprotocol/server-slack"])
composite = CompositeMCPRegistry(
tool_lists={
"github": github_tools,
"slack": slack_tools,
}
)
suite = EvalSuite(catalog=composite)
suite.add_case(
expected_tool_calls=[
ExpectedToolCall(tool_name="github_list_issues", args={...})
]
)
```
## Implementation
### Files Changed
- **`libs/arcade-evals/arcade_evals/registry.py`** (NEW): Registry
abstractions and implementations
- **`libs/arcade-evals/arcade_evals/loaders.py`** (NEW): Automatic tool
loading from MCP servers
- **`libs/arcade-evals/arcade_evals/eval.py`** (MODIFIED): Enhanced
`ExpectedToolCall` and evaluation logic
- **`libs/arcade-evals/arcade_evals/__init__.py`** (MODIFIED): Exported
new registries and loaders
### Key Technical Details
- Added `BaseToolRegistry` interface for abstraction
- `MCPToolRegistry` handles single server tools
- `CompositeMCPRegistry` manages multiple servers with collision
detection
- `load_from_stdio()` and `load_from_http()` for automatic tool
discovery
- Fixed name normalization bug: MCP tools use underscores (not dots)
- Optimized tool copying: 2.5x faster via shallow copy
## Testing
- ✅ 41 tests passing (25 new tests added)
- ✅ `test_eval_mcp_registry.py`: MCPToolRegistry functionality
- ✅ `test_eval_composite_mcp.py`: CompositeMCPRegistry with multiple
servers
- ✅ Verified backward compatibility with Python tools
## Backward Compatibility
✅ **100% backward compatible** - No breaking changes
## Breaking Changes
**None**
<!-- CURSOR_SUMMARY -->
---
> [!NOTE]
> Adds end-to-end eval UX: examples, a robust CLI runner, and rich
outputs.
>
> - **New examples**: `eval_arcade_gateway.py`,
`eval_stdio_mcp_server.py`, `eval_http_mcp_server.py`,
`eval_comprehensive_comparison.py` with timeouts, error handling, and
track-based comparisons; detailed `README.md`
> - **CLI runner**: `arcade_cli/evals_runner.py` to execute
evals/capture in parallel with progress, error isolation, failed-only
filtering, context inclusion, and multi-provider/model support
> - **Output formatters**: `arcade_cli/formatters/` (txt, md, html,
json) for evals and capture; comparative and multi-model HTML with tabs
and context rendering
> - **Display refactor**: `display.py` now supports writing multiple
formats, failed-only disclaimers, include-context, and improved console
summaries
>
> <sup>Written by [Cursor
Bugbot](https://cursor.com/dashboard?tab=bugbot) for commit
ff8acf9c34a6b61462a019a1ee9df081006517d0. This will update automatically
on new commits. Configure
[here](https://cursor.com/dashboard?tab=bugbot).</sup>
<!-- /CURSOR_SUMMARY -->
---------
Co-authored-by: Francisco Liberal <francisco@arcade.dev>
Co-authored-by: Mateo Torres <torresmateo@gmail.com>
270 lines
10 KiB
Python
270 lines
10 KiB
Python
import tempfile
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import httpx
|
|
import pytest
|
|
from arcade_cli.secret import (
|
|
_delete_secret,
|
|
_get_secrets,
|
|
_remove_inline_comment,
|
|
_upsert_secret,
|
|
load_env_file,
|
|
print_secret_table,
|
|
)
|
|
|
|
|
|
class TestPrintSecretTable:
|
|
"""Tests for print_secret_table function."""
|
|
|
|
def test_print_secret_table_empty(self, capsys):
|
|
"""Test printing a table with no secrets."""
|
|
secrets = []
|
|
print_secret_table(secrets)
|
|
|
|
captured = capsys.readouterr()
|
|
assert "Tool Secrets" in captured.out
|
|
|
|
|
|
class TestLoadEnvFile:
|
|
"""Tests for load_env_file function."""
|
|
|
|
def test_load_env_file_basic(self):
|
|
"""Test loading a basic .env file."""
|
|
env_content = """
|
|
KEY1=value1
|
|
KEY2=value2
|
|
# This is a comment
|
|
KEY3=value3
|
|
"""
|
|
with tempfile.NamedTemporaryFile(mode="w", suffix=".env", delete=False) as f:
|
|
f.write(env_content)
|
|
f.flush()
|
|
|
|
secrets = load_env_file(f.name)
|
|
|
|
assert secrets == {
|
|
"KEY1": "value1",
|
|
"KEY2": "value2",
|
|
"KEY3": "value3",
|
|
}
|
|
|
|
def test_load_env_file_with_quotes(self):
|
|
"""Test loading .env file with quoted values."""
|
|
env_content = """
|
|
KEY1="quoted value"
|
|
KEY2='single quoted'
|
|
KEY3="value with = sign"
|
|
KEY4="value with # comment inside"
|
|
"""
|
|
with tempfile.NamedTemporaryFile(mode="w", suffix=".env", delete=False) as f:
|
|
f.write(env_content)
|
|
f.flush()
|
|
|
|
secrets = load_env_file(f.name)
|
|
|
|
assert secrets == {
|
|
"KEY1": "quoted value",
|
|
"KEY2": "single quoted",
|
|
"KEY3": "value with = sign",
|
|
"KEY4": "value with # comment inside",
|
|
}
|
|
|
|
def test_load_env_file_with_inline_comments(self):
|
|
"""Test loading .env file with inline comments."""
|
|
env_content = """
|
|
KEY1=value1 # inline comment
|
|
KEY2="quoted value" # comment after quote
|
|
KEY3=value3# no space before comment
|
|
"""
|
|
with tempfile.NamedTemporaryFile(mode="w", suffix=".env", delete=False) as f:
|
|
f.write(env_content)
|
|
f.flush()
|
|
|
|
secrets = load_env_file(f.name)
|
|
|
|
assert secrets == {
|
|
"KEY1": "value1",
|
|
"KEY2": "quoted value",
|
|
"KEY3": "value3# no space before comment", # No space, so not treated as comment
|
|
}
|
|
|
|
def test_load_env_file_skip_empty_and_invalid(self):
|
|
"""Test that empty lines, comments, and invalid entries are skipped."""
|
|
env_content = """
|
|
# Comment line
|
|
KEY1=value1
|
|
|
|
KEY2=
|
|
=value_without_key
|
|
KEY3=value3
|
|
invalid_line_without_equals
|
|
KEY4=value4
|
|
"""
|
|
with tempfile.NamedTemporaryFile(mode="w", suffix=".env", delete=False) as f:
|
|
f.write(env_content)
|
|
f.flush()
|
|
|
|
secrets = load_env_file(f.name)
|
|
|
|
assert secrets == {
|
|
"KEY1": "value1",
|
|
"KEY3": "value3",
|
|
"KEY4": "value4",
|
|
}
|
|
|
|
|
|
class TestRemoveInlineComment:
|
|
"""Tests for _remove_inline_comment function."""
|
|
|
|
def test_remove_inline_comment_unquoted(self):
|
|
"""Test removing inline comments from unquoted values."""
|
|
assert _remove_inline_comment("value # comment") == "value"
|
|
assert _remove_inline_comment("value# no space") == "value# no space"
|
|
assert _remove_inline_comment("value") == "value"
|
|
assert _remove_inline_comment("value with spaces # comment") == "value with spaces"
|
|
|
|
def test_remove_inline_comment_double_quoted(self):
|
|
"""Test removing inline comments from double-quoted values."""
|
|
assert _remove_inline_comment('"quoted value" # comment') == "quoted value"
|
|
assert _remove_inline_comment('"value with # inside"') == "value with # inside"
|
|
assert _remove_inline_comment('"quoted value"') == "quoted value"
|
|
assert _remove_inline_comment('"unclosed quote') == '"unclosed quote'
|
|
|
|
def test_remove_inline_comment_single_quoted(self):
|
|
"""Test removing inline comments from single-quoted values."""
|
|
assert _remove_inline_comment("'quoted value' # comment") == "quoted value"
|
|
assert _remove_inline_comment("'value with # inside'") == "value with # inside"
|
|
assert _remove_inline_comment("'quoted value'") == "quoted value"
|
|
assert _remove_inline_comment("'unclosed quote") == "'unclosed quote"
|
|
|
|
def test_remove_inline_comment_edge_cases(self):
|
|
"""Test edge cases for inline comment removal."""
|
|
assert _remove_inline_comment("") == ""
|
|
|
|
|
|
class TestUpsertSecretToEngine:
|
|
"""Tests for _upsert_secret function."""
|
|
|
|
@patch("arcade_cli.secret.get_auth_headers")
|
|
@patch("arcade_cli.secret.get_org_scoped_url")
|
|
@patch("arcade_cli.secret.httpx.put")
|
|
def test_upsert_secret_success(self, mock_put, mock_get_url, mock_get_headers):
|
|
"""Test successful secret upsert."""
|
|
mock_response = MagicMock()
|
|
mock_response.raise_for_status.return_value = None
|
|
mock_put.return_value = mock_response
|
|
mock_get_url.return_value = "https://api.example.com/v1/org/test-org/secrets/SECRET_KEY"
|
|
mock_get_headers.return_value = {"Authorization": "Bearer test-api-key"}
|
|
|
|
_upsert_secret("SECRET_KEY", "secret-value")
|
|
|
|
mock_put.assert_called_once_with(
|
|
"https://api.example.com/v1/org/test-org/secrets/SECRET_KEY",
|
|
headers={"Authorization": "Bearer test-api-key"},
|
|
json={"description": "Secret set via CLI", "value": "secret-value"},
|
|
)
|
|
mock_response.raise_for_status.assert_called_once()
|
|
|
|
@patch("arcade_cli.secret.get_auth_headers")
|
|
@patch("arcade_cli.secret.get_org_scoped_url")
|
|
@patch("arcade_cli.secret.httpx.put")
|
|
def test_upsert_secret_http_error(self, mock_put, mock_get_url, mock_get_headers):
|
|
"""Test secret upsert with HTTP error."""
|
|
mock_response = MagicMock()
|
|
mock_response.raise_for_status.side_effect = httpx.HTTPStatusError(
|
|
"Bad Request", request=MagicMock(), response=MagicMock()
|
|
)
|
|
mock_put.return_value = mock_response
|
|
mock_get_url.return_value = "https://api.example.com/v1/org/test-org/secrets/SECRET_KEY"
|
|
mock_get_headers.return_value = {"Authorization": "Bearer test-api-key"}
|
|
|
|
with pytest.raises(httpx.HTTPStatusError):
|
|
_upsert_secret("SECRET_KEY", "secret-value")
|
|
|
|
|
|
class TestGetSecretsFromEngine:
|
|
"""Tests for _get_secrets function."""
|
|
|
|
@patch("arcade_cli.secret.get_auth_headers")
|
|
@patch("arcade_cli.secret.get_org_scoped_url")
|
|
@patch("arcade_cli.secret.httpx.get")
|
|
def test_get_secrets_success(self, mock_get, mock_get_url, mock_get_headers):
|
|
"""Test successful secrets retrieval."""
|
|
mock_response = MagicMock()
|
|
mock_response.raise_for_status.return_value = None
|
|
mock_response.json.return_value = {
|
|
"items": [
|
|
{"key": "SECRET1", "id": "id1"},
|
|
{"key": "SECRET2", "id": "id2"},
|
|
]
|
|
}
|
|
mock_get.return_value = mock_response
|
|
mock_get_url.return_value = "https://api.example.com/v1/org/test-org/secrets"
|
|
mock_get_headers.return_value = {"Authorization": "Bearer test-api-key"}
|
|
|
|
secrets = _get_secrets()
|
|
|
|
assert secrets == [
|
|
{"key": "SECRET1", "id": "id1"},
|
|
{"key": "SECRET2", "id": "id2"},
|
|
]
|
|
mock_get.assert_called_once_with(
|
|
"https://api.example.com/v1/org/test-org/secrets",
|
|
headers={"Authorization": "Bearer test-api-key"},
|
|
)
|
|
mock_response.raise_for_status.assert_called_once()
|
|
|
|
@patch("arcade_cli.secret.get_auth_headers")
|
|
@patch("arcade_cli.secret.get_org_scoped_url")
|
|
@patch("arcade_cli.secret.httpx.get")
|
|
def test_get_secrets_http_error(self, mock_get, mock_get_url, mock_get_headers):
|
|
"""Test secrets retrieval with HTTP error."""
|
|
mock_response = MagicMock()
|
|
mock_response.raise_for_status.side_effect = httpx.HTTPStatusError(
|
|
"Unauthorized", request=MagicMock(), response=MagicMock()
|
|
)
|
|
mock_get.return_value = mock_response
|
|
mock_get_url.return_value = "https://api.example.com/v1/org/test-org/secrets"
|
|
mock_get_headers.return_value = {"Authorization": "Bearer test-api-key"}
|
|
|
|
with pytest.raises(httpx.HTTPStatusError):
|
|
_get_secrets()
|
|
|
|
|
|
class TestDeleteSecretFromEngine:
|
|
"""Tests for _delete_secret function."""
|
|
|
|
@patch("arcade_cli.secret.get_auth_headers")
|
|
@patch("arcade_cli.secret.get_org_scoped_url")
|
|
@patch("arcade_cli.secret.httpx.delete")
|
|
def test_delete_secret_success(self, mock_delete, mock_get_url, mock_get_headers):
|
|
"""Test successful secret deletion."""
|
|
mock_response = MagicMock()
|
|
mock_response.raise_for_status.return_value = None
|
|
mock_delete.return_value = mock_response
|
|
mock_get_url.return_value = "https://api.example.com/v1/org/test-org/secrets/secret-id-123"
|
|
mock_get_headers.return_value = {"Authorization": "Bearer test-api-key"}
|
|
|
|
_delete_secret("secret-id-123")
|
|
|
|
mock_delete.assert_called_once_with(
|
|
"https://api.example.com/v1/org/test-org/secrets/secret-id-123",
|
|
headers={"Authorization": "Bearer test-api-key"},
|
|
)
|
|
mock_response.raise_for_status.assert_called_once()
|
|
|
|
@patch("arcade_cli.secret.get_auth_headers")
|
|
@patch("arcade_cli.secret.get_org_scoped_url")
|
|
@patch("arcade_cli.secret.httpx.delete")
|
|
def test_delete_secret_http_error(self, mock_delete, mock_get_url, mock_get_headers):
|
|
"""Test secret deletion with HTTP error."""
|
|
mock_response = MagicMock()
|
|
mock_response.raise_for_status.side_effect = httpx.HTTPStatusError(
|
|
"Not Found", request=MagicMock(), response=MagicMock()
|
|
)
|
|
mock_delete.return_value = mock_response
|
|
mock_get_url.return_value = "https://api.example.com/v1/org/test-org/secrets/secret-id-123"
|
|
mock_get_headers.return_value = {"Authorization": "Bearer test-api-key"}
|
|
|
|
with pytest.raises(httpx.HTTPStatusError):
|
|
_delete_secret("secret-id-123")
|