248 lines
9.7 KiB
Python
248 lines
9.7 KiB
Python
import pytest
|
|
from arcade_core.catalog import ToolCatalog
|
|
from arcade_core.errors import ToolDefinitionError
|
|
from arcade_core.metadata import (
|
|
_INDETERMINATE_OPERATIONS,
|
|
_MUTATING_OPERATIONS,
|
|
_READ_ONLY_OPERATIONS,
|
|
Behavior,
|
|
Classification,
|
|
Operation,
|
|
ServiceDomain,
|
|
ToolMetadata,
|
|
)
|
|
from arcade_tdk import tool
|
|
|
|
|
|
class TestEnumCoverage:
|
|
"""
|
|
Tests to ensure all enum values are accounted for in validation helper sets.
|
|
|
|
These tests will fail if new enum values are added without updating the
|
|
corresponding helper sets, ensuring future maintainers don't forget to
|
|
categorize new values.
|
|
"""
|
|
|
|
def test_all_operations_are_categorized(self):
|
|
"""Every Operation must be in _READ_ONLY_OPERATIONS, _MUTATING_OPERATIONS, or _INDETERMINATE_OPERATIONS."""
|
|
all_operations = set(Operation)
|
|
categorized_operations = _READ_ONLY_OPERATIONS | _MUTATING_OPERATIONS | _INDETERMINATE_OPERATIONS
|
|
|
|
# Check that every operation is categorized
|
|
uncategorized = all_operations - categorized_operations
|
|
assert not uncategorized, (
|
|
f"The following Operation values are not categorized in _READ_ONLY_OPERATIONS, "
|
|
f"_MUTATING_OPERATIONS, or _INDETERMINATE_OPERATIONS: {uncategorized}. "
|
|
f"Please add them to the appropriate set in arcade_core/metadata.py"
|
|
)
|
|
|
|
# Check that there are no extra operations in the sets that don't exist in the enum
|
|
extra = categorized_operations - all_operations
|
|
assert not extra, (
|
|
f"The following values are in _READ_ONLY_OPERATIONS, _MUTATING_OPERATIONS, or "
|
|
f"_INDETERMINATE_OPERATIONS but don't exist in the Operation enum: {extra}"
|
|
)
|
|
|
|
def test_operation_categories_are_disjoint(self):
|
|
"""_READ_ONLY_OPERATIONS, _MUTATING_OPERATIONS, and _INDETERMINATE_OPERATIONS should not overlap."""
|
|
ro_mut = _READ_ONLY_OPERATIONS & _MUTATING_OPERATIONS
|
|
assert not ro_mut, (
|
|
f"The following Operation values appear in both _READ_ONLY_OPERATIONS and "
|
|
f"_MUTATING_OPERATIONS: {ro_mut}. An operation should be in exactly one category."
|
|
)
|
|
|
|
ro_ind = _READ_ONLY_OPERATIONS & _INDETERMINATE_OPERATIONS
|
|
assert not ro_ind, (
|
|
f"The following Operation values appear in both _READ_ONLY_OPERATIONS and "
|
|
f"_INDETERMINATE_OPERATIONS: {ro_ind}. An operation should be in exactly one category."
|
|
)
|
|
|
|
mut_ind = _MUTATING_OPERATIONS & _INDETERMINATE_OPERATIONS
|
|
assert not mut_ind, (
|
|
f"The following Operation values appear in both _MUTATING_OPERATIONS and "
|
|
f"_INDETERMINATE_OPERATIONS: {mut_ind}. An operation should be in exactly one category."
|
|
)
|
|
|
|
|
|
class TestToolMetadataValidation:
|
|
"""Test strict mode validation rules for ToolMetadata."""
|
|
|
|
def test_valid_metadata_passes(self):
|
|
"""Valid metadata with consistent values should not raise."""
|
|
metadata = ToolMetadata(
|
|
classification=Classification(
|
|
service_domains=[ServiceDomain.EMAIL],
|
|
),
|
|
behavior=Behavior(
|
|
operations=[Operation.CREATE],
|
|
read_only=False,
|
|
destructive=False,
|
|
open_world=True,
|
|
),
|
|
)
|
|
assert metadata is not None
|
|
|
|
def test_mutating_operation_with_read_only_raises(self):
|
|
"""Mutating operations with read_only=True should raise when validated."""
|
|
metadata = ToolMetadata(
|
|
behavior=Behavior(operations=[Operation.CREATE], read_only=True),
|
|
)
|
|
with pytest.raises(
|
|
ToolDefinitionError, match="mutating operation.*but is marked read_only=True"
|
|
):
|
|
metadata.validate_for_tool()
|
|
|
|
def test_opaque_with_read_only_raises(self):
|
|
"""OPAQUE operation with read_only=True should raise when validated."""
|
|
metadata = ToolMetadata(
|
|
behavior=Behavior(operations=[Operation.OPAQUE], read_only=True),
|
|
)
|
|
with pytest.raises(
|
|
ToolDefinitionError, match="OPAQUE operation but is marked read_only=True"
|
|
):
|
|
metadata.validate_for_tool()
|
|
|
|
def test_delete_without_destructive_raises(self):
|
|
"""DELETE operation without destructive=True should raise when validated."""
|
|
metadata = ToolMetadata(
|
|
behavior=Behavior(operations=[Operation.DELETE], destructive=False),
|
|
)
|
|
with pytest.raises(
|
|
ToolDefinitionError, match="'DELETE' operation.*but is not marked destructive=True"
|
|
):
|
|
metadata.validate_for_tool()
|
|
|
|
def test_service_domain_with_open_world_false_raises(self):
|
|
"""ServiceDomain present with open_world=False should raise when validated."""
|
|
metadata = ToolMetadata(
|
|
classification=Classification(service_domains=[ServiceDomain.EMAIL]),
|
|
behavior=Behavior(open_world=False),
|
|
)
|
|
with pytest.raises(
|
|
ToolDefinitionError, match="ServiceDomain.*but is marked open_world=False"
|
|
):
|
|
metadata.validate_for_tool()
|
|
|
|
def test_strict_false_bypasses_validation(self):
|
|
"""Setting strict=False should bypass all validation rules."""
|
|
# This would normally raise due to contradiction
|
|
metadata = ToolMetadata(
|
|
behavior=Behavior(operations=[Operation.CREATE], read_only=True),
|
|
strict=False,
|
|
)
|
|
# No error should be raised when validate_for_tool is called
|
|
metadata.validate_for_tool() # Should not raise
|
|
assert metadata is not None
|
|
|
|
def test_error_message_includes_operation_name(self):
|
|
"""Error messages should include the operation name for debugging."""
|
|
metadata = ToolMetadata(
|
|
behavior=Behavior(operations=[Operation.CREATE], read_only=True),
|
|
)
|
|
with pytest.raises(ToolDefinitionError, match="Tool has the mutating operation"):
|
|
metadata.validate_for_tool()
|
|
|
|
def test_read_only_operation_with_read_only_true_passes(self):
|
|
"""READ operation with read_only=True should pass validation."""
|
|
metadata = ToolMetadata(
|
|
behavior=Behavior(operations=[Operation.READ], read_only=True),
|
|
)
|
|
assert metadata is not None
|
|
assert metadata.behavior.read_only is True
|
|
|
|
def test_multiple_service_domains_allowed(self):
|
|
"""Tools can have multiple service domains."""
|
|
metadata = ToolMetadata(
|
|
classification=Classification(
|
|
service_domains=[ServiceDomain.CLOUD_STORAGE, ServiceDomain.DOCUMENTS],
|
|
),
|
|
behavior=Behavior(operations=[Operation.READ], read_only=True, open_world=True),
|
|
)
|
|
assert len(metadata.classification.service_domains) == 2
|
|
|
|
def test_extras_accepts_arbitrary_dict(self):
|
|
"""Extras field accepts arbitrary key/value pairs."""
|
|
metadata = ToolMetadata(
|
|
extras={"idp": "entraID", "requires_mfa": True, "max_requests": 100},
|
|
)
|
|
assert metadata.extras["idp"] == "entraID"
|
|
assert metadata.extras["requires_mfa"] is True
|
|
assert metadata.extras["max_requests"] == 100
|
|
|
|
|
|
class TestToolDecoratorWithMetadata:
|
|
"""Test @tool decorator with metadata parameter."""
|
|
|
|
def test_decorator_accepts_metadata(self):
|
|
"""Decorator should store metadata as __tool_metadata__ attribute."""
|
|
|
|
@tool(
|
|
desc="Test tool",
|
|
metadata=ToolMetadata(
|
|
classification=Classification(service_domains=[ServiceDomain.MESSAGING]),
|
|
behavior=Behavior(operations=[Operation.CREATE], open_world=True),
|
|
),
|
|
)
|
|
def my_tool() -> str:
|
|
return "test"
|
|
|
|
assert hasattr(my_tool, "__tool_metadata__")
|
|
assert my_tool.__tool_metadata__.classification.service_domains == [ServiceDomain.MESSAGING]
|
|
|
|
def test_decorator_without_metadata_is_backward_compatible(self):
|
|
"""Decorator should work without metadata (existing tools unchanged)."""
|
|
|
|
@tool(desc="Test tool")
|
|
def my_tool() -> str:
|
|
return "test"
|
|
|
|
assert getattr(my_tool, "__tool_metadata__", None) is None
|
|
|
|
|
|
class TestToolDefinitionWithMetadata:
|
|
"""Test ToolDefinition includes metadata from decorator."""
|
|
|
|
def test_tool_definition_includes_metadata(self):
|
|
"""ToolDefinition.metadata should be populated from decorator."""
|
|
|
|
@tool(
|
|
desc="Send a message",
|
|
metadata=ToolMetadata(
|
|
classification=Classification(
|
|
service_domains=[ServiceDomain.MESSAGING],
|
|
),
|
|
behavior=Behavior(
|
|
operations=[Operation.CREATE],
|
|
read_only=False,
|
|
destructive=False,
|
|
open_world=True,
|
|
),
|
|
extras={"idp": "entraID"},
|
|
),
|
|
)
|
|
def send_message() -> str:
|
|
"""Send a message."""
|
|
return "sent"
|
|
|
|
definition = ToolCatalog.create_tool_definition(
|
|
send_message, toolkit_name="TestToolkit", toolkit_version="1.0.0"
|
|
)
|
|
|
|
assert definition.metadata is not None
|
|
assert definition.metadata.classification.service_domains == [ServiceDomain.MESSAGING]
|
|
assert definition.metadata.behavior.operations == [Operation.CREATE]
|
|
assert definition.metadata.extras == {"idp": "entraID"}
|
|
|
|
def test_tool_definition_without_metadata_is_none(self):
|
|
"""ToolDefinition.metadata should be None when not provided."""
|
|
|
|
@tool(desc="Simple tool")
|
|
def simple_tool() -> str:
|
|
"""A simple tool."""
|
|
return "done"
|
|
|
|
definition = ToolCatalog.create_tool_definition(
|
|
simple_tool, toolkit_name="TestToolkit", toolkit_version="1.0.0"
|
|
)
|
|
|
|
assert definition.metadata is None
|