f4558ef3a8
# Improvements to Arcade TDK Error Handling
I tried my very best to not make any breaking changes in this PR. So,
you will notice various "Deprecation" notices throughout.
### Instructions for PR reviewers
1. Pull down this PR's branch
2. Pull down the Engine's tool error handling PR's branch
3. Update your installed arcadepy to have the following:
- In `arcadepy/resources/tools/tools.py`, if you want to test out
including stacktraces, then you need to update `ToolsResource.execute`
to accept a `include_error_stacktrace` argument and also include the
"include_error_stacktrace" argument to the POST to the Engine inside of
the function's execute method's body.
- In `arcadepy/types/execute_tool_response.py` add the following enum
```py
class ErrorKind(str, Enum):
"""Error kind that is comprised of
- the who (toolkit, tool, upstream)
- the when (load time, definition parsing time, runtime)
- the what (bad_definition, bad_input, bad_output, retry,
context_required, fatal, etc.)"""
TOOLKIT_LOAD_FAILED = "TOOLKIT_LOAD_FAILED"
TOOL_DEFINITION_BAD_DEFINITION = "TOOL_DEFINITION_BAD_DEFINITION"
TOOL_DEFINITION_BAD_INPUT_SCHEMA = "TOOL_DEFINITION_BAD_INPUT_SCHEMA"
TOOL_DEFINITION_BAD_OUTPUT_SCHEMA = "TOOL_DEFINITION_BAD_OUTPUT_SCHEMA"
TOOL_RUNTIME_BAD_INPUT_VALUE = "TOOL_RUNTIME_BAD_INPUT_VALUE"
TOOL_RUNTIME_BAD_OUTPUT_VALUE = "TOOL_RUNTIME_BAD_OUTPUT_VALUE"
TOOL_RUNTIME_RETRY = "TOOL_RUNTIME_RETRY"
TOOL_RUNTIME_CONTEXT_REQUIRED = "TOOL_RUNTIME_CONTEXT_REQUIRED"
TOOL_RUNTIME_FATAL = "TOOL_RUNTIME_FATAL"
UPSTREAM_RUNTIME_BAD_REQUEST = "UPSTREAM_RUNTIME_BAD_REQUEST"
UPSTREAM_RUNTIME_AUTH_ERROR = "UPSTREAM_RUNTIME_AUTH_ERROR"
UPSTREAM_RUNTIME_NOT_FOUND = "UPSTREAM_RUNTIME_NOT_FOUND"
UPSTREAM_RUNTIME_VALIDATION_ERROR = "UPSTREAM_RUNTIME_VALIDATION_ERROR"
UPSTREAM_RUNTIME_RATE_LIMIT = "UPSTREAM_RUNTIME_RATE_LIMIT"
UPSTREAM_RUNTIME_SERVER_ERROR = "UPSTREAM_RUNTIME_SERVER_ERROR"
UPSTREAM_RUNTIME_UNMAPPED = "UPSTREAM_RUNTIME_UNMAPPED"
UNKNOWN = "UNKNOWN"
```
- In `arcadepy/types/execute_tool_response.py` add the following fields
to OutputError:
```py
kind: ErrorKind
status_code: Optional[int] = None
stacktrace: Optional[str] = None
extra: Optional[dict[str, Any]] = None
```
### Example Client Usage
```py
# Example of handling an upstream rate limit
error = response.output.error
if error and error.kind == ErrorKind.UPSTREAM_RUNTIME_RATE_LIMIT:
sleep_time = error.retry_after_ms / 1000
time.sleep(sleep_time)
# and then execute again
```
```py
# Examples of determining what type of runtime error it is
error = response.output.error
if error:
is_retryable_error = error.kind == ErrorKind.TOOL_RUNTIME_RETRY
is_a_bug_in_the_tool = error.kind == ErrorKind.TOOL_RUNTIME_FATAL
is_additional_context_required = error.kind == ErrorKind.TOOL_RUNTIME_CONTEXT_REQUIRED
```
### Example Tool Usage
```py
# EXAMPLE 1 letting Arcade handle upstream error handling for you
reddit_client.post(params) # Arcade's httpx adapter will handle error handling for you!
# ------------------------------------
# EXAMPLE 2 handling upstream bad request yourself, but letting Arcade handle the rest
try:
reddit_client.post(params)
except httpx.HTTPStatusError as e:
if e.status_code == 400:
raise UpstreamError("My extra custom message) from e
raise
```
```py
# EXAMPLE 1 letting Arcade handle it for you
risky_element = my_risky_list[42] # Arcade will raise a FatalToolError for you
# ------------------------------------
# EXAMPLE 2 handling it yourself for extra flexibility
try:
risky_element = my_risky_list[42]
except IndexError as e:
raise FatalToolError("My extra custom message") from e
```
### Non-runtime Error Message Examples
Example ToolkitLoadError Messages:
```
- [TOOLKIT_LOAD_FAILED] ToolkitLoadError when loading toolkit 'sample_tool': Could not import module mock_module. Reason: Mock import error
- [TOOLKIT_LOAD_FAILED] ToolkitLoadError when loading toolkit 'test_toolkit': Tool 'ValidTool' in toolkit 'test_toolkit' already exists in the catalog.
```
Example ToolDefinitionError Messages
```
- [TOOL_DEFINITION_BAD_DEFINITION] ToolDefinitionError in definition of tool 'tool_missing_description': Tool 'tool_missing_description' is missing a description
- [TOOL_DEFINITION_BAD_DEFINITION] ToolDefinitionError in definition of tool 'tool_with_invalid_secret_type': Secret keys must be strings (error in tool ToolWithInvalidSecretType).
- [TOOL_DEFINITION_BAD_DEFINITION] ToolDefinitionError in definition of tool 'tool_with_empty_secret': Secrets must have a non-empty key (error in tool ToolWithEmptySecret).
- [TOOL_DEFINITION_BAD_DEFINITION] ToolDefinitionError in definition of tool 'tool_with_invalid_metadata_type': Metadata must be strings (error in tool ToolWithInvalidMetadataType).
- [TOOL_DEFINITION_BAD_DEFINITION] ToolDefinitionError in definition of tool 'tool_with_metadata_requiring_auth_without_auth': Tool ToolWithMetadataRequiringAuthWithoutAuth declares metadata key 'client_id', which requires that the tool has an auth requirement, but no auth requirement was provided. Please specify an auth requirement.
- [TOOL_DEFINITION_BAD_DEFINITION] ToolDefinitionError in definition of tool 'tool_with_empty_metadata': Metadata must have a non-empty key (error in tool ToolWithEmptyMetadata).
- [TOOL_DEFINITION_BAD_DEFINITION] ToolDefinitionError in definition of tool 'tool_with_unsupported_param_type': Unsupported parameter type: <class 'test_catalog.MyFancyTestClass'>
```
Example ToolInputSchemaError Messages
```
- [TOOL_DEFINITION_BAD_INPUT_SCHEMA] ToolInputSchemaError in definition of tool 'tool_with_missing_input_parameter_annotation': Parameter 'input_text' is missing a description
- [TOOL_DEFINITION_BAD_INPUT_SCHEMA] ToolInputSchemaError in definition of tool 'tool_with_no_type_annotation': Parameter param has no type annotation.
- [TOOL_DEFINITION_BAD_INPUT_SCHEMA] ToolInputSchemaError in definition of tool 'tool_with_invalid_param_name': Invalid parameter name: '123invalid' is not a valid identifier. Identifiers must start with a letter or underscore, and can only contain letters, digits, or underscores.
- [TOOL_DEFINITION_BAD_INPUT_SCHEMA] ToolInputSchemaError in definition of tool 'tool_with_too_many_annotations': Parameter param: Annotated[str, 'name', 'desc', 'extra'] has too many string annotations. Expected 0, 1, or 2, got 3.
- [TOOL_DEFINITION_BAD_INPUT_SCHEMA] ToolInputSchemaError in definition of tool 'tool_with_required_union_param': Parameter param is a union type. Only optional types are supported.
- [TOOL_DEFINITION_BAD_INPUT_SCHEMA] ToolInputSchemaError in definition of tool 'tool_with_non_callable_default_factory': Default factory for parameter param: Annotated[str, 'Parameter'] = FieldInfo(annotation=NoneType, required=False, default_factory=str) is not callable.
- [TOOL_DEFINITION_BAD_INPUT_SCHEMA] ToolInputSchemaError in definition of tool 'tool_with_multiple_tool_contexts': Only one ToolContext parameter is supported, but tool tool_with_multiple_tool_contexts has multiple.
```
Example ToolOutputSchemaError Messages
```
- [TOOL_DEFINITION_BAD_OUTPUT_SCHEMA] ToolOutputSchemaError in definition of tool 'tool_missing_return_type_hint': Tool 'ToolMissingReturnTypeHint' must have a return type
- [TOOL_DEFINITION_BAD_OUTPUT_SCHEMA] ToolOutputSchemaError in definition of tool 'tool_with_unsupported_output_type': Unsupported output type '<class 'test_catalog.MyFancyTestClass'>'. Only built-in Python types, TypedDicts, Pydantic models, and standard collections are supported as tool output types.
```
### Runtime Error Message Examples
Example Tool Runtime Error Messages
```
- [TOOL_RUNTIME_FATAL] FatalToolError during execution of tool 'get_posts_in_subreddit': list index out of range
- [TOOL_RUNTIME_CONTEXT_REQUIRED] ContextRequiredToolError during execution of tool 'get_posts_in_subreddit': Ambiguous username. Please provide a more specific username
- [TOOL_RUNTIME_RETRY] RetryableToolError during execution of tool 'get_posts_in_subreddit': Retry with subreddit=learnpython or subreddit=learnprogramming
```
Example Upstream Runtime Error Messages
```
- [UPSTREAM_RUNTIME_RATE_LIMIT] UpstreamRateLimitError during execution of tool 'get_posts_in_subreddit': 429 Client Error: Too Many Requests
- [UPSTREAM_RUNTIME_BAD_REQUEST] UpstreamError during execution of tool 'get_posts_in_subreddit': 400 Client Error: Bad request. Missing 'id' parameter.
- [UPSTREAM_RUNTIME_BAD_REQUEST] UpstreamError during execution of tool 'search_files': Upstream Google API error: Invalid value '-23'. Values must be within the range: [value: 1\n, value: 1000\n]
```
194 lines
5.5 KiB
Python
194 lines
5.5 KiB
Python
from typing import Annotated
|
|
|
|
import pytest
|
|
from arcade_core.catalog import ToolCatalog
|
|
from arcade_core.errors import ToolDefinitionError, ToolInputSchemaError
|
|
from arcade_core.schema import ToolContext, ToolMetadataKey
|
|
from arcade_tdk import tool
|
|
|
|
|
|
@tool
|
|
def func_with_missing_description():
|
|
pass
|
|
|
|
|
|
@tool(desc="Returning function with declared no return type (illegal)")
|
|
def func_with_missing_return_type():
|
|
return "hello world"
|
|
|
|
|
|
@tool(desc="A function with a union return type (illegal)")
|
|
def func_with_union_return_type_1() -> str | int:
|
|
return "hello world"
|
|
|
|
|
|
@tool(desc="A function with a union return type (illegal)")
|
|
def func_with_union_return_type_2() -> str | int:
|
|
return "hello world"
|
|
|
|
|
|
@tool(desc="A function with a parameter type (illegal)")
|
|
def func_with_missing_param_type(param1):
|
|
pass
|
|
|
|
|
|
@tool(desc="A function with a parameter missing a description (illegal)")
|
|
def func_with_missing_param_description(param1: str):
|
|
pass
|
|
|
|
|
|
@tool(desc="A function with an unsupported parameter type (illegal)")
|
|
def func_with_unsupported_param(param1: complex):
|
|
pass
|
|
|
|
|
|
@tool(desc="A function with a union parameter (illegal)")
|
|
def func_with_union_param_1(param1: str | int):
|
|
pass
|
|
|
|
|
|
@tool(desc="A function with a union parameter (illegal)")
|
|
def func_with_union_param_2(param1: str | int):
|
|
pass
|
|
|
|
|
|
@tool(desc="A function with multiple context parameters (illegal)")
|
|
def func_with_multiple_context_params(context: ToolContext, context2: ToolContext):
|
|
pass
|
|
|
|
|
|
@tool(desc="A function with an invalid renamed parameter")
|
|
def func_with_invalid_renamed_param(
|
|
param1: Annotated[str, "invalid-param-name", "The first parameter"],
|
|
):
|
|
pass
|
|
|
|
|
|
@tool(
|
|
desc="A function with a required secret with a missing key (illegal)",
|
|
requires_secrets=[""],
|
|
)
|
|
def func_with_missing_secret_key(context: ToolContext):
|
|
pass
|
|
|
|
|
|
@tool(
|
|
desc="A function that requires a secret (invalid type)",
|
|
requires_secrets=[True],
|
|
)
|
|
def func_with_secret_requirement_invalid_type():
|
|
pass
|
|
|
|
|
|
@tool(
|
|
desc="A function with a required metadata with a missing key (illegal)",
|
|
requires_metadata=[""],
|
|
)
|
|
def func_with_missing_metadata_key(context: ToolContext):
|
|
pass
|
|
|
|
|
|
@tool(
|
|
desc="A function that requires metadata with an invalid type (illegal)",
|
|
requires_metadata=[True],
|
|
)
|
|
def func_with_metadata_requirement_invalid_type():
|
|
pass
|
|
|
|
|
|
@tool(
|
|
desc="A function with a required metadata key that depends on the tool having an auth requirement, but the tool does not have an auth requirement (illegal)",
|
|
requires_metadata=[ToolMetadataKey.CLIENT_ID],
|
|
)
|
|
def func_with_metadata_and_auth_dependency():
|
|
pass
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"func_under_test, exception_type",
|
|
[
|
|
pytest.param(
|
|
func_with_missing_description,
|
|
ToolDefinitionError,
|
|
id=func_with_missing_description.__name__,
|
|
),
|
|
pytest.param(
|
|
func_with_missing_return_type,
|
|
ToolDefinitionError,
|
|
id=func_with_missing_return_type.__name__,
|
|
),
|
|
pytest.param(
|
|
func_with_missing_param_type,
|
|
ToolDefinitionError,
|
|
id=func_with_missing_param_type.__name__,
|
|
),
|
|
pytest.param(
|
|
func_with_missing_param_description,
|
|
ToolDefinitionError,
|
|
id=func_with_missing_param_description.__name__,
|
|
),
|
|
pytest.param(
|
|
func_with_unsupported_param,
|
|
ToolDefinitionError,
|
|
id=func_with_unsupported_param.__name__,
|
|
),
|
|
pytest.param(
|
|
func_with_union_param_1,
|
|
ToolDefinitionError,
|
|
id=func_with_union_param_1.__name__,
|
|
),
|
|
pytest.param(
|
|
func_with_union_param_2,
|
|
ToolDefinitionError,
|
|
id=func_with_union_param_2.__name__,
|
|
),
|
|
pytest.param(
|
|
func_with_multiple_context_params,
|
|
ToolDefinitionError,
|
|
id=func_with_multiple_context_params.__name__,
|
|
),
|
|
pytest.param(
|
|
func_with_invalid_renamed_param,
|
|
ToolInputSchemaError,
|
|
id=func_with_invalid_renamed_param.__name__,
|
|
),
|
|
pytest.param(
|
|
func_with_missing_secret_key,
|
|
ToolDefinitionError,
|
|
id=func_with_missing_secret_key.__name__,
|
|
),
|
|
pytest.param(
|
|
func_with_secret_requirement_invalid_type,
|
|
ToolDefinitionError,
|
|
id=func_with_secret_requirement_invalid_type.__name__,
|
|
),
|
|
pytest.param(
|
|
func_with_missing_metadata_key,
|
|
ToolDefinitionError,
|
|
id=func_with_missing_metadata_key.__name__,
|
|
),
|
|
pytest.param(
|
|
func_with_metadata_requirement_invalid_type,
|
|
ToolDefinitionError,
|
|
id=func_with_metadata_requirement_invalid_type.__name__,
|
|
),
|
|
pytest.param(
|
|
func_with_metadata_and_auth_dependency,
|
|
ToolDefinitionError,
|
|
id=func_with_metadata_and_auth_dependency.__name__,
|
|
),
|
|
pytest.param(
|
|
func_with_union_return_type_1,
|
|
ToolDefinitionError,
|
|
id=func_with_union_return_type_1.__name__,
|
|
),
|
|
pytest.param(
|
|
func_with_union_return_type_2,
|
|
ToolDefinitionError,
|
|
id=func_with_union_return_type_2.__name__,
|
|
),
|
|
],
|
|
)
|
|
def test_missing_info_raises_error(func_under_test, exception_type):
|
|
with pytest.raises(exception_type):
|
|
ToolCatalog.create_tool_definition(func_under_test, "1.0")
|