admin-valentin/arcade-mcp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

arcade docs: add support for GPT-5 series; improve error handling (#529)

Browse source 
Adds support for GPT-5 series of models in `arcade docs`.

Improves error handling when the LLM does not generate a valid JSON for
a given tool sample inputs. Instead of raising an exception, the CLI
uses an empty input, moves on to the next tool, and prints a warning
message asking the user the fill in the input sample manually in
Javascript and Python files.

This PR also moves the Enumerations from a separate `reference.mdx` file
to the main toolkit file, as requested by @EricGustin to simplify the
docs structure.
This commit is contained in:
Renato Byrro 2025-08-10 20:14:59 -03:00 committed by GitHub
parent a85fa76997
commit 19c1e18a8a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 173 additions and 145 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
libs/arcade-cli/arcade_cli/main.py
View file

@ -820,13 +820,12 @@ def docs(
), ),
), ),
openai_model: str = typer.Option( openai_model: str = typer.Option(
"gpt-4o-mini", "gpt-5-mini",
"--openai-model", "--openai-model",
"-m", "-m",
help=( help=(
"A few parts of the documentation are generated using OpenAI API. " "A few parts of the documentation are generated using OpenAI API. "
"This argument controls which OpenAI model to use. " "Choose one of the 'gpt-4o' and 'gpt-5' series models."
"E.g. 'gpt-4o', 'gpt-4o-mini'."
), ),
show_default=True, show_default=True,
), ),
@ -845,6 +844,14 @@ def docs(
), ),
debug: bool = typer.Option(False, "--debug", "-d", help="Show debug information"), debug: bool = typer.Option(False, "--debug", "-d", help="Show debug information"),
) -> None: ) -> None:
if not openai_model.startswith("gpt-4o") and not openai_model.startswith("gpt-5"):
console.print(
f"Attention: '{openai_model}' is not a valid OpenAI model. "
"Please choose one of the 'gpt-4o' and 'gpt-5' series models.",
style="bold red",
)
raise typer.Exit()
try: try:
success = generate_toolkit_docs( success = generate_toolkit_docs(
console=console, console=console,

11
libs/arcade-cli/arcade_cli/toolkit_docs/__init__.py
View file

@ -6,7 +6,6 @@ from rich.console import Console
from arcade_cli.toolkit_docs.docs_builder import ( from arcade_cli.toolkit_docs.docs_builder import (
build_example_path, build_example_path,
build_examples, build_examples,
build_reference_mdx_path,
build_toolkit_mdx, build_toolkit_mdx,
build_toolkit_mdx_path, build_toolkit_mdx_path,
) )
@ -58,8 +57,7 @@ def generate_toolkit_docs(
enums = get_all_enumerations(toolkit_dir) enums = get_all_enumerations(toolkit_dir)
print_debug(f"Building /{toolkit_name.lower()}.mdx file") print_debug(f"Building /{toolkit_name.lower()}.mdx file")
reference_mdx, toolkit_mdx = build_toolkit_mdx( toolkit_mdx = build_toolkit_mdx(
toolkit_dir=toolkit_dir,
tools=tools, tools=tools,
docs_section=docs_section, docs_section=docs_section,
enums=enums, enums=enums,
@ -69,13 +67,6 @@ def generate_toolkit_docs(
toolkit_mdx_path = build_toolkit_mdx_path(docs_section, docs_dir, toolkit_name) toolkit_mdx_path = build_toolkit_mdx_path(docs_section, docs_dir, toolkit_name)
write_file(toolkit_mdx_path, toolkit_mdx) write_file(toolkit_mdx_path, toolkit_mdx)
if reference_mdx:
print_debug(f"Building /{toolkit_name.lower()}/reference.mdx file")
reference_mdx_path = build_reference_mdx_path(docs_section, docs_dir, toolkit_name)
write_file(reference_mdx_path, reference_mdx)
else:
print_debug("No Enums referenced by tool interfaces. Skipping reference.mdx file")
if tool_call_examples: if tool_call_examples:
print_debug("Building tool-call examples in Python and JavaScript") print_debug("Building tool-call examples in Python and JavaScript")
examples = build_examples(print_debug, tools, openai_model) examples = build_examples(print_debug, tools, openai_model)

288
libs/arcade-cli/arcade_cli/toolkit_docs/docs_builder.py
View file

@ -12,6 +12,7 @@ from arcade_core.schema import (
ToolInput, ToolInput,
ToolSecretRequirement, ToolSecretRequirement,
) )
from rich.console import Console
from arcade_cli.toolkit_docs.templates import ( from arcade_cli.toolkit_docs.templates import (
ENUM_ITEM, ENUM_ITEM,
@ -40,6 +41,8 @@ from arcade_cli.toolkit_docs.utils import (
pascal_to_snake_case, pascal_to_snake_case,
) )
console = Console()
def build_toolkit_mdx_path(docs_section: str, docs_root_dir: str, toolkit_name: str) -> str: def build_toolkit_mdx_path(docs_section: str, docs_root_dir: str, toolkit_name: str) -> str:
return os.path.join( return os.path.join(
@ -51,17 +54,6 @@ def build_toolkit_mdx_path(docs_section: str, docs_root_dir: str, toolkit_name:
) )
def build_reference_mdx_path(docs_section: str, docs_root_dir: str, toolkit_name: str) -> str:
return os.path.join(
docs_root_dir,
"pages",
"toolkits",
docs_section,
toolkit_name.lower(),
"reference.mdx",
)
def build_example_path(example_filename: str, docs_root_dir: str, toolkit_name: str) -> str: def build_example_path(example_filename: str, docs_root_dir: str, toolkit_name: str) -> str:
return os.path.join( return os.path.join(
docs_root_dir, docs_root_dir,
@ -75,7 +67,6 @@ def build_example_path(example_filename: str, docs_root_dir: str, toolkit_name:
def build_toolkit_mdx( def build_toolkit_mdx(
toolkit_dir: str,
tools: list[ToolDefinition], tools: list[ToolDefinition],
docs_section: str, docs_section: str,
enums: dict[str, type[Enum]], enums: dict[str, type[Enum]],
@ -102,16 +93,20 @@ def build_toolkit_mdx(
) )
table_of_contents = build_table_of_contents(tools) table_of_contents = build_table_of_contents(tools)
footer = build_footer(toolkit_name, pip_package_name, sample_tool.requirements.authorization) footer = build_footer(toolkit_name, pip_package_name, sample_tool.requirements.authorization)
referenced_enums, tools_specs = build_tools_specs(tools, docs_section, enums) referenced_enums, tools_specs = build_tools_specs(tools, docs_section, enums)
reference_mdx = build_reference_mdx(toolkit_name, referenced_enums) if referenced_enums else "" reference_mdx = build_reference_mdx(toolkit_name, referenced_enums) if referenced_enums else ""
return reference_mdx, toolkit_page_template.format( toolkit_mdx = toolkit_page_template.format(
header=header, header=header,
table_of_contents=table_of_contents, table_of_contents=table_of_contents,
tools_specs=tools_specs, tools_specs=tools_specs,
reference_mdx=reference_mdx,
footer=footer, footer=footer,
) )
return toolkit_mdx.strip()
def build_reference_mdx( def build_reference_mdx(
toolkit_name: str, toolkit_name: str,
@ -327,7 +322,8 @@ def build_examples(
examples = [] examples = []
for tool in tools: for tool in tools:
print_debug(f"Generating tool-call examples for {tool.name}") print_debug(f"Generating tool-call examples for {tool.name}")
input_map = generate_tool_input_map(tool, openai_model) interface_signature = build_tool_interface_signature(tool)
input_map = generate_tool_input_map(interface_signature, openai_model)
fully_qualified_name = tool.fully_qualified_name.split("@")[0] fully_qualified_name = tool.fully_qualified_name.split("@")[0]
examples.append(( examples.append((
f"{pascal_to_snake_case(tool.name)}_example_call_tool.py", f"{pascal_to_snake_case(tool.name)}_example_call_tool.py",
@ -375,134 +371,131 @@ def generate_toolkit_description(
tools: list[tuple[str, str]], tools: list[tuple[str, str]],
openai_model: str, openai_model: str,
) -> str: ) -> str:
response = openai.chat.completions.create( messages = [
model=openai_model, {
messages=[ "role": "system",
{ "content": (
"role": "system", "You are a helpful assistant. "
"content": ( "When given a toolkit name and a list of tools, you will generate a "
"You are a helpful assistant. " "short, yet descriptive of the toolkit and the main actions a user "
"When given a toolkit name and a list of tools, you will generate a " "or LLM can perform with it.\n\n"
"short, yet descriptive of the toolkit and the main actions a user " "As an example, here is the Asana toolkit description:\n\n"
"or LLM can perform with it.\n\n" "The Arcade Asana toolkit provides a pre-built set of tools for "
"As an example, here is the Asana toolkit description:\n\n" "interacting with Asana. These tools make it easy to build agents "
"The Arcade Asana toolkit provides a pre-built set of tools for " "and AI apps that can:\n\n"
"interacting with Asana. These tools make it easy to build agents " "- Manage teams, projects, and workspaces.\n"
"and AI apps that can:\n\n" "- Create, update, and search for tasks.\n"
"- Manage teams, projects, and workspaces.\n" "- Retrieve data about tasks, projects, workspaces, users, etc.\n"
"- Create, update, and search for tasks.\n" "- Manage task attachments.\n\n"
"- Retrieve data about tasks, projects, workspaces, users, etc.\n" "And here is a JSON string with the list of tools in the Asana toolkit:\n\n"
"- Manage task attachments.\n\n" "```json\n\n"
"And here is a JSON string with the list of tools in the Asana toolkit:\n\n" '[["AttachFileToTask", "Attaches a file to an Asana task\n\nProvide exactly '
"```json\n\n" "one of file_content_str, file_content_base64, or file_content_url, never "
'[["AttachFileToTask", "Attaches a file to an Asana task\n\nProvide exactly ' "more\nthan one.\n\n- Use file_content_str for text files (will be encoded "
"one of file_content_str, file_content_base64, or file_content_url, never " "using file_encoding)\n- Use file_content_base64 for binary files like images, "
"more\nthan one.\n\n- Use file_content_str for text files (will be encoded " 'PDFs, etc.\n- Use file_content_url if the file is hosted on an external URL"], '
"using file_encoding)\n- Use file_content_base64 for binary files like images, " '["CreateTag", "Create a tag in Asana"], ["CreateTask", "Creates a task in '
'PDFs, etc.\n- Use file_content_url if the file is hosted on an external URL"], ' "Asana\n\nThe task must be associated to at least one of the following: "
'["CreateTag", "Create a tag in Asana"], ["CreateTask", "Creates a task in ' "parent_task_id, project, or\nworkspace_id. If none of these are provided and "
"Asana\n\nThe task must be associated to at least one of the following: " "the account has only one workspace, the task\nwill be associated to that "
"parent_task_id, project, or\nworkspace_id. If none of these are provided and " "workspace. If the account has multiple workspaces, an error will\nbe raised "
"the account has only one workspace, the task\nwill be associated to that " 'with a list of available workspaces."], ["GetProjectById", "Get an Asana '
"workspace. If the account has multiple workspaces, an error will\nbe raised " 'project by its ID"], ["GetSubtasksFromATask", "Get the subtasks of a task"], '
'with a list of available workspaces."], ["GetProjectById", "Get an Asana ' '["GetTagById", "Get an Asana tag by its ID"], ["GetTaskById", "Get a task by '
'project by its ID"], ["GetSubtasksFromATask", "Get the subtasks of a task"], ' 'its ID"], ["GetTasksWithoutId", "Search for tasks"], ["GetTeamById", "Get an '
'["GetTagById", "Get an Asana tag by its ID"], ["GetTaskById", "Get a task by ' 'Asana team by its ID"], ["GetUserById", "Get a user by ID"], ["GetWorkspaceById", '
'its ID"], ["GetTasksWithoutId", "Search for tasks"], ["GetTeamById", "Get an ' '"Get an Asana workspace by its ID"], ["ListProjects", "List projects in Asana"], '
'Asana team by its ID"], ["GetUserById", "Get a user by ID"], ["GetWorkspaceById", ' '["ListTags", "List tags in an Asana workspace"], ["ListTeams", "List teams in '
'"Get an Asana workspace by its ID"], ["ListProjects", "List projects in Asana"], ' 'an Asana workspace"], ["ListTeamsTheCurrentUserIsAMemberOf", "List teams in '
'["ListTags", "List tags in an Asana workspace"], ["ListTeams", "List teams in ' 'Asana that the current user is a member of"], ["ListUsers", "List users in '
'an Asana workspace"], ["ListTeamsTheCurrentUserIsAMemberOf", "List teams in ' 'Asana"], ["ListWorkspaces", "List workspaces in Asana that are visible to the '
'Asana that the current user is a member of"], ["ListUsers", "List users in ' 'authenticated user"], ["MarkTaskAsCompleted", "Mark a task in Asana as '
'Asana"], ["ListWorkspaces", "List workspaces in Asana that are visible to the ' 'completed"], ["UpdateTask", "Updates a task in Asana"]]\n\n```\n\n'
'authenticated user"], ["MarkTaskAsCompleted", "Mark a task in Asana as ' "Keep the description concise and to the point. The user will provide you with "
'completed"], ["UpdateTask", "Updates a task in Asana"]]\n\n```\n\n' "the toolkit name and the list of tools. Generate the description according to "
"Keep the description concise and to the point. The user will provide you with " "the instructions above."
"the toolkit name and the list of tools. Generate the description according to " ),
"the instructions above." },
), {
}, "role": "user",
{ "content": (
"role": "user", f"The toolkit name is {toolkit_name} and the list of tools is:\n\n"
"content": ( "```json\n\n"
f"The toolkit name is {toolkit_name} and the list of tools is:\n\n" f"{json.dumps(tools, ensure_ascii=False)}\n\n"
"```json\n\n" "```\n\n"
f"{json.dumps(tools, ensure_ascii=False)}\n\n" "Please generate a description for the toolkit."
"```\n\n" ),
"Please generate a description for the toolkit." },
), ]
},
],
temperature=0.0,
max_tokens=2048,
)
response_str = cast(str, response.choices[0].message.content) return request_openai_generation(model=openai_model, max_tokens=512, messages=messages)
return response_str.strip()
def generate_tool_input_map( def generate_tool_input_map(
tool: ToolDefinition, interface_signature: dict[str, Any],
openai_model: str, openai_model: str,
retries: int = 0, retries: int = 0,
max_retries: int = 3, max_retries: int = 3,
) -> dict[str, Any]: ) -> dict[str, Any]:
interface_signature = build_tool_interface_signature(tool) messages = [
response = openai.chat.completions.create( {
model=openai_model, "role": "system",
messages=[ "content": (
{ "You are a helpful assistant expert in generating data for documenting "
"role": "system", "sample scripts to calling tools. A tool is a function that is used in "
"content": ( "context of LLM tool-calling / function-calling.\n\n"
"You are a helpful assistant expert in generating data for documenting " "When given a tool signature with typed arguments, "
"sample scripts to calling tools. A tool is a function that is used in " "you must return exactly one JSON object (no markdown, no extra text) "
"context of LLM tool-calling / function-calling.\n\n" "where each key is an argument name, and each value is a sample value "
"When given a tool signature with typed arguments, " "for that argument that would make sense in a sample script to showcase "
"you must return exactly one JSON object (no markdown, no extra text) " "human software engineers how the tool may be called. Generate the "
"where each key is an argument name, and each value is a sample value " "argument sample value based on its name and description\n\n"
"for that argument that would make sense in a sample script to showcase " "Not every single argument must always be present in the input map. "
"human software engineers how the tool may be called. Generate the " "In some cases, the tool may require only one of two arguments to be "
"argument sample value based on its name and description\n\n" "provided, for example. In such cases, an indication will be present "
"Not every single argument must always be present in the input map. " "either/or in the tool description or the argument description. "
"In some cases, the tool may require only one of two arguments to be " "Always follow such instructions when present in the tool interface.\n\n"
"provided, for example. In such cases, an indication will be present " "Keep argument values as short as possible. Values don't have to always "
"either/or in the tool description or the argument description. " "be valid. For instance, for file content base64-encoded arguments, "
"Always follow such instructions when present in the tool interface.\n\n" "you can use a short text or a placeholder like `[file_content]`, it is "
"Keep argument values as short as possible. Values don't have to always " "not necessary that the value is a valid base64-encoded string.\n\n"
"be valid. For instance, for file content base64-encoded arguments, " "Remember that you MUST RESPOND ONLY WITH A VALID JSON STRING, NO ADDED "
"you can use a short text or a placeholder like `[file_content]`, it is " "TEXT. Your response will be json.load'ed, so it must be a valid JSON "
"not necessary that the value is a valid base64-encoded string.\n\n" "string."
"Remember that you MUST RESPOND ONLY WITH A VALID JSON STRING, NO ADDED " ),
"TEXT. Your response will be json.load'ed, so it must be a valid JSON " },
"string." {
), "role": "user",
}, "content": (
{ "Here is a tool interface:\n\n"
"role": "user", f"{json.dumps(interface_signature, ensure_ascii=False)}\n\n"
"content": ( "Please provide a sample input map as a JSON object."
"Here is a tool interface:\n\n" ),
f"{interface_signature}\n\n" },
"Please provide a sample input map as a JSON object." ]
),
},
],
temperature=0.0,
max_tokens=1024,
stop=["\n\n"],
)
response_str = cast(str, response.choices[0].message.content) text = request_openai_generation(model=openai_model, max_tokens=512, messages=messages)
text = response_str.strip()
try: try:
return cast(dict[str, Any], json.loads(text)) return cast(dict[str, Any], json.loads(text))
except json.JSONDecodeError: except (json.JSONDecodeError, TypeError):
if retries < max_retries: if retries < max_retries:
return generate_tool_input_map(tool, openai_model, retries + 1, max_retries) return generate_tool_input_map(
raise ValueError(f"Failed to generate input map for tool {tool.name}: {text}") interface_signature=interface_signature,
openai_model=openai_model,
retries=retries + 1,
max_retries=max_retries,
)
tool_name = interface_signature["tool_name"]
console.print(
f"Attention: {openai_model} failed to generate a valid inputs JSON for the tool '{tool_name}'. "
"Please check the Python & Javascript example scripts generated and enter a sample input manually.",
style="red",
)
return {}
def build_tool_interface_signature(tool: ToolDefinition) -> str: def build_tool_interface_signature(tool: ToolDefinition) -> dict[str, Any]:
args = [] args = []
for arg in tool.input.parameters: for arg in tool.input.parameters:
data: dict[str, Any] = { data: dict[str, Any] = {
@ -519,8 +512,45 @@ def build_tool_interface_signature(tool: ToolDefinition) -> str:
args.append(data) args.append(data)
return json.dumps({ return {
"tool_name": tool.name, "tool_name": tool.name,
"tool_description": tool.description, "tool_description": tool.description,
"tool_args": args, "tool_args": args,
}) }
def request_openai_generation(
model: str,
max_tokens: int,
messages: list[dict[str, Any]],
) -> str:
if model.startswith("gpt-5"):
response = openai.responses.create(
model=model,
input=messages,
max_output_tokens=max_tokens,
reasoning={
"effort": "minimal",
},
text={
"verbosity": "low",
},
)
response_str = cast(str, response.output_text)
elif model.startswith("gpt-4o"):
response = openai.chat.completions.create(
model=model,
messages=messages,
temperature=0.0,
max_completion_tokens=max_tokens,
stop=["\n\n"],
)
response_str = cast(str, response.choices[0].message.content)
else:
raise ValueError(
f"Unsupported OpenAI model: {model}. Choose a model from the 'gpt-4o' or 'gpt-5' series."
)
return response_str.strip()

6
libs/arcade-cli/arcade_cli/toolkit_docs/templates.py
View file

@ -3,7 +3,7 @@ TOOLKIT_PAGE = """{header}
{table_of_contents} {table_of_contents}
{tools_specs} {tools_specs}
{reference_mdx}
{footer} {footer}
""" """
@ -143,9 +143,9 @@ response = client.tools.execute(
print(json.dumps(response.output.value, indent=2)) print(json.dumps(response.output.value, indent=2))
""" """
ENUM_MDX = """# {toolkit_name} Reference ENUM_MDX = """## Reference
Below is a reference of enumerations used by some tools in the {toolkit_name} toolkit: Below is a reference of enumerations used by some of the tools in the {toolkit_name} toolkit:
{enum_items} {enum_items}
""" """