arcade docs: add support for GPT-5 series; improve error handling (#529)

Adds support for GPT-5 series of models in `arcade docs`.

Improves error handling when the LLM does not generate a valid JSON for
a given tool sample inputs. Instead of raising an exception, the CLI
uses an empty input, moves on to the next tool, and prints a warning
message asking the user the fill in the input sample manually in
Javascript and Python files.

This PR also moves the Enumerations from a separate `reference.mdx` file
to the main toolkit file, as requested by @EricGustin to simplify the
docs structure.

libs/arcade-cli/arcade_cli/main.py

@@ -820,13 +820,12 @@ def docs(
         ),
     ),
     openai_model: str = typer.Option(
-        "gpt-4o-mini",
+        "gpt-5-mini",
         "--openai-model",
         "-m",
         help=(
             "A few parts of the documentation are generated using OpenAI API. "
-            "This argument controls which OpenAI model to use. "
+            "Choose one of the 'gpt-4o' and 'gpt-5' series models."
-            "E.g. 'gpt-4o', 'gpt-4o-mini'."
         ),
         show_default=True,
     ),
@@ -845,6 +844,14 @@ def docs(
     ),
     debug: bool = typer.Option(False, "--debug", "-d", help="Show debug information"),
 ) -> None:
+    if not openai_model.startswith("gpt-4o") and not openai_model.startswith("gpt-5"):
+        console.print(
+            f"Attention: '{openai_model}' is not a valid OpenAI model. "
+            "Please choose one of the 'gpt-4o' and 'gpt-5' series models.",
+            style="bold red",
+        )
+        raise typer.Exit()
+
     try:
         success = generate_toolkit_docs(
             console=console,

libs/arcade-cli/arcade_cli/toolkit_docs/__init__.py

@@ -6,7 +6,6 @@ from rich.console import Console
 from arcade_cli.toolkit_docs.docs_builder import (
     build_example_path,
     build_examples,
-    build_reference_mdx_path,
     build_toolkit_mdx,
     build_toolkit_mdx_path,
 )
@@ -58,8 +57,7 @@ def generate_toolkit_docs(
     enums = get_all_enumerations(toolkit_dir)
 
     print_debug(f"Building /{toolkit_name.lower()}.mdx file")
-    reference_mdx, toolkit_mdx = build_toolkit_mdx(
+    toolkit_mdx = build_toolkit_mdx(
-        toolkit_dir=toolkit_dir,
         tools=tools,
         docs_section=docs_section,
         enums=enums,
@@ -69,13 +67,6 @@ def generate_toolkit_docs(
     toolkit_mdx_path = build_toolkit_mdx_path(docs_section, docs_dir, toolkit_name)
     write_file(toolkit_mdx_path, toolkit_mdx)
 
-    if reference_mdx:
-        print_debug(f"Building /{toolkit_name.lower()}/reference.mdx file")
-        reference_mdx_path = build_reference_mdx_path(docs_section, docs_dir, toolkit_name)
-        write_file(reference_mdx_path, reference_mdx)
-    else:
-        print_debug("No Enums referenced by tool interfaces. Skipping reference.mdx file")
-
     if tool_call_examples:
         print_debug("Building tool-call examples in Python and JavaScript")
         examples = build_examples(print_debug, tools, openai_model)

libs/arcade-cli/arcade_cli/toolkit_docs/docs_builder.py

@@ -12,6 +12,7 @@ from arcade_core.schema import (
     ToolInput,
     ToolSecretRequirement,
 )
+from rich.console import Console
 
 from arcade_cli.toolkit_docs.templates import (
     ENUM_ITEM,
@@ -40,6 +41,8 @@ from arcade_cli.toolkit_docs.utils import (
     pascal_to_snake_case,
 )
 
+console = Console()
+
 
 def build_toolkit_mdx_path(docs_section: str, docs_root_dir: str, toolkit_name: str) -> str:
     return os.path.join(
@@ -51,17 +54,6 @@ def build_toolkit_mdx_path(docs_section: str, docs_root_dir: str, toolkit_name:
     )
 
 
-def build_reference_mdx_path(docs_section: str, docs_root_dir: str, toolkit_name: str) -> str:
-    return os.path.join(
-        docs_root_dir,
-        "pages",
-        "toolkits",
-        docs_section,
-        toolkit_name.lower(),
-        "reference.mdx",
-    )
-
-
 def build_example_path(example_filename: str, docs_root_dir: str, toolkit_name: str) -> str:
     return os.path.join(
         docs_root_dir,
@@ -75,7 +67,6 @@ def build_example_path(example_filename: str, docs_root_dir: str, toolkit_name:
 
 
 def build_toolkit_mdx(
-    toolkit_dir: str,
     tools: list[ToolDefinition],
     docs_section: str,
     enums: dict[str, type[Enum]],
@@ -102,16 +93,20 @@ def build_toolkit_mdx(
     )
     table_of_contents = build_table_of_contents(tools)
     footer = build_footer(toolkit_name, pip_package_name, sample_tool.requirements.authorization)
+
     referenced_enums, tools_specs = build_tools_specs(tools, docs_section, enums)
     reference_mdx = build_reference_mdx(toolkit_name, referenced_enums) if referenced_enums else ""
 
-    return reference_mdx, toolkit_page_template.format(
+    toolkit_mdx = toolkit_page_template.format(
         header=header,
         table_of_contents=table_of_contents,
         tools_specs=tools_specs,
+        reference_mdx=reference_mdx,
         footer=footer,
     )
 
+    return toolkit_mdx.strip()
+
 
 def build_reference_mdx(
     toolkit_name: str,
@@ -327,7 +322,8 @@ def build_examples(
     examples = []
     for tool in tools:
         print_debug(f"Generating tool-call examples for {tool.name}")
-        input_map = generate_tool_input_map(tool, openai_model)
+        interface_signature = build_tool_interface_signature(tool)
+        input_map = generate_tool_input_map(interface_signature, openai_model)
         fully_qualified_name = tool.fully_qualified_name.split("@")[0]
         examples.append((
             f"{pascal_to_snake_case(tool.name)}_example_call_tool.py",
@@ -375,8 +371,6 @@ def generate_toolkit_description(
     tools: list[tuple[str, str]],
     openai_model: str,
 ) -> str:
-    response = openai.chat.completions.create(
-        model=openai_model,
     messages = [
         {
             "role": "system",
@@ -432,24 +426,17 @@ def generate_toolkit_description(
                 "Please generate a description for the toolkit."
             ),
         },
-        ],
+    ]
-        temperature=0.0,
-        max_tokens=2048,
-    )
 
-    response_str = cast(str, response.choices[0].message.content)
+    return request_openai_generation(model=openai_model, max_tokens=512, messages=messages)
-    return response_str.strip()
 
 
 def generate_tool_input_map(
-    tool: ToolDefinition,
+    interface_signature: dict[str, Any],
     openai_model: str,
     retries: int = 0,
     max_retries: int = 3,
 ) -> dict[str, Any]:
-    interface_signature = build_tool_interface_signature(tool)
-    response = openai.chat.completions.create(
-        model=openai_model,
     messages = [
         {
             "role": "system",
@@ -481,28 +468,34 @@ def generate_tool_input_map(
             "role": "user",
             "content": (
                 "Here is a tool interface:\n\n"
-                    f"{interface_signature}\n\n"
+                f"{json.dumps(interface_signature, ensure_ascii=False)}\n\n"
                 "Please provide a sample input map as a JSON object."
             ),
         },
-        ],
+    ]
-        temperature=0.0,
-        max_tokens=1024,
-        stop=["\n\n"],
-    )
 
-    response_str = cast(str, response.choices[0].message.content)
+    text = request_openai_generation(model=openai_model, max_tokens=512, messages=messages)
-    text = response_str.strip()
 
     try:
         return cast(dict[str, Any], json.loads(text))
-    except json.JSONDecodeError:
+    except (json.JSONDecodeError, TypeError):
         if retries < max_retries:
-            return generate_tool_input_map(tool, openai_model, retries + 1, max_retries)
+            return generate_tool_input_map(
-        raise ValueError(f"Failed to generate input map for tool {tool.name}: {text}")
+                interface_signature=interface_signature,
+                openai_model=openai_model,
+                retries=retries + 1,
+                max_retries=max_retries,
+            )
+        tool_name = interface_signature["tool_name"]
+        console.print(
+            f"Attention: {openai_model} failed to generate a valid inputs JSON for the tool '{tool_name}'. "
+            "Please check the Python & Javascript example scripts generated and enter a sample input manually.",
+            style="red",
+        )
+        return {}
 
 
-def build_tool_interface_signature(tool: ToolDefinition) -> str:
+def build_tool_interface_signature(tool: ToolDefinition) -> dict[str, Any]:
     args = []
     for arg in tool.input.parameters:
         data: dict[str, Any] = {
@@ -519,8 +512,45 @@ def build_tool_interface_signature(tool: ToolDefinition) -> str:
 
         args.append(data)
 
-    return json.dumps({
+    return {
         "tool_name": tool.name,
         "tool_description": tool.description,
         "tool_args": args,
-    })
+    }
+
+
+def request_openai_generation(
+    model: str,
+    max_tokens: int,
+    messages: list[dict[str, Any]],
+) -> str:
+    if model.startswith("gpt-5"):
+        response = openai.responses.create(
+            model=model,
+            input=messages,
+            max_output_tokens=max_tokens,
+            reasoning={
+                "effort": "minimal",
+            },
+            text={
+                "verbosity": "low",
+            },
+        )
+        response_str = cast(str, response.output_text)
+
+    elif model.startswith("gpt-4o"):
+        response = openai.chat.completions.create(
+            model=model,
+            messages=messages,
+            temperature=0.0,
+            max_completion_tokens=max_tokens,
+            stop=["\n\n"],
+        )
+        response_str = cast(str, response.choices[0].message.content)
+
+    else:
+        raise ValueError(
+            f"Unsupported OpenAI model: {model}. Choose a model from the 'gpt-4o' or 'gpt-5' series."
+        )
+
+    return response_str.strip()

libs/arcade-cli/arcade_cli/toolkit_docs/templates.py

@@ -3,7 +3,7 @@ TOOLKIT_PAGE = """{header}
 {table_of_contents}
 
 {tools_specs}
-
+{reference_mdx}
 {footer}
 """
 
@@ -143,9 +143,9 @@ response = client.tools.execute(
 print(json.dumps(response.output.value, indent=2))
 """
 
-ENUM_MDX = """# {toolkit_name} Reference
+ENUM_MDX = """## Reference
 
-Below is a reference of enumerations used by some tools in the {toolkit_name} toolkit:
+Below is a reference of enumerations used by some of the tools in the {toolkit_name} toolkit:
 
 {enum_items}
 """