diff --git a/libs/arcade-cli/arcade_cli/toolkit_docs/__init__.py b/libs/arcade-cli/arcade_cli/toolkit_docs/__init__.py index c0452ca4..b13ab96b 100644 --- a/libs/arcade-cli/arcade_cli/toolkit_docs/__init__.py +++ b/libs/arcade-cli/arcade_cli/toolkit_docs/__init__.py @@ -7,11 +7,12 @@ from arcade_cli.toolkit_docs.docs_builder import ( build_example_path, build_examples, build_toolkit_mdx, - build_toolkit_mdx_path, + build_toolkit_mdx_file_path, ) from arcade_cli.toolkit_docs.utils import ( get_all_enumerations, get_list_of_tools, + has_wrapper_tools_directory, print_debug_func, read_toolkit_metadata, resolve_api_key, @@ -44,8 +45,9 @@ def generate_toolkit_docs( docs_dir = standardize_dir_path(docs_dir) toolkit_dir = standardize_dir_path(toolkit_dir) + is_wrapper_toolkit = has_wrapper_tools_directory(toolkit_dir) - print_debug("Reading toolkit metadata") + print_debug("Reading server metadata") pip_package_name = read_toolkit_metadata(toolkit_dir) print_debug(f"Getting list of tools for {toolkit_name} from the local Python environment") @@ -56,16 +58,19 @@ def generate_toolkit_docs( print_debug("Getting all enumerations potentially used in tool argument specs") enums = get_all_enumerations(toolkit_dir) - print_debug(f"Building /{toolkit_name.lower()}.mdx file") + toolkit_mdx_file_path = build_toolkit_mdx_file_path(docs_section, docs_dir, toolkit_name) + print_debug(f"Building {toolkit_mdx_file_path} file") toolkit_mdx = build_toolkit_mdx( + toolkit_package_name=toolkit_name, + toolkit_dir=toolkit_dir, tools=tools, docs_section=docs_section, enums=enums, pip_package_name=pip_package_name, openai_model=openai_model, + is_wrapper_toolkit=is_wrapper_toolkit, ) - toolkit_mdx_path = build_toolkit_mdx_path(docs_section, docs_dir, toolkit_name) - write_file(toolkit_mdx_path, toolkit_mdx) + write_file(toolkit_mdx_file_path, toolkit_mdx) if tool_call_examples: print_debug("Building tool-call examples in Python and JavaScript") diff --git a/libs/arcade-cli/arcade_cli/toolkit_docs/docs_builder.py b/libs/arcade-cli/arcade_cli/toolkit_docs/docs_builder.py index 04e054ba..e337c199 100644 --- a/libs/arcade-cli/arcade_cli/toolkit_docs/docs_builder.py +++ b/libs/arcade-cli/arcade_cli/toolkit_docs/docs_builder.py @@ -19,6 +19,8 @@ from arcade_cli.toolkit_docs.templates import ( ENUM_MDX, ENUM_VALUE, GENERIC_PROVIDER_CONFIG, + STARTER_TOOL_INFO_CALL, + STARTER_TOOLKIT_HEADER_IMPORT, TABBED_EXAMPLES_LIST, TABLE_OF_CONTENTS, TABLE_OF_CONTENTS_ITEM, @@ -36,6 +38,8 @@ from arcade_cli.toolkit_docs.templates import ( from arcade_cli.toolkit_docs.utils import ( clean_fully_qualified_name, find_enum_by_options, + find_pyproject_toml, + get_pyproject_description, get_toolkit_auth_type, is_well_known_provider, pascal_to_snake_case, @@ -44,15 +48,30 @@ from arcade_cli.toolkit_docs.utils import ( console = Console() -def build_toolkit_mdx_path(docs_section: str, docs_root_dir: str, toolkit_name: str) -> str: - return os.path.join( +def build_toolkit_mdx_dir_path( + docs_section: str, + docs_root_dir: str, + toolkit_name: str, + ensure_exists: bool = True, +) -> str: + dir_path = os.path.join( docs_root_dir, - "pages", - "toolkits", + "app", + "en", + "mcp-servers", docs_section, - f"{toolkit_name.lower()}.mdx", + f"{toolkit_name.lower().replace('_', '-')}", ) + if ensure_exists: + os.makedirs(dir_path, exist_ok=True) + return dir_path + + +def build_toolkit_mdx_file_path(docs_section: str, docs_root_dir: str, toolkit_name: str) -> str: + toolkit_dir_path = build_toolkit_mdx_dir_path(docs_section, docs_root_dir, toolkit_name) + return os.path.join(toolkit_dir_path, "page.mdx") + def build_example_path(example_filename: str, docs_root_dir: str, toolkit_name: str) -> str: return os.path.join( @@ -60,13 +79,15 @@ def build_example_path(example_filename: str, docs_root_dir: str, toolkit_name: "public", "examples", "integrations", - "toolkits", + "mcp-servers", toolkit_name.lower(), example_filename, ) def build_toolkit_mdx( + toolkit_package_name: str, + toolkit_dir: str, tools: list[ToolDefinition], docs_section: str, enums: dict[str, type[Enum]], @@ -74,14 +95,32 @@ def build_toolkit_mdx( openai_model: str, toolkit_header_template: str = TOOLKIT_HEADER, toolkit_page_template: str = TOOLKIT_PAGE, + is_wrapper_toolkit: bool = False, ) -> tuple[str, str]: sample_tool = tools[0] toolkit_name = sample_tool.toolkit.name toolkit_version = sample_tool.toolkit.version - auth_type = get_toolkit_auth_type(sample_tool.requirements.authorization) + auth_type = get_toolkit_auth_type(sample_tool.requirements) + + if is_wrapper_toolkit: + starter_tool_info_import = STARTER_TOOLKIT_HEADER_IMPORT + starter_tool_info_warning = STARTER_TOOL_INFO_CALL.format(toolkit_name=toolkit_name) + else: + starter_tool_info_import = "" + starter_tool_info_warning = "" + + try: + pyproject_path = find_pyproject_toml(toolkit_dir) + tool_info_description = get_pyproject_description(pyproject_path) + + except ValueError: + tool_info_description = f"Enable Agents to interact with the {toolkit_name} MCP Server" header = toolkit_header_template.format( toolkit_title=toolkit_name, + tool_info_description=tool_info_description, + starter_tool_info_import=starter_tool_info_import, + starter_tool_info_warning=starter_tool_info_warning, description=generate_toolkit_description( toolkit_name, [(tool.name, tool.description) for tool in tools], @@ -94,7 +133,9 @@ def build_toolkit_mdx( table_of_contents = build_table_of_contents(tools) footer = build_footer(toolkit_name, pip_package_name, sample_tool.requirements.authorization) - referenced_enums, tools_specs = build_tools_specs(tools, docs_section, enums) + referenced_enums, tools_specs = build_tools_specs( + toolkit_package_name, tools, docs_section, enums + ) reference_mdx = build_reference_mdx(toolkit_name, referenced_enums) if referenced_enums else "" toolkit_mdx = toolkit_page_template.format( @@ -201,6 +242,7 @@ def build_footer( def build_tools_specs( + toolkit_name: str, tools: list[ToolDefinition], docs_section: str, enums: dict[str, type[Enum]], @@ -212,6 +254,7 @@ def build_tools_specs( referenced_enums = [] for tool in tools: tool_referenced_enums, tool_spec = build_tool_spec( + toolkit_name=toolkit_name, tool=tool, docs_section=docs_section, enums=enums, @@ -226,6 +269,7 @@ def build_tools_specs( def build_tool_spec( + toolkit_name: str, tool: ToolDefinition, docs_section: str, enums: dict[str, type[Enum]], @@ -234,7 +278,7 @@ def build_tool_spec( tool_spec_secrets_template: str = TOOL_SPEC_SECRETS, ) -> tuple[list[tuple[str, type[Enum]]], str]: tabbed_examples_list = TABBED_EXAMPLES_LIST.format( - toolkit_name=tool.toolkit.name.lower(), + toolkit_name=toolkit_name.lower(), tool_name=pascal_to_snake_case(tool.name), ) referenced_enums, parameters = build_tool_parameters( @@ -290,7 +334,7 @@ def build_tool_parameters( if schema.enum: enum_name, enum_class = find_enum_by_options(enums, schema.enum) referenced_enums.append((enum_name, enum_class)) - param_definition = f"`Enum` [{enum_name}](/toolkits/{docs_section}/{toolkit_name}/reference#{enum_name})" + param_definition = f"`Enum` [{enum_name}](/mcp-servers/{docs_section}/{toolkit_name}/reference#{enum_name})" else: if schema.inner_val_type: param_definition = f"`{schema.val_type}[{schema.inner_val_type}]`" @@ -325,12 +369,15 @@ def build_examples( interface_signature = build_tool_interface_signature(tool) input_map = generate_tool_input_map(interface_signature, openai_model) fully_qualified_name = tool.fully_qualified_name.split("@")[0] + + py_file_name = f"{pascal_to_snake_case(tool.name)}_example_call_tool.py" examples.append(( - f"{pascal_to_snake_case(tool.name)}_example_call_tool.py", + py_file_name, build_python_example(fully_qualified_name, input_map), )) + js_file_name = f"{pascal_to_snake_case(tool.name)}_example_call_tool.js" examples.append(( - f"{pascal_to_snake_case(tool.name)}_example_call_tool.js", + js_file_name, build_javascript_example(fully_qualified_name, input_map), )) return examples @@ -376,18 +423,18 @@ def generate_toolkit_description( "role": "system", "content": ( "You are a helpful assistant. " - "When given a toolkit name and a list of tools, you will generate a " - "short, yet descriptive of the toolkit and the main actions a user " + "When given an MCP Server name and a list of tools, you will generate a " + "short, yet descriptive of the MCP Server and the main actions a user " "or LLM can perform with it.\n\n" - "As an example, here is the Asana toolkit description:\n\n" - "The Arcade Asana toolkit provides a pre-built set of tools for " + "As an example, here is the Asana MCP Server description:\n\n" + "The Arcade Asana MCP Server provides a pre-built set of tools for " "interacting with Asana. These tools make it easy to build agents " "and AI apps that can:\n\n" "- Manage teams, projects, and workspaces.\n" "- Create, update, and search for tasks.\n" "- Retrieve data about tasks, projects, workspaces, users, etc.\n" "- Manage task attachments.\n\n" - "And here is a JSON string with the list of tools in the Asana toolkit:\n\n" + "And here is a JSON string with the list of tools in the Asana MCP Server:\n\n" "```json\n\n" '[["AttachFileToTask", "Attaches a file to an Asana task\n\nProvide exactly ' "one of file_content_str, file_content_base64, or file_content_url, never " @@ -412,18 +459,18 @@ def generate_toolkit_description( 'authenticated user"], ["MarkTaskAsCompleted", "Mark a task in Asana as ' 'completed"], ["UpdateTask", "Updates a task in Asana"]]\n\n```\n\n' "Keep the description concise and to the point. The user will provide you with " - "the toolkit name and the list of tools. Generate the description according to " + "the MCP Server name and the list of tools. Generate the description according to " "the instructions above." ), }, { "role": "user", "content": ( - f"The toolkit name is {toolkit_name} and the list of tools is:\n\n" + f"The MCP Server name is {toolkit_name} and the list of tools is:\n\n" "```json\n\n" f"{json.dumps(tools, ensure_ascii=False)}\n\n" "```\n\n" - "Please generate a description for the toolkit." + "Please generate a description for the MCP Server." ), }, ] diff --git a/libs/arcade-cli/arcade_cli/toolkit_docs/templates.py b/libs/arcade-cli/arcade_cli/toolkit_docs/templates.py index 49436347..399bc8d7 100644 --- a/libs/arcade-cli/arcade_cli/toolkit_docs/templates.py +++ b/libs/arcade-cli/arcade_cli/toolkit_docs/templates.py @@ -7,16 +7,21 @@ TOOLKIT_PAGE = """{header} {footer} """ -TOOLKIT_HEADER = """# {toolkit_title} +STARTER_TOOLKIT_HEADER_IMPORT = 'import StarterToolInfo from "@/app/_components/starter-tool-info";' -import ToolInfo from "@/components/ToolInfo"; -import Badges from "@/components/Badges"; -import TabbedCodeBlock from "@/components/TabbedCodeBlock"; -import TableOfContents from "@/components/TableOfContents"; -import ToolFooter from "@/components/ToolFooter"; +STARTER_TOOL_INFO_CALL = '' + +TOOLKIT_HEADER = """# {toolkit_title} +{starter_tool_info_import} +import ToolInfo from "@/app/_components/tool-info"; +import Badges from "@/app/_components/badges"; +import TabbedCodeBlock from "@/app/_components/tabbed-code-block"; +import TableOfContents from "@/app/_components/table-of-contents"; +import ToolFooter from "@/app/_components/tool-footer"; +import {{ Callout }} from "nextra/components"; +{starter_tool_info_warning} + {description}""" TABLE_OF_CONTENTS = """## Available Tools @@ -36,11 +43,11 @@ TABLE_OF_CONTENTS = """## Available Tools }} /> - + If you need to perform an action that's not listed here, you can [get in touch with us](mailto:contact@arcade.dev) to request a new tool, or [create your - own tools](/home/build-tools/create-a-toolkit). -""" + own tools](/home/build-tools/create-a-mcp-server). +""" TABLE_OF_CONTENTS_ITEM = '\n ["{tool_fully_qualified_name}", "{description}"],' @@ -59,7 +66,7 @@ TOOL_SPEC = """## {tool_fully_qualified_name} TOOL_SPEC_SECRETS = """**Secrets** -This tool requires the following secrets: {secrets} (learn how to [configure secrets](/home/build-tools/create-a-tool-with-secrets#supplying-the-secret)) +This tool requires the following secrets: {secrets} (learn how to [configure secrets](/home/build-tools/create-a-tool-with-secrets#set-the-secret-in-the-arcade-dashboard)) """ TABBED_EXAMPLES_LIST = """ """ -WELL_KNOWN_PROVIDER_CONFIG = "The Arcade {toolkit_name} toolkit uses the [{provider_name} auth provider](/home/auth-providers/{provider_id}) to connect to users' {toolkit_name} accounts. Please refer to the [{provider_name} auth provider](/home/auth-providers/{provider_id}) documentation to learn how to configure auth." +WELL_KNOWN_PROVIDER_CONFIG = "The Arcade {toolkit_name} MCP Server uses the [{provider_name} auth provider](/home/auth-providers/{provider_id}) to connect to users' {toolkit_name} accounts. Please refer to the [{provider_name} auth provider](/home/auth-providers/{provider_id}) documentation to learn how to configure auth." -GENERIC_PROVIDER_CONFIG = "The {toolkit_name} toolkit uses the Auth Provider with id `{provider_id}` to connect to users' {toolkit_name} accounts. In order to use the toolkit, you will need to configure the `{provider_id}` auth provider." +GENERIC_PROVIDER_CONFIG = "The {toolkit_name} MCP Server uses the Auth Provider with id `{provider_id}` to connect to users' {toolkit_name} accounts. In order to use the MCP Server, you will need to configure the `{provider_id}` auth provider." TOOL_CALL_EXAMPLE_JS = """import {{ Arcade }} from "@arcadeai/arcadejs"; @@ -148,7 +155,7 @@ print(json.dumps(response.output.value, indent=2)) ENUM_MDX = """## Reference -Below is a reference of enumerations used by some of the tools in the {toolkit_name} toolkit: +Below is a reference of enumerations used by some of the tools in the {toolkit_name} MCP Server: {enum_items} """ diff --git a/libs/arcade-cli/arcade_cli/toolkit_docs/utils.py b/libs/arcade-cli/arcade_cli/toolkit_docs/utils.py index a34b972d..b5d105d2 100644 --- a/libs/arcade-cli/arcade_cli/toolkit_docs/utils.py +++ b/libs/arcade-cli/arcade_cli/toolkit_docs/utils.py @@ -2,13 +2,19 @@ import importlib import inspect import os import re +import sys from enum import Enum from pathlib import Path from types import ModuleType +if sys.version_info >= (3, 11): + import tomllib +else: + tomllib = None + from arcade_core.auth import AuthProviderType from arcade_core.catalog import ToolCatalog -from arcade_core.schema import ToolAuthRequirement, ToolDefinition +from arcade_core.schema import ToolDefinition, ToolRequirements from rich.console import Console from arcade_cli.utils import discover_toolkits @@ -42,14 +48,22 @@ def write_file(path: str, content: str) -> None: def read_toolkit_metadata(toolkit_dir: str) -> str: pyproject_path = os.path.join(toolkit_dir, "pyproject.toml") - with open(pyproject_path) as f: - content = f.read() - project_section_match = re.search(r"\[project\](.*?)(?=\n\[|$)", content, re.DOTALL) - if project_section_match: - project_content = project_section_match.group(1) - name_match = re.search(r'name\s*=\s*["\']([^"\']+)["\']', project_content) - if name_match: - return name_match.group(1).strip() + + if tomllib is not None: + with open(pyproject_path, "rb") as f: + data = tomllib.load(f) + if "project" in data and "name" in data["project"]: + return data["project"]["name"] + else: + # Fallback to regex for Python < 3.11 + with open(pyproject_path) as f: + content = f.read() + project_section_match = re.search(r"\[project\](.*?)(?=\n\[|$)", content, re.DOTALL) + if project_section_match: + project_content = project_section_match.group(1) + name_match = re.search(r'name\s*=\s*["\']([^"\']+)["\']', project_content) + if name_match: + return name_match.group(1).strip() raise ValueError(f"Could not find package name in '{pyproject_path}'") @@ -113,14 +127,15 @@ def get_all_enumerations(toolkit_root_dir: str) -> dict[str, type[Enum]]: return enums -def get_toolkit_auth_type(requirement: ToolAuthRequirement | None) -> str: - if requirement is None: - return "" - elif requirement.provider_type == AuthProviderType.oauth2.value: - return 'authType="OAuth2"' - elif requirement.provider_type: - return f'authType="{requirement.provider_type}"' - return "" +def get_toolkit_auth_type(tool_req: ToolRequirements | None) -> str: + if tool_req.authorization: + if tool_req.authorization.provider_type == AuthProviderType.oauth2.value: + return 'authType="OAuth2"' + else: + return f'authType="{tool_req.authorization.provider_type}"' + elif tool_req.secrets: + return 'authType="API Key"' + return 'authType="None"' def find_enum_by_options( @@ -159,3 +174,47 @@ def is_well_known_provider( def clean_fully_qualified_name(fully_qualified_name: str) -> str: return fully_qualified_name.split("@")[0] + + +def has_wrapper_tools_directory(toolkit_package_path: str) -> bool: + has_dir = os.path.exists(os.path.join(toolkit_package_path, "wrapper_tools")) + if has_dir: + return True + + # Check one level deep + for dir_name in os.listdir(toolkit_package_path): + if os.path.exists(os.path.join(toolkit_package_path, dir_name, "wrapper_tools")): + return True + + return False + + +def find_pyproject_toml(toolkit_package_path: str) -> str: + for root, _, files in os.walk(toolkit_package_path): + for file in files: + if file == "pyproject.toml": + return os.path.join(root, file) + + raise ValueError(f"No pyproject.toml found in {toolkit_package_path}") + + +def get_pyproject_description(pyproject_path: str) -> str: + if tomllib is not None: + with open(pyproject_path, "rb") as f: + data = tomllib.load(f) + if "project" in data and "description" in data["project"]: + return data["project"]["description"] + else: + # Fallback to regex for Python < 3.11 + with open(pyproject_path) as f: + content = f.read() + project_section_match = re.search(r"\[project\](.*?)(?=\n\[|$)", content, re.DOTALL) + if project_section_match: + project_content = project_section_match.group(1) + description_match = re.search( + r'description\s*=\s*["\']([^"\']+)["\']', project_content + ) + if description_match: + return description_match.group(1).strip() + + raise ValueError(f"Could not find description in '{pyproject_path}'") diff --git a/libs/tests/cli/toolkit_docs/test_docs_builder_utils.py b/libs/tests/cli/toolkit_docs/test_docs_builder_utils.py index c1a641f7..c04cdb40 100644 --- a/libs/tests/cli/toolkit_docs/test_docs_builder_utils.py +++ b/libs/tests/cli/toolkit_docs/test_docs_builder_utils.py @@ -14,7 +14,10 @@ from arcade_core.schema import ToolAuthRequirement @patch("arcade_cli.toolkit_docs.utils.open") -def test_read_toolkit_metadata(mock_open): +@patch("arcade_cli.toolkit_docs.utils.tomllib") +def test_read_toolkit_metadata(mock_tomllib, mock_open): + from unittest.mock import MagicMock, mock_open as mock_open_func + sample_pyproject_toml = """ [build-system] requires = [ "hatchling",] @@ -74,13 +77,25 @@ skip_empty = true [tool.hatch.build.targets.wheel] packages = [ "arcade_jira",] """ - mock_open.return_value.__enter__.return_value.read.return_value = sample_pyproject_toml + + # Setup mock to handle both binary and text mode + def open_side_effect(path, mode="r"): + if mode == "rb": + return mock_open_func(read_data=sample_pyproject_toml.encode()).return_value + else: + return mock_open_func(read_data=sample_pyproject_toml).return_value + + mock_open.side_effect = open_side_effect + mock_tomllib.load.return_value = {"project": {"name": "arcade_jira"}} + assert read_toolkit_metadata("path/to/toolkits/jira") == "arcade_jira" - mock_open.assert_called_once_with("path/to/toolkits/jira/pyproject.toml") @patch("arcade_cli.toolkit_docs.utils.open") -def test_read_toolkit_metadata_missing_project_name(mock_open): +@patch("arcade_cli.toolkit_docs.utils.tomllib") +def test_read_toolkit_metadata_missing_project_name(mock_tomllib, mock_open): + from unittest.mock import mock_open as mock_open_func + sample_pyproject_toml = """ [build-system] requires = [ "hatchling",] @@ -98,9 +113,19 @@ dependencies = [ name = "Arcade" email = "dev@arcade.dev" """ - mock_open.return_value.__enter__.return_value.read.return_value = sample_pyproject_toml + + # Setup mock to handle both binary and text mode + def open_side_effect(path, mode="r"): + if mode == "rb": + return mock_open_func(read_data=sample_pyproject_toml.encode()).return_value + else: + return mock_open_func(read_data=sample_pyproject_toml).return_value + + mock_open.side_effect = open_side_effect + mock_tomllib.load.return_value = {"project": {}} # Missing "name" + with pytest.raises(ValueError): - print("\n\n\n", read_toolkit_metadata("path/to/toolkits/jira")) + read_toolkit_metadata("path/to/toolkits/jira") def test_pascal_to_snake_case(): @@ -109,18 +134,23 @@ def test_pascal_to_snake_case(): def test_get_toolkit_auth_type_none(): - assert get_toolkit_auth_type(requirement=None) == "" + from arcade_core.schema import ToolRequirements + + tool_req = ToolRequirements() + assert get_toolkit_auth_type(tool_req=tool_req) == 'authType="None"' def test_get_toolkit_auth_type_with_provider_type(): - requirement = ToolAuthRequirement(provider_type=AuthProviderType.oauth2.value) - assert get_toolkit_auth_type(requirement=requirement) == 'authType="OAuth2"' + from arcade_core.schema import ToolRequirements, ToolSecretRequirement - requirement = ToolAuthRequirement(provider_type="another_type") - assert get_toolkit_auth_type(requirement=requirement) == 'authType="another_type"' + tool_req = ToolRequirements(authorization=ToolAuthRequirement(provider_type=AuthProviderType.oauth2.value)) + assert get_toolkit_auth_type(tool_req=tool_req) == 'authType="OAuth2"' - requirement = ToolAuthRequirement(provider_type="") - assert get_toolkit_auth_type(requirement=requirement) == "" + tool_req = ToolRequirements(authorization=ToolAuthRequirement(provider_type="another_type")) + assert get_toolkit_auth_type(tool_req=tool_req) == 'authType="another_type"' + + tool_req = ToolRequirements(secrets=[ToolSecretRequirement(key="API_KEY")]) + assert get_toolkit_auth_type(tool_req=tool_req) == 'authType="API Key"' def test_is_well_known_provider_none(): diff --git a/pyproject.toml b/pyproject.toml index d63dd066..947f2f8f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "arcade-mcp" -version = "1.0.0rc2" +version = "1.0.0rc3" description = "Arcade.dev - Tool Calling platform for Agents" readme = "README.md" license = {file = "LICENSE"}