Track whether a tool call event happened (#661)

We are now tracking whether a tool call event happens. We track generic
"failure reasons" if the tool call fails. We DO NOT track names of
tools, tool parameters, or any PII.

Event name: 
- MCP tool called

Properties:
- is_execution_success
- failure_reason - one of "missing requirements", "transport
restriction", "error during tool execution", "unknown tool", "internal
error calling tool" or doesn't exist in the case of successful tool
execution.
- arcade_mcp_server_version
- runtime_language
- os_type
- os_release
- device_timestamp

As always you can opt out via setting the `ARCADE_USAGE_TRACKING`
environment variable to 0.

libs/arcade-mcp-server/arcade_mcp_server/server.py

@@ -74,6 +74,7 @@ from arcade_mcp_server.types import (
     TextResourceContents,
     UnsubscribeRequest,
 )
+from arcade_mcp_server.usage import ServerTracker
 
 logger = logging.getLogger("arcade.mcp")
 
@@ -207,6 +208,9 @@ class MCPServer:
         self._sessions: dict[str, ServerSession] = {}
         self._sessions_lock = asyncio.Lock()
 
+        # Usage tracking
+        self._tracker = ServerTracker()
+
         # Handler registration
         self._handlers = self._register_handlers()
 
@@ -666,10 +670,18 @@ class MCPServer:
             # Create tool context
             tool_context = await self._create_tool_context(tool, session)
 
+            # Check restrictions for unauthenticated HTTP transport
+            if transport_restriction_response := self._check_transport_restrictions(
+                tool, tool_context, message, tool_name, session
+            ):
+                self._tracker.track_tool_call(False, "transport restriction")
+                return transport_restriction_response
+
             # Handle authorization and secrets requirements if required
             if missing_requirements_response := await self._check_tool_requirements(
                 tool, tool_context, message, tool_name, session
             ):
+                self._tracker.track_tool_call(False, "missing requirements")
                 return missing_requirements_response
 
             # Attach tool_context to current model context for this request
@@ -709,6 +721,7 @@ class MCPServer:
                 # structuredContent should be the raw result value as a JSON object
                 structured_content = convert_content_to_structured_content(result.value)
 
+                self._tracker.track_tool_call(True)
                 return JSONRPCResponse(
                     id=message.id,
                     result=CallToolResult(
@@ -724,6 +737,7 @@ class MCPServer:
                 # structuredContent should be the error as a JSON object
                 structured_content = convert_content_to_structured_content({"error": str(error)})
 
+                self._tracker.track_tool_call(False, "error during tool execution")
                 return JSONRPCResponse(
                     id=message.id,
                     result=CallToolResult(
@@ -740,6 +754,7 @@ class MCPServer:
             # structuredContent should be the error as a JSON object
             structured_content = convert_content_to_structured_content({"error": error_message})
 
+            self._tracker.track_tool_call(False, "unknown tool")
             return JSONRPCResponse(
                 id=message.id,
                 result=CallToolResult(
@@ -750,6 +765,7 @@ class MCPServer:
             )
         except Exception:
             logger.exception("Error calling tool")
+            self._tracker.track_tool_call(False, "internal error calling tool")
             return JSONRPCError(
                 id=message.id,
                 error={"code": -32603, "message": "Internal error calling tool"},
@@ -770,7 +786,7 @@ class MCPServer:
             ),
         )
 
-    async def _check_tool_requirements(
+    def _check_transport_restrictions(
         self,
         tool: MaterializedTool,
         tool_context: ToolContext,
@@ -778,7 +794,7 @@ class MCPServer:
         tool_name: str,
         session: ServerSession | None = None,
     ) -> JSONRPCResponse[CallToolResult] | None:
-        """Check tool requirements before executing the tool"""
+        """Check transport restrictions for tools requiring auth or secrets"""
         # Check transport restrictions for tools requiring auth or secrets
         if session and session.init_options:
             transport_type = session.init_options.get("transport_type")
@@ -799,7 +815,17 @@ class MCPServer:
                         ),
                     }
                     return self._create_error_response(message, tool_response)
+        return None
 
+    async def _check_tool_requirements(
+        self,
+        tool: MaterializedTool,
+        tool_context: ToolContext,
+        message: CallToolRequest,
+        tool_name: str,
+        session: ServerSession | None = None,
+    ) -> JSONRPCResponse[CallToolResult] | None:
+        """Check tool requirements before executing the tool"""
         # Check authorization
         if tool.definition.requirements and tool.definition.requirements.authorization:
             # First check if Arcade API key is configured

libs/arcade-mcp-server/arcade_mcp_server/usage/constants.py

@@ -1,5 +1,6 @@
 # MCP Server Specific Event Names
 EVENT_MCP_SERVER_STARTED = "MCP server started"
+EVENT_MCP_TOOL_CALLED = "MCP tool called"
 
 # MCP Server Specific Property Names
 PROP_TRANSPORT = "transport"
@@ -7,3 +8,5 @@ PROP_HOST = "host"
 PROP_PORT = "port"
 PROP_TOOL_COUNT = "tool_count"
 PROP_MCP_SERVER_VERSION = "arcade_mcp_server_version"
+PROP_IS_EXECUTION_SUCCESS = "is_execution_success"
+PROP_FAILURE_REASON = "failure_reason"

libs/arcade-mcp-server/arcade_mcp_server/usage/server_tracker.py

@@ -14,7 +14,10 @@ from arcade_core.usage.constants import (
 
 from arcade_mcp_server.usage.constants import (
     EVENT_MCP_SERVER_STARTED,
+    EVENT_MCP_TOOL_CALLED,
+    PROP_FAILURE_REASON,
     PROP_HOST,
+    PROP_IS_EXECUTION_SUCCESS,
     PROP_MCP_SERVER_VERSION,
     PROP_PORT,
     PROP_TOOL_COUNT,
@@ -107,3 +110,33 @@ class ServerTracker:
         self.usage_service.capture(
             EVENT_MCP_SERVER_STARTED, self.user_id, properties=properties, is_anon=is_anon
         )
+
+    def track_tool_call(
+        self,
+        success: bool,
+        failure_reason: str | None = None,
+    ) -> None:
+        """Track MCP tool call event.
+
+        Args:
+            success: Whether the tool call succeeded (True) or failed (False)
+            reason: The reason for the failure (if any)
+        """
+        if not is_tracking_enabled():
+            return
+
+        properties: dict[str, str | int | float | bool | None] = {
+            PROP_IS_EXECUTION_SUCCESS: success,
+            PROP_FAILURE_REASON: failure_reason if not success else None,
+            PROP_MCP_SERVER_VERSION: self.mcp_server_version,
+            PROP_RUNTIME_LANGUAGE: "python",
+            PROP_RUNTIME_VERSION: self.runtime_version,
+            PROP_OS_TYPE: platform.system(),
+            PROP_OS_RELEASE: platform.release(),
+            PROP_DEVICE_TIMESTAMP: time.monotonic(),
+        }
+
+        is_anon = self.user_id == self.identity.anon_id
+        self.usage_service.capture(
+            EVENT_MCP_TOOL_CALLED, self.user_id, properties=properties, is_anon=is_anon
+        )

libs/arcade-mcp-server/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "arcade-mcp-server"
-version = "1.5.0"
+version = "1.6.0"
 description = "Model Context Protocol (MCP) server framework for Arcade.dev"
 readme = "README.md"
 authors = [{ name = "Arcade.dev" }]