Serving multiple tools and builtins with data m

toolserve/toolserve/builtin/default/query.py

@@ -0,0 +1,119 @@
+from typing import Any, Dict, Optional
+import io
+
+from toolserve.sdk.client import list_data, log
+from toolserve.sdk.dataframe import get_df, save_df
+from toolserve.sdk.tool import tool, Param
+import duckdb
+import pandas as pd
+
+@tool
+async def list_data_sources() -> Dict[str, Dict[str, str]]:
+    """List all data sources.
+
+    Returns:
+        Dict[str, str]: A dictionary mapping data source IDs to their details.
+    """
+    data = await list_data()
+    return {str(item["id"]): {
+        "file_name": item["file_name"],
+        "created_at": item["created_time"],
+        "updated_at": item["updated_time"]
+    } for item in data}
+
+@tool
+async def get_data_schema(
+    data_id: Param(int, "id of the data source"),
+    ) -> Param(str, "schema of the data source"):
+    """Get the schema of the data source by id.
+
+    Args:
+        data_id (int): The id of the data source to get the schema of.
+
+    Returns:
+        str: The schema of the data source.
+    """
+    # TODO read in only a few lines
+    df = await get_df(data_id)
+    return get_df_info(df)
+
+
+@tool
+async def query_sql(
+    data_id: Param(int, "id of the data source"),
+    sql: Param(str, "parameterized SQL query to execute"),
+    params: Param(Optional[Dict[str, Any]], "parameters to pass to the SQL query") = None,
+    ) -> Param(str, "schema of the data source after executing the query"):
+    """Query a data source using SQL
+
+    The SQL query should be parameterized with Python's named parameter syntax,
+    e.g. `SELECT * FROM table WHERE column = :value`.
+
+    After the query, a new data source at a new id will be created with the results and
+    the schema of the data source will be returned.
+
+    Args:
+        data_id (int): The id of the data source to query.
+        sql (str): The parameterized SQL query to execute.
+        params (Optional[Dict[str, Any]]): Parameters to pass to the SQL query.
+
+    Returns:
+        str: The schema of the data source after executing the query.
+    """
+    try:
+        # Retrieve the DataFrame and execute the SQL query using DuckDB
+        import duckdb
+        df = await get_df(data_id)
+        con = duckdb.connect(database=':memory:', read_only=False)
+        con.register('df_table', df)
+        if params:
+            result_df = con.execute(sql, params).fetchdf()
+        else:
+            result_df = con.execute(sql).fetchdf()
+
+        # Save the resulting DataFrame and create a new data source
+        result = await save_df(result_df, f"query_result_{data_id}")
+        result_id = result["id"]
+        # Retrieve and return the schema of the new data source
+        return get_df_info(result_df, data_id=result_id)
+
+    except Exception as e:
+        # Log the error and raise an exception
+        await log(f"Failed to execute query: {str(e)}", level="ERROR")
+        raise RuntimeError(f"Query execution failed: {str(e)}")
+
+
+def get_df_info(df: pd.DataFrame, data_id: Optional[int]=None) -> str:
+    """
+    Generate a compact string representation of a DataFrame including the count of columns,
+    rows, overall size, and details for each column such as name and datatype.
+
+    Parameters:
+    df (pd.DataFrame): The Pandas DataFrame to describe.
+
+    Returns:
+    str: A string that contains the compact representation of the DataFrame.
+    """
+
+    # Create an output stream to collect strings
+    output = io.StringIO()
+
+    # Write general information about the DataFrame
+    if data_id:
+        output.write(f"Result Data ID: {data_id}\n")
+    output.write("Table Name: df\n")
+    output.write(f"Columns: {len(df.columns)}\n")
+    output.write(f"Rows: {len(df.index)}\n")
+    output.write(f"Size: {df.memory_usage(deep=True).sum()} bytes\n")
+
+    # Iterate through each column to get details
+    for column in df.columns:
+        output.write("---\n")
+        output.write(f"Column: {column}\n")
+        output.write(f"type: {df[column].dtype}\n")
+
+    # Get the complete string from the output stream
+    result = output.getvalue()
+    output.close()
+
+    return result

toolserve/toolserve/cli/main.py

@@ -6,7 +6,6 @@ from pathlib import Path
 from rich.console import Console
 from rich.markup import escape
 
-from toolserve.common.log import log
 from toolserve.server.core.conf import settings
 from toolserve.server.main import app
 from toolserve.apm.pack import Packer

toolserve/toolserve/common/serializers.py

@@ -1,15 +0,0 @@
-from decimal import Decimal
-from typing import Any, Sequence, TypeVar
-
-import msgspec
-
-from starlette.responses import JSONResponse
-
-
-class MsgSpecJSONResponse(JSONResponse):
-    """
-    JSON response using the high-performance msgspec library to serialize data to JSON.
-    """
-
-    def render(self, content: Any) -> bytes:
-        return msgspec.json.encode(content)

toolserve/toolserve/sdk/client.py

@@ -0,0 +1,201 @@
+
+import os
+import json
+import httpx
+
+from typing import Optional, Dict, Any, List
+from contextlib import asynccontextmanager
+from enum import Enum
+
+
+class HttpClient:
+    """
+    A simple HTTP client class to handle requests to a specified base URL with optional authentication.
+    """
+    def __init__(self, base_url: str, auth_token: Optional[str] = None):
+        """
+        Initializes the HttpClient with a base URL and an optional authentication token.
+
+        Args:
+            base_url (str): The base URL for the HTTP requests.
+            auth_token (Optional[str]): Optional bearer token for authorization.
+        """
+        self.base_url = base_url
+        self.auth_token = auth_token
+        self.client = httpx.AsyncClient()
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        await self.client.aclose()
+
+    async def post(self, endpoint: str, data: Dict[str, Any], files: Optional[Dict[str, Any]] = None) -> Any:
+        """
+        Sends a POST request to the specified endpoint with the provided data and files.
+
+        Args:
+            endpoint (str): The endpoint to send the POST request to.
+            data (Dict[str, Any]): The data to send in the POST request.
+            files (Optional[Dict[str, Any]]): Optional files to send with the request.
+
+        Returns:
+            Any: The JSON response from the server.
+        """
+        headers = {"Authorization": f"Bearer {self.auth_token}"} if self.auth_token else {}
+        try:
+            if files:
+                response = await self.client.post(f"{self.base_url}{endpoint}", files=files, headers=headers)
+            else:
+                response = await self.client.post(f"{self.base_url}{endpoint}", json=data, headers=headers)
+            response.raise_for_status()
+            return response.json()
+        except httpx.HTTPStatusError as e:
+            raise RuntimeError(f"HTTP error occurred: {e.response.status_code}")
+        except httpx.RequestError as e:
+            raise RuntimeError(f"Request error occurred: {e}")
+
+    async def get(self, endpoint: str, params: Optional[Dict[str, Any]] = None) -> Any:
+        """
+        Sends a GET request to the specified endpoint with optional parameters.
+
+        Args:
+            endpoint (str): The endpoint to send the GET request to.
+            params (Optional[Dict[str, Any]]): Optional parameters to include in the request.
+
+        Returns:
+            Any: The JSON response from the server.
+        """
+        headers = {"Authorization": f"Bearer {self.auth_token}"} if self.auth_token else {}
+        try:
+            response = await self.client.get(f"{self.base_url}{endpoint}", params=params, headers=headers)
+            response.raise_for_status()
+            return response.json()
+        except httpx.HTTPStatusError as e:
+            raise RuntimeError(f"HTTP error occurred: {e.response.status_code}")
+        except httpx.RequestError as e:
+            raise RuntimeError(f"Request error occurred: {e}")
+
+@asynccontextmanager
+async def managed_http_client(base_url: str, auth_token: Optional[str] = None):
+    """
+    Context manager to handle the lifecycle of HttpClient instances.
+
+    Args:
+        base_url (str): The base URL for the HTTP requests.
+        auth_token (Optional[str]): Optional bearer token for authorization.
+    """
+    client = HttpClient(base_url, auth_token)
+    try:
+        yield client
+    finally:
+        await client.__aexit__(None, None, None)
+
+def get_base_url() -> str:
+    return os.getenv('TOOLSERVE_URL', 'http://localhost:8000')
+
+
+# ----- SDK Functions -----
+
+
+class LogLevel(Enum):
+    DEBUG = "DEBUG"
+    INFO = "INFO"
+    WARNING = "WARNING"
+    ERROR = "ERROR"
+    CRITICAL = "CRITICAL"
+
+async def log(message: str = "", level: LogLevel = LogLevel.INFO, auth_token: Optional[str] = None, endpoint: str = "/api/v1/log"):
+    """
+    Asynchronously sends a log message to a specified endpoint.
+
+    This function constructs a log entry with a message and a log level, then sends it to the server using the provided endpoint. It uses an HTTP POST request within a managed HTTP client context.
+
+    Args:
+        message (str): The log message to send. Defaults to an empty string.
+        level (LogLevel): The severity level of the log message. Defaults to LogLevel.INFO.
+        auth_token (Optional[str]): An optional authorization token for the request.
+        endpoint (str): The API endpoint to which the log message is sent. Defaults to "/api/v1/log".
+
+    Returns:
+        Any: The response from the server as a result of the log message post request.
+    """
+    base_url = get_base_url()
+    if isinstance(level, str):
+        level = LogLevel(level)
+    async with managed_http_client(base_url, auth_token) as client:
+        log_data = {"msg": message, "level": level.value}
+        return await client.post(endpoint, data=log_data)
+
+
+async def list_data(auth_token: Optional[str] = None, endpoint: str = "/api/v1/data") -> List[Dict[str, Any]]:
+    """
+    Retrieve a list of data objects from a specified endpoint.
+
+    Args:
+        auth_token (Optional[str]): Optional authorization token.
+        endpoint (str): API endpoint to send the request to. Defaults to "/api/v1/data".
+
+    Returns:
+        Dict[str, Any]: The deserialized JSON data retrieved from the server.
+    """
+    base_url = get_base_url()
+    async with managed_http_client(base_url, auth_token) as client:
+        response = await client.get(endpoint)
+    return response["data"]
+
+async def get_data(data_id: int, auth_token: Optional[str] = None, endpoint: str = "/api/v1/data/object") -> Any:
+    """
+    Retrieve data object by its primary key from a specified endpoint.
+
+    Args:
+        data_id (int): The primary key of the data object to retrieve.
+        auth_token (Optional[str]): Optional authorization token.
+        endpoint (str): API endpoint to send the request to. Defaults to "/api/v1/data/object".
+
+    Returns:
+        Any: The deserialized JSON data retrieved from the server.
+    """
+    base_url = get_base_url()
+    endpoint = f"{endpoint}/{str(data_id)}"  # Append the data ID to the endpoint URL
+    async with managed_http_client(base_url, auth_token) as client:
+        response = await client.get(endpoint)
+    json_blob = response["data"].get('json_blob', '{}')
+    return json.loads(json_blob)
+
+
+async def send_data(name: str, data: Dict[str, Any], auth_token: Optional[str] = None, endpoint: str = "/api/v1/data") -> Dict[str, Any]:
+    """
+    Send data to a specified endpoint, serializing the data into JSON under the key 'json_blob'.
+
+    Args:
+        data (Dict[str, Any]): Data to be serialized and sent.
+        auth_token (Optional[str]): Optional authorization token.
+        endpoint (str): API endpoint to send the data to.
+
+    Returns:
+        Dict[str, Any]: The response from the server after sending the data.
+    """
+    base_url = get_base_url()
+    json_blob = json.dumps(data)
+    payload = {'file_name': name, 'json_blob': json_blob}
+    async with managed_http_client(base_url, auth_token) as client:
+        response = await client.post(endpoint, data=payload)
+    if response["code"] != 200:
+        raise RuntimeError(f"Failed to send data: {response['msg']}")
+    else:
+        return {
+            "id": response["data"]["id"],
+            "file_path": response["data"]["file_path"]
+        }
+
+
+async def save_artifact_from_file(file_path: str = "", auth_token: Optional[str] = None, endpoint: str = "/api/v1/artifact"):
+    base_url = get_base_url()
+    async with managed_http_client(base_url, auth_token) as client:
+        with open(file_path, 'rb') as file:
+            files = {'file': file}
+            return await client.post(endpoint, data={}, files=files)
+
+
+

toolserve/toolserve/sdk/dataframe.py

@@ -0,0 +1,39 @@
+
+try:
+    import pandas as pd
+except ImportError:
+    raise ImportError("Pandas is required for this SDK component. Please install it using `pip install pandas`.")
+
+from typing import Any, Dict
+from toolserve.sdk.client import get_data, send_data
+
+
+async def save_df(df: pd.DataFrame, name: str) -> Dict[str, Any]:
+    """
+    Asynchronously saves a DataFrame to the server by converting it to a dictionary and using the SDK's send_data function.
+
+    Args:
+        df (pd.DataFrame): The DataFrame to save.
+        name (str): The name under which the DataFrame should be saved.
+
+    Returns:
+        Dict[str, Any]: The server's response after saving the DataFrame.
+    """
+    data_dict = df.to_dict(orient='records')
+    response = await send_data(name=name, data={"data": data_dict})
+    return response
+
+
+async def get_df(data_id: int) -> pd.DataFrame:
+    """
+    Asynchronously retrieves a DataFrame from the server using its data ID.
+
+    Args:
+        data_id (int): The unique identifier for the DataFrame to retrieve.
+
+    Returns:
+        pd.DataFrame: The DataFrame retrieved from the server.
+    """
+    response = await get_data(data_id=data_id)
+    df = pd.DataFrame(response['data'])
+    return df

toolserve/toolserve/server/common/exception/__init__.py

@@ -0,0 +1,2 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-

toolserve/toolserve/server/common/exception/errors.py

@@ -0,0 +1,79 @@
+
+from typing import Any
+
+from fastapi import HTTPException
+from starlette.background import BackgroundTask
+
+from toolserve.server.common.response_code import CustomErrorCode, StandardResponseCode
+
+
+class BaseExceptionMixin(Exception):
+    code: int
+
+    def __init__(self, *, msg: str = None, data: Any = None, background: BackgroundTask | None = None):
+        self.msg = msg
+        self.data = data
+        # The original background task: https://www.starlette.io/background/
+        self.background = background
+
+
+class HTTPError(HTTPException):
+    def __init__(self, *, code: int, msg: Any = None, headers: dict[str, Any] | None = None):
+        super().__init__(status_code=code, detail=msg, headers=headers)
+
+
+class CustomError(BaseExceptionMixin):
+    def __init__(self, *, error: CustomErrorCode, data: Any = None, background: BackgroundTask | None = None):
+        self.code = error.code
+        super().__init__(msg=error.msg, data=data, background=background)
+
+
+class RequestError(BaseExceptionMixin):
+    code = StandardResponseCode.HTTP_400
+
+    def __init__(self, *, msg: str = 'Bad Request', data: Any = None, background: BackgroundTask | None = None):
+        super().__init__(msg=msg, data=data, background=background)
+
+
+class ForbiddenError(BaseExceptionMixin):
+    code = StandardResponseCode.HTTP_403
+
+    def __init__(self, *, msg: str = 'Forbidden', data: Any = None, background: BackgroundTask | None = None):
+        super().__init__(msg=msg, data=data, background=background)
+
+
+class NotFoundError(BaseExceptionMixin):
+    code = StandardResponseCode.HTTP_404
+
+    def __init__(self, *, msg: str = 'Not Found', data: Any = None, background: BackgroundTask | None = None):
+        super().__init__(msg=msg, data=data, background=background)
+
+
+class ServerError(BaseExceptionMixin):
+    code = StandardResponseCode.HTTP_500
+
+    def __init__(
+        self, *, msg: str = 'Internal Server Error', data: Any = None, background: BackgroundTask | None = None
+    ):
+        super().__init__(msg=msg, data=data, background=background)
+
+
+class GatewayError(BaseExceptionMixin):
+    code = StandardResponseCode.HTTP_502
+
+    def __init__(self, *, msg: str = 'Bad Gateway', data: Any = None, background: BackgroundTask | None = None):
+        super().__init__(msg=msg, data=data, background=background)
+
+
+class AuthorizationError(BaseExceptionMixin):
+    code = StandardResponseCode.HTTP_401
+
+    def __init__(self, *, msg: str = 'Permission Denied', data: Any = None, background: BackgroundTask | None = None):
+        super().__init__(msg=msg, data=data, background=background)
+
+
+class TokenError(HTTPError):
+    code = StandardResponseCode.HTTP_401
+
+    def __init__(self, *, msg: str = 'Not Authenticated', headers: dict[str, Any] | None = None):
+        super().__init__(code=self.code, msg=msg, headers=headers or {'WWW-Authenticate': 'Bearer'})

toolserve/toolserve/server/common/exception/exception_handler.py

@@ -0,0 +1,222 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+from fastapi import FastAPI, Request
+from fastapi.exceptions import RequestValidationError
+from pydantic import ValidationError
+from pydantic.errors import PydanticUserError
+from starlette.exceptions import HTTPException
+from starlette.middleware.cors import CORSMiddleware
+
+from toolserve.server.common.exception.errors import BaseExceptionMixin
+from toolserve.server.common.log import log
+from toolserve.server.common.response_code import CustomResponseCode, StandardResponseCode
+from toolserve.server.common.response_code import response_base
+from toolserve.server.core.conf import settings
+from toolserve.server.schemas.base import (
+    CUSTOM_USAGE_ERROR_MESSAGES,
+    CUSTOM_VALIDATION_ERROR_MESSAGES,
+)
+from toolserve.server.utils.serializers import MsgSpecJSONResponse
+
+
+async def _validation_exception_handler(request: Request, e: RequestValidationError | ValidationError):
+    """
+    Data validation exception handling
+
+    :param e:
+    :return:
+    """
+    error = e.errors()[0]
+    if error.get('type') == 'json_invalid':
+        message = 'JSON parsing failed'
+    else:
+        error_input = error.get('input')
+        field = str(error.get('loc')[-1])
+        error_msg = error.get('msg')
+        message = f'{field} {error_msg}, input: {error_input}'
+    msg = f'Invalid request parameters: {message}'
+    data = {'errors': error} if settings.ENVIRONMENT == 'dev' else None
+    content = {
+        'code': StandardResponseCode.HTTP_422,
+        'msg': msg,
+        'data': data,
+    }
+    request.state.__request_validation_exception__ = content  # For obtaining exception information in middleware
+    return MsgSpecJSONResponse(status_code=422, content=content)
+
+
+def register_exception(app: FastAPI):
+    @app.exception_handler(HTTPException)
+    async def http_exception_handler(request: Request, exc: HTTPException):
+        """
+        Global HTTP exception handling
+
+        :param request:
+        :param exc:
+        :return:
+        """
+        if settings.ENVIRONMENT == 'dev':
+            content = {
+                'code': exc.status_code,
+                'msg': exc.detail,
+                'data': None,
+            }
+        else:
+            res = await response_base.fail(res=CustomResponseCode.HTTP_400)
+            content = res.model_dump()
+        request.state.__request_http_exception__ = content  # For obtaining exception information in middleware
+        return MsgSpecJSONResponse(
+            status_code=StandardResponseCode.HTTP_400,
+            content=content,
+            headers=exc.headers,
+        )
+
+    @app.exception_handler(RequestValidationError)
+    async def fastapi_validation_exception_handler(request: Request, exc: RequestValidationError):
+        """
+        FastAPI data validation exception handling
+
+        :param request:
+        :param exc:
+        :return:
+        """
+        return await _validation_exception_handler(request, exc)
+
+    @app.exception_handler(ValidationError)
+    async def pydantic_validation_exception_handler(request: Request, exc: ValidationError):
+        """
+        Pydantic data validation exception handling
+
+        :param request:
+        :param exc:
+        :return:
+        """
+        return await _validation_exception_handler(request, exc)
+
+    @app.exception_handler(PydanticUserError)
+    async def pydantic_user_error_handler(request: Request, exc: PydanticUserError):
+        """
+        Pydantic user exception handling
+
+        :param request:
+        :param exc:
+        :return:
+        """
+        return MsgSpecJSONResponse(
+            status_code=StandardResponseCode.HTTP_500,
+            content={
+                'code': StandardResponseCode.HTTP_500,
+                'msg': CUSTOM_USAGE_ERROR_MESSAGES.get(exc.code),
+                'data': None,
+            },
+        )
+
+    @app.exception_handler(AssertionError)
+    async def assertion_error_handler(request: Request, exc: AssertionError):
+        """
+        Assertion error handling
+
+        :param request:
+        :param exc:
+        :return:
+        """
+        if settings.ENVIRONMENT == 'dev':
+            content = {
+                'code': StandardResponseCode.HTTP_500,
+                'msg': str(''.join(exc.args) if exc.args else exc.__doc__),
+                'data': None,
+            }
+        else:
+            res = await response_base.fail(res=CustomResponseCode.HTTP_500)
+            content = res.model_dump()
+        return MsgSpecJSONResponse(
+            status_code=StandardResponseCode.HTTP_500,
+            content=content,
+        )
+
+    @app.exception_handler(Exception)
+    async def all_exception_handler(request: Request, exc: Exception):
+        """
+        Global exception handling
+
+        :param request:
+        :param exc:
+        :return:
+        """
+        if isinstance(exc, BaseExceptionMixin):
+            return MsgSpecJSONResponse(
+                status_code=StandardResponseCode.HTTP_400,
+                content={
+                    'code': exc.code,
+                    'msg': str(exc.msg),
+                    'data': exc.data if exc.data else None,
+                },
+                background=exc.background,
+            )
+        else:
+            import traceback
+
+            log.error(f'Unknown exception: {exc}')
+            log.error(traceback.format_exc())
+            if settings.ENVIRONMENT == 'dev':
+                content = {
+                    'code': 500,
+                    'msg': str(exc),
+                    'data': None,
+                }
+            else:
+                res = await response_base.fail(res=CustomResponseCode.HTTP_500)
+                content = res.model_dump()
+            return MsgSpecJSONResponse(status_code=StandardResponseCode.HTTP_500, content=content)
+
+    if settings.MIDDLEWARE_CORS:
+
+        @app.exception_handler(StandardResponseCode.HTTP_500)
+        async def cors_status_code_500_exception_handler(request, exc):
+            """
+            CORS 500 exception handling
+
+            `Related issue <https://github.com/encode/starlette/issues/1175>`_
+
+            :param request:
+            :param exc:
+            :return:
+            """
+            if isinstance(exc, BaseExceptionMixin):
+                content = {
+                    'code': exc.code,
+                    'msg': exc.msg,
+                    'data': exc.data,
+                }
+            else:
+                if settings.ENVIRONMENT == 'dev':
+                    content = {
+                        'code': StandardResponseCode.HTTP_500,
+                        'msg': str(exc),
+                        'data': None,
+                    }
+                else:
+                    res = await response_base.fail(res=CustomResponseCode.HTTP_500)
+                    content = res.model_dump()
+            response = MsgSpecJSONResponse(
+                status_code=exc.code if isinstance(exc, BaseExceptionMixin) else StandardResponseCode.HTTP_500,
+                content=content,
+                background=exc.background if isinstance(exc, BaseExceptionMixin) else None,
+            )
+            origin = request.headers.get('origin')
+            if origin:
+                cors = CORSMiddleware(
+                    app=app,
+                    allow_origins=['*'],
+                    allow_credentials=True,
+                    allow_methods=['*'],
+                    allow_headers=['*'],
+                )
+                response.headers.update(cors.simple_headers)
+                has_cookie = 'cookie' in request.headers
+                if cors.allow_all_origins and has_cookie:
+                    response.headers['Access-Control-Allow-Origin'] = origin
+                elif not cors.allow_all_origins and cors.is_allowed_origin(origin=origin):
+                    response.headers['Access-Control-Allow-Origin'] = origin
+                    response.headers.add_vary_header('Origin')
+            return response

toolserve/toolserve/server/common/response.py

@@ -5,7 +5,7 @@ from typing import Any
 
 from pydantic import BaseModel, ConfigDict
 
-from toolserve.common.response_code import CustomResponse, CustomResponseCode
+from toolserve.server.common.response_code import CustomResponse, CustomResponseCode
 from toolserve.server.core.conf import settings
 
 _ExcludeData = set[int | str] | dict[int | str, Any]

toolserve/toolserve/server/common/serializers.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+from decimal import Decimal
+from typing import Any, Sequence, TypeVar
+
+import msgspec
+
+from asgiref.sync import sync_to_async
+from sqlalchemy import Row, RowMapping
+from starlette.responses import JSONResponse
+
+RowData = Row | RowMapping | Any
+
+R = TypeVar('R', bound=RowData)
+
+
+@sync_to_async
+def select_columns_serialize(row: R) -> dict:
+    """
+    Serialize SQLAlchemy select table columns, does not contain relational columns
+
+    :param row:
+    :return:
+    """
+    obj_dict = {}
+    for column in row.__table__.columns.keys():
+        val = getattr(row, column)
+        if isinstance(val, Decimal):
+            if val % 1 == 0:
+                val = int(val)
+            val = float(val)
+        obj_dict[column] = val
+    return obj_dict
+
+
+async def select_list_serialize(row: Sequence[R]) -> list:
+    """
+    Serialize SQLAlchemy select list
+
+    :param row:
+    :return:
+    """
+    ret_list = [await select_columns_serialize(_) for _ in row]
+    return ret_list
+
+
+@sync_to_async
+def select_as_dict(row: R) -> dict:
+    """
+    Converting SQLAlchemy select to dict, which can contain relational data,
+    depends on the properties of the select object itself
+
+    :param row:
+    :return:
+    """
+    obj_dict = row.__dict__
+    if '_sa_instance_state' in obj_dict:
+        del obj_dict['_sa_instance_state']
+        return obj_dict
+
+
+class MsgSpecJSONResponse(JSONResponse):
+    """
+    JSON response using the high-performance msgspec library to serialize data to JSON.
+    """
+
+    def render(self, content: Any) -> bytes:
+        return msgspec.json.encode(content)

toolserve/toolserve/server/core/catalog.py

@@ -13,8 +13,8 @@ from pydantic import BaseModel, ValidationError, Field, create_model
 from importlib import import_module
 
 from toolserve.server.core.conf import settings
-from toolserve.common.response_code import CustomResponseCode
-from toolserve.common.response import ResponseModel, response_base
+from toolserve.server.common.response_code import CustomResponseCode
+from toolserve.server.common.response import ResponseModel, response_base
 from toolserve.apm.base import ToolPack
 from toolserve.sdk import Param
 from toolserve.utils import snake_to_camel
@@ -41,6 +41,7 @@ class ToolSchema(BaseModel):
 class ToolCatalog:
     def __init__(self, tools_dir: str = settings.TOOLS_DIR):
         self.tools = self.read_tools(tools_dir)
+        self.tools.update(self.__get_builitin_tools())
 
     @staticmethod
     def read_tools(directory: str) -> List[ToolSchema]:
@@ -76,10 +77,39 @@ class ToolCatalog:
 
         return tools
 
+    def __get_builitin_tools(self) -> Dict[str, ToolSchema]:
+        tools = {}
+        sys.path.append(str(settings.BUILTIN_TOOLS_DIR))
+
+        for tool_spec in settings.BUILTIN_TOOLS:
+            print(tool_spec)
+
+            module_name, versioned_tool = tool_spec.split('.', 1)
+            func_name, version = versioned_tool.split('@')
+
+            module = import_module(module_name)
+            tool = getattr(module, func_name)
+
+            input_model, output_model = create_func_models(tool)
+            response_model = create_response_model(func_name, output_model)
+            tool_schema = ToolSchema(
+                name=func_name,
+                description=tool.__doc__,
+                version='builtin',
+                tool=tool,
+                input_model=input_model,
+                output_model=response_model,
+                meta=ToolMeta(module=module_name, path=module.__file__)
+            )
+            tools[func_name] = tool_schema
+
+        return tools
+
+
     def __getitem__(self, name: str) -> Optional[ToolSchema]:
         #TODO error handling
-        for tool in self.tools:
-            if tool.name == name:
+        for tool_name, tool in self.tools.items():
+            if tool_name == name:
                 return tool
         return None
 
@@ -90,7 +120,7 @@ class ToolCatalog:
         return None
 
     def list_tools(self) -> List[Dict[str, str]]:
-        return [{'name': t.name, 'description': t.description} for t in self.tools]
+        return [{'name': t.name, 'description': t.description} for t in self.tools.values()]
 
 
 
@@ -160,10 +190,11 @@ def determine_output_model(func: Callable) -> Type[BaseModel]:
     if return_annotation is inspect.Signature.empty:
         return create_model(f"{snake_to_camel(func.__name__)}Output")
     elif hasattr(return_annotation, '__origin__'):
-        field_type = Optional[return_annotation.__args__[0]]
-        description = return_annotation.__metadata__[0] if return_annotation.__metadata__ else ""
-        if description:
-            return create_model(f"{snake_to_camel(func.__name__)}Output", result=(field_type, Field(description=str(description))))
+        if hasattr(return_annotation, '__metadata__'):
+            field_type = Optional[return_annotation.__args__[0]]
+            description = return_annotation.__metadata__[0] if return_annotation.__metadata__ else ""
+            if description:
+                return create_model(f"{snake_to_camel(func.__name__)}Output", result=(field_type, Field(description=str(description))))
         else:
             return create_model(f"{snake_to_camel(func.__name__)}Output", result=(return_annotation, Field(description="No description provided.")))
 
@@ -173,7 +204,7 @@ def create_response_model(name: str, output_model: Type[BaseModel]) -> Type[Resp
     """
     # Create a new response model
     response_model = create_model(
-        f"{name}Response",
+        f"{snake_to_camel(name)}Response",
         code=(int, CustomResponseCode.HTTP_200.code),
         msg=(str, CustomResponseCode.HTTP_200.msg),
         data=(Optional[output_model], None)

toolserve/toolserve/server/core/conf.py

@@ -14,8 +14,17 @@ from pydantic_settings import BaseSettings, SettingsConfigDict
 class Settings(BaseSettings):
     model_config = SettingsConfigDict(env_file='.env')
 
-    WORK_DIR: Path = Path.home() / '.darkstar'
+    WORK_DIR: Path = Path.home() / '.arcade'
     TOOLS_DIR: Path = os.getcwd()
+    ARTIFACTS_DIR: Path = WORK_DIR / 'artifacts'
+    DATA_DIR: Path = WORK_DIR / 'data'
+
+    BUILTIN_TOOLS_DIR: Path = Path(__file__).parent.parent.parent / 'builtin' / 'default'
+    BUILTIN_TOOLS: list[str] = [
+        "query.list_data_sources@builtin",
+        "query.get_data_schema@builtin",
+        "query.query_sql@builtin",
+    ]
 
     # Env Config
     ENVIRONMENT: Literal['dev', 'pro'] = 'dev'
@@ -34,10 +43,10 @@ class Settings(BaseSettings):
 
     # FastAPI
     API_V1_STR: str = '/api/v1'
-    API_ACTION_STR: str = '/action'
-    TITLE: str = 'Darkstar Toolserver'
+    API_ACTION_STR: str = '/tool'
+    TITLE: str = 'Arcade AI Toolserver'
     VERSION: str = '0.1.0'
-    DESCRIPTION: str = 'Darkstar Toolserver API'
+    DESCRIPTION: str = 'Arcade AI Toolserver API'
     DOCS_URL: str | None = f'{API_V1_STR}/docs'
     REDOCS_URL: str | None = f'{API_V1_STR}/redocs'
     OPENAPI_URL: str | None = f'{API_V1_STR}/openapi'
@@ -88,7 +97,15 @@ class Settings(BaseSettings):
     TOKEN_SECRET_KEY: str = "secret"
     OPERA_LOG_ENCRYPT_SECRET_KEY: str = "secret"
 
+    # SQL Database
+    DB_HOST: str = "localhost"
+    DB_PORT: int = "3306"
+    DB_USER: str = "arcade"
+    DB_PASSWORD: str = "arcade"
 
+    DB_ECHO: bool = False
+    DB_DATABASE: str = 'arcade'
+    DB_CHARSET: str = 'utf8mb4'
 
 @lru_cache
 def get_settings():

toolserve/toolserve/server/core/generate.py

@@ -1,5 +1,6 @@
 import os
 import sys
+import traceback
 import inspect
 from textwrap import dedent
 from typing import List, Optional, Type, Annotated, Dict
@@ -11,8 +12,8 @@ from importlib import import_module
 
 from toolserve.server.core.catalog import ToolSchema
 from toolserve.server.core.conf import settings
-from toolserve.common.response_code import CustomResponseCode
-from toolserve.common.response import ResponseModel, response_base
+from toolserve.server.common.response_code import CustomResponseCode
+from toolserve.server.common.response import ResponseModel, response_base
 
 
 def create_endpoint_function(name, description, func, input_model, output_model):
@@ -28,7 +29,6 @@ def create_endpoint_function(name, description, func, input_model, output_model)
         except ValidationError as e:
             return await response_base.error(res=CustomResponseCode.HTTP_400, msg=str(e))
         except Exception as e:
-            import traceback
             print(traceback.format_exc())
             return await response_base.error(res=CustomResponseCode.HTTP_500, msg=str(e))
 

toolserve/toolserve/server/core/registrar.py

@@ -5,8 +5,22 @@ from contextlib import asynccontextmanager
 from fastapi import Depends, FastAPI
 
 from toolserve.server.routes import v1
+from toolserve.server.database.db_sqlite import create_table
 from toolserve.server.core.conf import settings
-from toolserve.common.serializers import MsgSpecJSONResponse
+from toolserve.server.common.serializers import MsgSpecJSONResponse
+
+
+@asynccontextmanager
+async def register_init(app: FastAPI):
+    """
+
+    :return:
+    """
+    # create database tables
+    await create_table()
+
+    yield
+
 
 def register_app():
     # FastAPI
@@ -18,6 +32,7 @@ def register_app():
         redoc_url=settings.REDOCS_URL,
         openapi_url=settings.OPENAPI_URL,
         default_response_class=MsgSpecJSONResponse,
+        lifespan=register_init,
     )
 
     register_static_file(app)

toolserve/toolserve/server/crud/base.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+from typing import Any, Dict, Generic, Type, TypeVar
+
+from pydantic import BaseModel
+from sqlalchemy import and_, delete, select, update
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from toolserve.server.models.base import MappedBase
+
+ModelType = TypeVar('ModelType', bound=MappedBase)
+CreateSchemaType = TypeVar('CreateSchemaType', bound=BaseModel)
+UpdateSchemaType = TypeVar('UpdateSchemaType', bound=BaseModel)
+
+
+class CRUDBase(Generic[ModelType, CreateSchemaType, UpdateSchemaType]):
+    def __init__(self, model: Type[ModelType]):
+        self.model = model
+
+    async def get_(
+        self,
+        db: AsyncSession,
+        *,
+        pk: int | None = None,
+        name: str | None = None,
+        status: int | None = None,
+        del_flag: int | None = None,
+    ) -> ModelType | None:
+        """
+        Get a record by primary key id or name
+
+        :param db:
+        :param pk:
+        :param name:
+        :param status:
+        :param del_flag:
+        :return:
+        """
+        assert pk is not None or name is not None, 'Query error, pk and name parameters cannot be empty at the same time'
+        assert pk is None or name is None, 'Query error, pk and name parameters cannot exist at the same time'
+        where_list = [self.model.id == pk] if pk is not None else [self.model.name == name]
+        if status is not None:
+            assert status in (0, 1), 'Query error, status parameter can only be 0 or 1'
+            where_list.append(self.model.status == status)
+        if del_flag is not None:
+            assert del_flag in (0, 1), 'Query error, del_flag parameter can only be 0 or 1'
+            where_list.append(self.model.del_flag == del_flag)
+
+        result = await db.execute(select(self.model).where(and_(*where_list)))
+        return result.scalars().first()
+
+    async def create_(self, db: AsyncSession, obj_in: CreateSchemaType, user_id: int | None = None) -> None:
+        """
+        Add a new record
+
+        :param db:
+        :param obj_in: Pydantic model class
+        :param user_id:
+        :return:
+        """
+        if user_id:
+            create_data = self.model(**obj_in.model_dump(), create_user=user_id)
+        else:
+            create_data = self.model(**obj_in.model_dump())
+        db.add(create_data)
+
+    async def update_(
+        self, db: AsyncSession, pk: int, obj_in: UpdateSchemaType | Dict[str, Any], user_id: int | None = None
+    ) -> int:
+        """
+        Update a record by primary key id
+
+        :param db:
+        :param pk:
+        :param obj_in: Pydantic model class or dictionary corresponding to database fields
+        :param user_id:
+        :return:
+        """
+        if isinstance(obj_in, dict):
+            update_data = obj_in
+        else:
+            update_data = obj_in.model_dump(exclude_unset=True)
+        if user_id:
+            update_data.update({'update_user': user_id})
+        result = await db.execute(update(self.model).where(self.model.id == pk).values(**update_data))
+        return result.rowcount
+
+    async def delete_(self, db: AsyncSession, pk: int, *, del_flag: int | None = None) -> int:
+        """
+        Delete a record by primary key id
+
+        :param db:
+        :param pk:
+        :param del_flag:
+        :return:
+        """
+        if del_flag is None:
+            result = await db.execute(delete(self.model).where(self.model.id == pk))
+        else:
+            assert del_flag == 1, 'Delete error, del_flag parameter can only be 1'
+            result = await db.execute(update(self.model).where(self.model.id == pk).values(del_flag=del_flag))
+        return result.rowcount

toolserve/toolserve/server/crud/crud_artifact.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+from typing import Sequence
+
+from sqlalchemy import Select, and_, delete, desc, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from toolserve.server.crud.base import CRUDBase
+from toolserve.server.models.sys_artifact import Artifact
+from toolserve.server.schemas.artifact import CreateArtifactParam, UpdateArtifactParam
+
+
+class CRUDArtifact(CRUDBase[Artifact, CreateArtifactParam, UpdateArtifactParam]):
+    async def get(self, db: AsyncSession, pk: int) -> Artifact | None:
+        return await self.get_(db, pk=pk)
+
+    async def get_list(self, name: str = None, file_path: str = None) -> Select:
+        se = select(self.model).order_by(desc(self.model.created_time))
+        where_list = []
+        if name:
+            where_list.append(self.model.name.like(f'%{name}%'))
+        if file_path:
+            where_list.append(self.model.file_path.like(f'%{file_path}%', escape='/'))
+        if where_list:
+            se = se.where(and_(*where_list))
+        return se
+
+    async def get_all(self, db: AsyncSession) -> Sequence[Artifact]:
+        artifacts = await db.execute(select(self.model))
+        return artifacts.scalars().all()
+
+    async def get_by_name(self, db: AsyncSession, name: str) -> Artifact | None:
+        artifact = await db.execute(select(self.model).where(self.model.name == name))
+        return artifact.scalars().first()
+
+    async def create(self, db: AsyncSession, obj_in: CreateArtifactParam) -> None:
+        await self.create_(db, obj_in)
+
+    async def update(self, db: AsyncSession, pk: int, obj_in: UpdateArtifactParam) -> int:
+        return await self.update_(db, pk, obj_in)
+
+    async def delete(self, db: AsyncSession, pk: list[int]) -> int:
+        artifacts = await db.execute(delete(self.model).where(self.model.id.in_(pk)))
+        return artifacts.rowcount
+
+
+artifact_dao: CRUDArtifact = CRUDArtifact(Artifact)

toolserve/toolserve/server/crud/crud_data.py

@@ -0,0 +1,51 @@
+import datetime
+from typing import Sequence
+from sqlalchemy import Select, and_, delete, desc, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from toolserve.server.crud.base import CRUDBase
+from toolserve.server.models.sys_data import Data
+from toolserve.server.schemas.data import CreateDataParam, DataSchemaBase
+
+class CRUDData(CRUDBase[Data, CreateDataParam, DataSchemaBase]):
+    async def get(self, db: AsyncSession, pk: int) -> Data | None:
+        return await self.get_(db, pk=pk)
+
+    async def get_list(self, file_name: str = None, file_path: str = None) -> Select:
+        query = select(self.model).order_by(desc(self.model.created_time))
+        conditions = []
+        if file_name:
+            conditions.append(self.model.file_name.like(f'%{file_name}%'))
+        if file_path:
+            conditions.append(self.model.file_path.like(f'%{file_path}%'))
+        if conditions:
+            query = query.where(and_(*conditions))
+        return query
+
+    async def get_all(self, db: AsyncSession) -> Sequence[Data]:
+        result = await db.execute(select(self.model))
+        return result.scalars().all()
+
+    async def get_by_file_name(self, db: AsyncSession, file_name: str) -> Data | None:
+        result = await db.execute(select(self.model).where(self.model.file_name == file_name))
+        return result.scalars().first()
+
+    async def create(self, db: AsyncSession, obj_in: CreateDataParam) -> Data:
+        existing_data = await self.get_by_file_name(db, obj_in.file_name)
+        if existing_data:
+            existing_data.updated_time = datetime.datetime.now()
+            db.add(existing_data)
+            return existing_data
+        else:
+            obj = self.model(**obj_in.dict())
+            db.add(obj)
+            return obj
+
+    async def update(self, db: AsyncSession, pk: int, obj_in: DataSchemaBase) -> int:
+        return await self.update_(db, pk, obj_in)
+
+    async def delete(self, db: AsyncSession, pk: list[int]) -> int:
+        result = await db.execute(delete(self.model).where(self.model.id.in_(pk)))
+        return result.rowcount
+
+data_dao: CRUDData = CRUDData(Data)

toolserve/toolserve/server/crud/crud_log.py

@@ -0,0 +1,45 @@
+from typing import Sequence
+
+from sqlalchemy import Select, and_, delete, desc, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from toolserve.server.crud.base import CRUDBase
+from toolserve.server.models.sys_log import Log
+from toolserve.server.schemas.log import CreateLog, LogSchemaBase
+
+
+class CRUDLog(CRUDBase[Log, CreateLog, LogSchemaBase]):
+    async def get(self, db: AsyncSession, pk: int) -> Log | None:
+        return await self.get_(db, pk=pk)
+
+    async def get_list(self, level: str = None, msg: str = None) -> Select:
+        query = select(self.model).order_by(desc(self.model.created_time))
+        conditions = []
+        if level:
+            conditions.append(self.model.level == level)
+        if msg:
+            conditions.append(self.model.msg.like(f'%{msg}%'))
+        if conditions:
+            query = query.where(and_(*conditions))
+        return query
+
+    async def get_all(self, db: AsyncSession) -> Sequence[Log]:
+        result = await db.execute(select(self.model))
+        return result.scalars().all()
+
+    async def get_by_level(self, db: AsyncSession, level: str) -> Log | None:
+        result = await db.execute(select(self.model).where(self.model.level == level))
+        return result.scalars().first()
+
+    async def create(self, db: AsyncSession, obj_in: CreateLog) -> None:
+        await self.create_(db, obj_in)
+
+    async def update(self, db: AsyncSession, pk: int, obj_in: LogSchemaBase) -> int:
+        return await self.update_(db, pk, obj_in)
+
+    async def delete(self, db: AsyncSession, pk: list[int]) -> int:
+        result = await db.execute(delete(self.model).where(self.model.id.in_(pk)))
+        return result.rowcount
+
+
+log_dao: CRUDLog = CRUDLog(Log)

toolserve/toolserve/server/database/__init__.py

@@ -0,0 +1,2 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-

toolserve/toolserve/server/database/db_sqlite.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+import sys
+
+from typing import Annotated
+from uuid import uuid4
+
+from fastapi import Depends
+from sqlalchemy import URL
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
+
+from toolserve.server.common.log import log
+from toolserve.server.core.conf import settings
+from toolserve.server.models import (
+    MappedBase,
+    Log,
+    Artifact,
+    Data,
+)
+
+
+def create_engine_and_session(url: str | URL):
+    try:
+        engine = create_async_engine(url, echo=settings.DB_ECHO, future=True, pool_pre_ping=True)
+    except Exception as e:
+        log.error('❌ Error starting db session {}', e)
+        sys.exit()
+    else:
+        db_session = async_sessionmaker(bind=engine, autoflush=False, expire_on_commit=False)
+        return engine, db_session
+
+
+SQLALCHEMY_DATABASE_URL = (
+    f'sqlite+aiosqlite:///{settings.WORK_DIR}/{settings.DB_DATABASE}.db'
+)
+
+async_engine, async_db_session = create_engine_and_session(SQLALCHEMY_DATABASE_URL)
+
+
+async def get_db() -> AsyncSession:
+    """Provide a database session for a single request."""
+    session = async_db_session()
+    try:
+        yield session
+    except Exception as se:
+        await session.rollback()
+        raise se
+    finally:
+        await session.close()
+
+
+# Session Annotated
+CurrentSession = Annotated[AsyncSession, Depends(get_db)]
+
+
+async def create_table():
+    """Create the database tables if they do not exist."""
+    async with async_engine.begin() as conn:
+        try:
+            await conn.run_sync(MappedBase.metadata.create_all)
+        except Exception as e:
+            log.error('❌ Error creating tables {}', e)
+            raise e
+
+
+def uuid4_str() -> str:
+    """Generate a UUID4 string."""
+    return str(uuid4())

toolserve/toolserve/server/main.py

@@ -2,7 +2,7 @@ import uvicorn
 
 from pathlib import Path
 import asyncio
-from toolserve.common.log import log
+from toolserve.server.common.log import log
 from toolserve.server.core.conf import settings
 from toolserve.server.core.registrar import register_app
 
@@ -12,7 +12,7 @@ app = register_app()
 if __name__ == '__main__':
     try:
         log.info(
-            "Darkstar Toolserve is starting..."
+            "Arcade AI Toolserve is starting..."
         )
         uvicorn.run(
             app=f'{Path(__file__).stem}:app',

toolserve/toolserve/server/models/__init__.py

@@ -0,0 +1,7 @@
+
+from toolserve.server.models.base import MappedBase
+from toolserve.server.models.sys_data import Data
+from toolserve.server.models.sys_log import Log
+from toolserve.server.models.sys_artifact import Artifact
+
+__all__ = ['MappedBase', 'Data', 'Log', 'Artifact']

toolserve/toolserve/server/models/base.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+from datetime import datetime
+from typing import Annotated
+
+from sqlalchemy.orm import (
+    DeclarativeBase,
+    Mapped,
+    MappedAsDataclass,
+    declared_attr,
+    mapped_column,
+)
+
+from toolserve.server.utils.timezone import timezone
+
+# Common Mapped type primary key, manual addition required, refer to the following usage
+# MappedBase -> id: Mapped[id_key]
+# DataClassBase && Base -> id: Mapped[id_key] = mapped_column(init=False)
+id_key = Annotated[
+    int,
+    mapped_column(
+        primary_key=True,
+        index=True,
+        autoincrement=True,
+        sort_order=-999,
+        comment="Primary key id",
+    ),
+]
+
+
+# Mixin: An object-oriented programming concept, makes the structure clearer, `Wiki <https://en.wikipedia.org/wiki/Mixin/>`__
+class UserMixin(MappedAsDataclass):
+    """User Mixin data class"""
+
+    create_user: Mapped[int] = mapped_column(sort_order=998, comment="Creator")
+    update_user: Mapped[int | None] = mapped_column(
+        init=False, default=None, sort_order=998, comment="Modifier"
+    )
+
+
+class DateTimeMixin(MappedAsDataclass):
+    """Datetime Mixin data class"""
+
+    created_time: Mapped[datetime] = mapped_column(
+        init=False,
+        default_factory=timezone.now,
+        sort_order=999,
+        comment="Creation time",
+    )
+    updated_time: Mapped[datetime | None] = mapped_column(
+        init=False, onupdate=timezone.now, sort_order=999, comment="Update time"
+    )
+
+
+class MappedBase(DeclarativeBase):
+    """
+    Declarative base class, the original DeclarativeBase class, serves as the parent class for all base or data model classes
+
+    `DeclarativeBase <https://docs.sqlalchemy.org/en/20/orm/declarative_config.html>`__
+    `mapped_column() <https://docs.sqlalchemy.org/en/20/orm/mapping_api.html#sqlalchemy.orm.mapped_column>`__
+    """
+
+    @declared_attr.directive
+    def __tablename__(cls) -> str:
+        return cls.__name__.lower()
+
+
+class DataClassBase(MappedAsDataclass, MappedBase):
+    """
+    Declarative data class base class, integrates with data classes, allows for advanced configurations, but you must be aware of its characteristics, especially when used with DeclarativeBase
+
+    `MappedAsDataclass <https://docs.sqlalchemy.org/en/20/orm/dataclasses.html#orm-declarative-native-dataclasses>`__
+    """  # noqa: E501
+
+    __abstract__ = True
+
+
+class Base(DataClassBase, DateTimeMixin):
+    """
+    Declarative Mixin data class base class, integrates data classes, includes the Mixin data class basic table structure, you can simply understand it as a data class base class with basic table structure
+    """  # noqa: E501
+
+    __abstract__ = True

toolserve/toolserve/server/models/sys_artifact.py

@@ -0,0 +1,13 @@
+from sqlalchemy import String
+from sqlalchemy.orm import Mapped, mapped_column
+
+from toolserve.server.models.base import Base, id_key
+
+
+class Artifact(Base):
+
+    __tablename__ = 'sys_artifact'
+
+    id: Mapped[id_key] = mapped_column(init=False)
+    name: Mapped[str] = mapped_column(String(255), comment='Artifact name')
+    file_path: Mapped[str] = mapped_column(String(255), comment='File path')

toolserve/toolserve/server/models/sys_data.py

@@ -0,0 +1,14 @@
+
+from sqlalchemy import String
+from sqlalchemy.orm import Mapped, mapped_column
+
+from toolserve.server.models.base import Base, id_key
+
+
+class Data(Base):
+
+    __tablename__ = 'sys_data'
+
+    id: Mapped[id_key] = mapped_column(init=False)
+    file_name: Mapped[str] = mapped_column(String(255), comment='File name')
+    file_path: Mapped[str] = mapped_column(String(255), comment='File path')

toolserve/toolserve/server/models/sys_log.py

@@ -0,0 +1,15 @@
+
+
+from sqlalchemy import String
+from sqlalchemy.orm import Mapped, mapped_column
+
+from toolserve.server.models.base import Base, id_key
+
+
+class Log(Base):
+
+    __tablename__ = 'sys_log'
+
+    id: Mapped[id_key] = mapped_column(init=False)
+    level: Mapped[str] = mapped_column(String(50), comment='Log level')
+    msg: Mapped[str] = mapped_column(String(500), comment='Log message')

toolserve/toolserve/server/routes/__init__.py

@@ -1,6 +1,15 @@
 from fastapi import APIRouter
 from toolserve.server.core.conf import settings
-from toolserve.server.routes.action import router as action_router
+from toolserve.server.routes.tool import router as tool_router
+from toolserve.server.routes.data import router as data_router
+from toolserve.server.routes.artifact import router as artifact_router
+from toolserve.server.routes.log import router as log_router
+
 
 v1 = APIRouter(prefix=settings.API_V1_STR)
-v1.include_router(action_router, tags=["action"])
+v1.include_router(tool_router, prefix="/tools", tags=["Tool Catalog"])
+v1.include_router(data_router, prefix="/data", tags=["Data Management"])
+v1.include_router(artifact_router, prefix="/artifact", tags=["Artifact Management"])
+v1.include_router(log_router, prefix="/log", tags=["Tool Logging API"])
+
+

toolserve/toolserve/server/routes/artifact.py

@@ -0,0 +1,45 @@
+
+from typing import Annotated
+from fastapi import APIRouter, Path, Query
+from toolserve.server.schemas.artifact import (
+    ArtifactSchemaBase,
+    CreateArtifactParam,
+    DeleteArtifactParam,
+    GetArtifactDetails
+)
+from toolserve.server.services.artifact_service import artifact_service
+from toolserve.server.common.response import ResponseModel, response_base
+from toolserve.server.common.serializers import select_as_dict
+
+router = APIRouter()
+
+# Get artifact details by artifact id
+@router.get('/{pk}', summary='Get artifact details')
+async def get_artifact(
+    pk: Annotated[int, Path(...)],
+) -> ResponseModel:
+    artifact = await artifact_service.get(pk=pk)
+    data = GetArtifactDetails(**await select_as_dict(artifact))
+    return await response_base.success(data=data)
+
+# Create a new artifact
+@router.post('', summary='Create artifact')
+async def create_artifact(
+    obj: CreateArtifactParam
+) -> ResponseModel:
+    await artifact_service.create(obj=obj)
+    return await response_base.success()
+
+# Delete artifact
+@router.delete('', summary='Delete artifact')
+async def delete_artifact(pk: Annotated[list[int], Query(...)]) -> ResponseModel:
+    count = await artifact_service.delete(pk=pk)
+    if count > 0:
+        return await response_base.success()
+    return await response_base.fail()
+
+# Get all artifacts
+@router.get('', summary='Get all artifacts')
+async def get_all_artifacts() -> ResponseModel:
+    data = await artifact_service.get_all_artifacts()
+    return await response_base.success(data=data)

toolserve/toolserve/server/routes/data.py

@@ -0,0 +1,79 @@
+
+from typing import Annotated
+from fastapi import APIRouter, Path, Query
+from toolserve.server.schemas.data import (
+    CreateDataParam,
+    DataSchemaBase,
+    GetDataDetails,
+    GetDataObject
+)
+from toolserve.server.services.data_service import data_service
+from toolserve.server.common.response import ResponseModel, response_base
+from toolserve.server.common.serializers import select_as_dict
+
+router = APIRouter()
+
+# Get data details by data id
+@router.get('/{pk}', summary='Get data details')
+async def get_data(
+    pk: Annotated[int, Path(...)],
+) -> ResponseModel:
+    data_entry = await data_service.get(pk=pk)
+    data = GetDataDetails(**await select_as_dict(data_entry))
+    return await response_base.success(data=data)
+
+@router.get('', summary='Get all data files')
+async def get_all_data() -> ResponseModel:
+    data = await data_service.get_all_data()
+    all_data = [GetDataDetails(**await select_as_dict(data_entry)) for data_entry in data]
+    return await response_base.success(data=all_data)
+
+# Create a new data entry
+@router.post('', summary='Create data')
+async def create_data(
+    obj: CreateDataParam
+) -> ResponseModel:
+    data_obj = await data_service.create(obj=obj)
+    data = {
+        "id": data_obj.id,
+        "file_path": data_obj.file_path
+    }
+    return await response_base.success(data=data)
+
+# Delete data
+@router.delete('', summary='Delete data')
+async def delete_data(pk: Annotated[list[int], Query(...)]) -> ResponseModel:
+    count = await data_service.delete(pk=pk)
+    if count > 0:
+        return await response_base.success()
+    return await response_base.fail()
+
+@router.get('/object/{pk}', summary='Get data object')
+async def get_data_object(
+    pk: Annotated[int, Path(...)],
+) -> ResponseModel:
+    """
+    Retrieve a data object by its primary key using the GetDataObject schema.
+
+    Args:
+        pk (int): The primary key of the data object to retrieve.
+
+    Returns:
+        ResponseModel: The response model containing the data object or an error message.
+    """
+    data_entry = await data_service.get(pk=pk)
+    obj = await select_as_dict(data_entry)
+    try:
+        with open(obj["file_path"], 'r', encoding='utf-8') as file:
+            json_data = file.read()
+    except FileNotFoundError:
+        raise HTTPException(status_code=404, detail="File not found")
+    except IOError:
+        raise HTTPException(status_code=500, detail="File read error")
+
+    data_object = GetDataObject(
+        file_name=obj["file_name"],
+        file_path=obj["file_path"],
+        json_blob=json_data
+    )
+    return await response_base.success(data=data_object)

toolserve/toolserve/server/routes/log.py

@@ -0,0 +1,40 @@
+
+from typing import Annotated
+from fastapi import APIRouter, Path, Query
+from toolserve.server.schemas.log import CreateLog, LogSchemaBase, GetLogDetails
+from toolserve.server.services.log_service import log_service
+from toolserve.server.common.response import ResponseModel, response_base
+from toolserve.server.common.serializers import select_as_dict
+
+router = APIRouter()
+
+# Get log details by log id
+@router.get('/{pk}', summary='Get log details')
+async def get_log(
+    pk: Annotated[int, Path(...)],
+) -> ResponseModel:
+    log = await log_service.get(pk=pk)
+    data = GetLogDetails(**await select_as_dict(log))
+    return await response_base.success(data=data)
+
+# Create a new log
+@router.post('', summary='Create log')
+async def create_log(
+    obj: CreateLog
+) -> ResponseModel:
+    await log_service.create(obj=obj)
+    return await response_base.success()
+
+# Delete log
+@router.delete('', summary='Delete log')
+async def delete_log(pk: Annotated[list[int], Query(...)]) -> ResponseModel:
+    count = await log_service.delete(pk=pk)
+    if count > 0:
+        return await response_base.success()
+    return await response_base.fail()
+
+# Get all logs
+@router.get('', summary='Get all logs')
+async def get_all_logs() -> ResponseModel:
+    data = await log_service.get_log_list()
+    return await response_base.success(data=data)

toolserve/toolserve/server/routes/tool.py

@@ -6,9 +6,9 @@ from pydantic import ValidationError
 
 from toolserve.server.core.conf import settings
 from toolserve.server.core.depends import get_catalog
-from toolserve.common.response_code import CustomResponseCode
-from toolserve.common.response import ResponseModel, response_base
-#from toolserve.utils.openai_tool import schema_to_openai_tool
+from toolserve.server.common.response_code import CustomResponseCode
+from toolserve.server.common.response import ResponseModel, response_base
+from toolserve.utils.openai_tool import schema_to_openai_tool
 
 router = APIRouter()
 
@@ -17,25 +17,25 @@ router = APIRouter()
     summary='List available tools',
 )
 async def list_tools(catalog=Depends(get_catalog)) -> ResponseModel:
-    """List all available actions"""
+    """List all available tools"""
 
     tools = catalog.list_tools()
     return await response_base.success(data=tools)
 
 @router.get(
     '/oai_function',
-    summary="Get the OpenAI function format of an action"
+    summary="Get the OpenAI function format of an tool"
 )
 async def get_oai_function(
-    action_name: str = Query(..., title="Action Name", description="The name of the action"),
+    tool_name: str = Query(..., title="Tool Name", description="The name of the tool"),
     catalog=Depends(get_catalog)
 ) -> ResponseModel:
-    """Get the OpenAI function format of an action"""
+    """Get the OpenAI function format of an tool"""
 
     try:
         # TODO handle keyerror
-        action = catalog[action_name]
-        json_data = schema_to_openai_tool(action)
+        tool = catalog[tool_name]
+        json_data = schema_to_openai_tool(tool)
 
         return await response_base.success(data=json_data)
     except ValidationError as e:

toolserve/toolserve/server/schemas/artifact.py

@@ -0,0 +1,29 @@
+from datetime import datetime
+
+from pydantic import ConfigDict, Field
+
+from toolserve.server.schemas.base import SchemaBase
+
+
+class ArtifactSchemaBase(SchemaBase):
+    name: str
+    file_path: str
+
+
+class CreateArtifactParam(ArtifactSchemaBase):
+    data: bytes
+    media_type: str
+
+
+class DeleteArtifactParam(ArtifactSchemaBase):
+    pass
+
+class UpdateArtifactParam(ArtifactSchemaBase):
+    pass
+
+class GetArtifactDetails(ArtifactSchemaBase):
+    model_config = ConfigDict(from_attributes=True)
+
+    id: int
+    created_time: datetime
+    updated_time: datetime | None = None

toolserve/toolserve/server/schemas/base.py

@@ -0,0 +1,148 @@
+# -*- coding: utf-8 -*-
+
+from pydantic import BaseModel, ConfigDict, EmailStr, validate_email
+
+# Custom validation error messages do not include the expected content of validation (i.e., input content). For supported expected content fields, refer to the following link:
+# https://github.com/pydantic/pydantic-core/blob/a5cb7382643415b716b1a7a5392914e50f726528/tests/test_errors.py#L266
+# For replacing expected content fields, refer to the following link:
+# https://github.com/pydantic/pydantic/blob/caa78016433ec9b16a973f92f187a7b6bfde6cb5/docs/errors/errors.md?plain=1#L232
+CUSTOM_VALIDATION_ERROR_MESSAGES = {
+    'arguments_type': 'Incorrect argument type input',
+    'assertion_error': 'Assertion execution error',
+    'bool_parsing': 'Boolean value parsing error',
+    'bool_type': 'Boolean type input error',
+    'bytes_too_long': 'Byte length input too long',
+    'bytes_too_short': 'Byte length input too short',
+    'bytes_type': 'Byte type input error',
+    'callable_type': 'Callable object type input error',
+    'dataclass_exact_type': 'Dataclass instance type input error',
+    'dataclass_type': 'Dataclass type input error',
+    'date_from_datetime_inexact': 'Non-zero date component input',
+    'date_from_datetime_parsing': 'Date input parsing error',
+    'date_future': 'Date input is not in the future',
+    'date_parsing': 'Date input validation error',
+    'date_past': 'Date input is not in the past',
+    'date_type': 'Date type input error',
+    'datetime_future': 'Datetime input is not in the future',
+    'datetime_object_invalid': 'Datetime input object invalid',
+    'datetime_parsing': 'Datetime input parsing error',
+    'datetime_past': 'Datetime input is not in the past',
+    'datetime_type': 'Datetime type input error',
+    'decimal_max_digits': 'Decimal input has too many digits',
+    'decimal_max_places': 'Decimal places input error',
+    'decimal_parsing': 'Decimal input parsing error',
+    'decimal_type': 'Decimal type input error',
+    'decimal_whole_digits': 'Decimal whole digits input error',
+    'dict_type': 'Dictionary type input error',
+    'enum': 'Enum member input error, allowed {expected}',
+    'extra_forbidden': 'Extra fields input forbidden',
+    'finite_number': 'Finite value input error',
+    'float_parsing': 'Float parsing error',
+    'float_type': 'Float type input error',
+    'frozen_field': 'Frozen field input error',
+    'frozen_instance': 'Modification of frozen instance forbidden',
+    'frozen_set_type': 'Frozen set type input forbidden',
+    'get_attribute_error': 'Attribute retrieval error',
+    'greater_than': 'Input value too large',
+    'greater_than_equal': 'Input value too large or equal',
+    'int_from_float': 'Integer type input error',
+    'int_parsing': 'Integer input parsing error',
+    'int_parsing_size': 'Integer input parsing size error',
+    'int_type': 'Integer type input error',
+    'invalid_key': 'Invalid key input',
+    'is_instance_of': 'Instance type input error',
+    'is_subclass_of': 'Subclass type input error',
+    'iterable_type': 'Iterable type input error',
+    'iteration_error': 'Iteration value input error',
+    'json_invalid': 'JSON string input error',
+    'json_type': 'JSON type input error',
+    'less_than': 'Input value too small',
+    'less_than_equal': 'Input value too small or equal',
+    'list_type': 'List type input error',
+    'literal_error': 'Literal input error',
+    'mapping_type': 'Mapping type input error',
+    'missing': 'Missing required field',
+    'missing_argument': 'Missing argument',
+    'missing_keyword_only_argument': 'Missing keyword-only argument',
+    'missing_positional_only_argument': 'Missing positional-only argument',
+    'model_attributes_type': 'Model attributes type input error',
+    'model_type': 'Model instance input error',
+    'multiple_argument_values': 'Multiple argument values input',
+    'multiple_of': 'Input value not a multiple',
+    'no_such_attribute': 'Invalid attribute assignment',
+    'none_required': 'Input value must be None',
+    'recursion_loop': 'Recursion loop in input',
+    'set_type': 'Set type input error',
+    'string_pattern_mismatch': 'String pattern mismatch input',
+    'string_sub_type': 'String subtype (non-strict instance) input error',
+    'string_too_long': 'String input too long',
+    'string_too_short': 'String input too short',
+    'string_type': 'String type input error',
+    'string_unicode': 'String input not Unicode',
+    'time_delta_parsing': 'Time delta parsing error',
+    'time_delta_type': 'Time delta type input error',
+    'time_parsing': 'Time input parsing error',
+    'time_type': 'Time type input error',
+    'timezone_aware': 'Missing timezone input',
+    'timezone_naive': 'Timezone input forbidden',
+    'too_long': 'Input too long',
+    'too_short': 'Input too short',
+    'tuple_type': 'Tuple type input error',
+    'unexpected_keyword_argument': 'Unexpected keyword argument input',
+    'unexpected_positional_argument': 'Unexpected positional argument input',
+    'union_tag_invalid': 'Union tag literal input error',
+    'union_tag_not_found': 'Union tag argument not found',
+    'url_parsing': 'URL input parsing error',
+    'url_scheme': 'URL scheme input error',
+    'url_syntax_violation': 'URL syntax violation',
+    'url_too_long': 'URL input too long',
+    'url_type': 'URL type input error',
+    'uuid_parsing': 'UUID parsing error',
+    'uuid_type': 'UUID type input error',
+    'uuid_version': 'UUID version type input error',
+    'value_error': 'Value input error',
+}
+
+CUSTOM_USAGE_ERROR_MESSAGES = {
+    'class-not-fully-defined': 'Class attributes type not fully defined',
+    'custom-json-schema': '__modify_schema__ method deprecated in V2',
+    'decorator-missing-field': 'Invalid field validator defined',
+    'discriminator-no-field': 'Discriminator field not fully defined',
+    'discriminator-alias-type': 'Discriminator field defined using non-string type',
+    'discriminator-needs-literal': 'Discriminator field requires literal definition',
+    'discriminator-alias': 'Inconsistent discriminator field alias definition',
+    'discriminator-validator': 'Field validator forbidden on discriminator field',
+    'model-field-overridden': 'Typeless field override forbidden',
+    'model-field-missing-annotation': 'Missing field type definition',
+    'config-both': 'Duplicate configuration item defined',
+    'removed-kwargs': 'Removed keyword configuration parameter called',
+    'invalid-for-json-schema': 'Invalid JSON type present',
+    'base-model-instantiated': 'Instantiation of base model forbidden',
+    'undefined-annotation': 'Missing type definition',
+    'schema-for-unknown-type': 'Unknown type definition',
+    'create-model-field-definitions': 'Field definition error',
+    'create-model-config-base': 'Configuration item definition error',
+    'validator-no-fields': 'Field validator without specified fields',
+    'validator-invalid-fields': 'Field validator fields definition error',
+    'validator-instance-method': 'Field validator must be a class method',
+    'model-serializer-instance-method': 'Serializer must be an instance method',
+    'validator-v1-signature': 'V1 field validator error deprecated',
+    'validator-signature': 'Field validator signature error',
+    'field-serializer-signature': 'Field serializer signature unrecognized',
+    'model-serializer-signature': 'Model serializer signature unrecognized',
+    'multiple-field-serializers': 'Field serializers defined multiple times',
+    'invalid_annotated_type': 'Invalid type definition',
+    'type-adapter-config-unused': 'Type adapter configuration item definition error',
+    'root-model-extra': 'Extra fields on root model forbidden',
+}
+
+
+
+class CustomEmailStr(EmailStr):
+    @classmethod
+    def _validate(cls, __input_value: str) -> str:
+        return None if __input_value == '' else validate_email(__input_value)[1]
+
+
+class SchemaBase(BaseModel):
+    model_config = ConfigDict(use_enum_values=True)

toolserve/toolserve/server/schemas/data.py

@@ -0,0 +1,46 @@
+
+from datetime import datetime
+
+from pydantic import BaseModel, Field
+
+from toolserve.server.schemas.base import SchemaBase
+
+
+class DataSchemaBase(SchemaBase):
+    """
+    Base schema for Data model.
+    """
+    file_name: str = Field(..., title="File Name", description="Name of the file")
+
+
+class CreateDataParam(DataSchemaBase):
+    """
+    Parameters required to create a Data entry.
+    """
+    json_blob: str = Field(..., title="JSON Blob", description="JSON blob containing the data")
+    file_path: str | None = Field(default=None, title="File Path", description="Path of the file")
+
+
+class DeleteDataParam(DataSchemaBase):
+    """
+    Parameters required to delete a Data entry.
+    """
+    pass
+
+
+class GetDataObject(DataSchemaBase):
+    """
+    Schema to retrieve details of a Data entry.
+    """
+    json_blob: str = Field(..., title="JSON Blob", description="JSON blob containing the data")
+
+
+class GetDataDetails(DataSchemaBase):
+    """
+    Schema to retrieve details of a Data entry.
+    """
+    id: int = Field(..., title="ID", description="Unique identifier of the Data entry")
+    file_path: str = Field(..., title="File Path", description="Path of the file")
+
+    created_time: datetime = Field(..., title="Creation Time", description="Time when the Data entry was created")
+    updated_time: datetime | None = Field(default=None, title="Updated Time", description="Time when the Data entry was last updated")

toolserve/toolserve/server/schemas/log.py

@@ -0,0 +1,22 @@
+from datetime import datetime
+
+from pydantic import ConfigDict, Field
+
+from toolserve.server.schemas.base import SchemaBase
+
+
+class LogSchemaBase(SchemaBase):
+    level: str = Field(..., title='Log level', description='Log level')
+    msg: str = Field(..., title='Log message', description='Log message')
+
+
+class CreateLog(LogSchemaBase):
+    pass
+
+
+class GetLogDetails(LogSchemaBase):
+    model_config = ConfigDict(from_attributes=True)
+
+    id: int
+    created_time: datetime
+    updated_time: datetime | None = None

toolserve/toolserve/server/services/artifact_service.py

@@ -0,0 +1,47 @@
+from typing import Sequence
+
+from sqlalchemy import select
+from toolserve.server.common.exception import errors
+from toolserve.server.crud.crud_artifact import artifact_dao
+from toolserve.server.database.db_sqlite import async_db_session
+from toolserve.server.models.sys_artifact import Artifact
+from toolserve.server.schemas.artifact import CreateArtifactParam, ArtifactSchemaBase
+from toolserve.server.core.conf import settings
+import os
+
+class ArtifactService:
+    @staticmethod
+    async def get(*, pk: int) -> Artifact:
+        async with async_db_session() as db:
+            artifact = await artifact_dao.get(db, pk)
+            if not artifact:
+                raise errors.NotFoundError(msg='Artifact not found')
+            return artifact
+
+    @staticmethod
+    async def get_all_artifacts() -> Sequence[Artifact]:
+        async with async_db_session() as db:
+            artifacts = await artifact_dao.get_all(db)
+            return artifacts
+
+    @staticmethod
+    async def create(*, obj: CreateArtifactParam) -> None:
+        async with async_db_session.begin() as db:
+            artifact = Artifact(name=obj.name, file_path=os.path.join(settings.ARTIFACTS_DIR, obj.name))
+            await artifact_dao.create(db, artifact)  # This line persists the artifact in the database.
+            with open(artifact.file_path, 'wb') as file:
+                file.write(obj.data)
+
+    @staticmethod
+    async def update(*, pk: int, obj: ArtifactSchemaBase) -> int:
+        async with async_db_session.begin() as db:
+            count = await artifact_dao.update(db, pk, obj)
+            return count
+
+    @staticmethod
+    async def delete(*, pk: list[int]) -> int:
+        async with async_db_session.begin() as db:
+            count = await artifact_dao.delete(db, pk)
+            return count
+
+artifact_service: ArtifactService = ArtifactService()

toolserve/toolserve/server/services/data_service.py

@@ -0,0 +1,60 @@
+from typing import Sequence
+
+from sqlalchemy import select
+from toolserve.server.common.exception import errors
+from toolserve.server.crud.crud_data import data_dao
+from toolserve.server.database.db_sqlite import async_db_session
+from toolserve.server.models.sys_data import Data
+from toolserve.server.schemas.data import CreateDataParam, DataSchemaBase
+from toolserve.server.core.conf import settings
+import os
+import json
+
+class DataService:
+    @staticmethod
+    async def get(*, pk: int) -> Data:
+        async with async_db_session() as db:
+            data = await data_dao.get(db, pk)
+            if not data:
+                raise errors.NotFoundError(msg='Data not found')
+            return data
+
+    @staticmethod
+    async def get_all_data() -> Sequence[Data]:
+        async with async_db_session() as db:
+            data_entries = await data_dao.get_all(db)
+            return data_entries
+
+    @staticmethod
+    async def create(*, obj: CreateDataParam) -> Data:
+        async with async_db_session.begin() as db:
+            file_name = obj.file_name + ".json" if not obj.file_name.endswith(".json") else obj.file_name
+            obj.file_path = os.path.join(settings.DATA_DIR, file_name)
+
+            data = obj.copy(exclude={'json_blob'})  # Save everything but the data
+            db_model = await data_dao.create(db, data)  # This now returns the ID
+
+            await db.commit()
+
+        # TODO figure out how to save it so it doesnt overwrite the existing file
+        os.makedirs(os.path.dirname(obj.file_path), exist_ok=True)
+        with open(obj.file_path, 'w') as file:
+            file.write(obj.json_blob)
+
+        return db_model
+
+
+    @staticmethod
+    async def update(*, pk: int, obj: DataSchemaBase) -> int:
+        async with async_db_session.begin() as db:
+            count = await data_dao.update(db, pk, obj)
+            return count
+
+    @staticmethod
+    async def delete(*, pk: list[int]) -> int:
+        async with async_db_session.begin() as db:
+            count = await data_dao.delete(db, pk)
+            return count
+
+
+data_service: DataService = DataService()

toolserve/toolserve/server/services/log_service.py

@@ -0,0 +1,47 @@
+
+from typing import Sequence
+
+from sqlalchemy import Select
+
+from toolserve.server.common.exception import errors
+from toolserve.server.crud.crud_log import log_dao
+from toolserve.server.database.db_sqlite import async_db_session
+from toolserve.server.models.sys_log import Log
+from toolserve.server.schemas.log import CreateLog, LogSchemaBase
+
+
+class LogService:
+    @staticmethod
+    async def get(*, pk: int) -> Log:
+        async with async_db_session() as db:
+            log = await log_dao.get(db, pk)
+            if not log:
+                raise errors.NotFoundError(msg='Log entry not found')
+            return log
+
+
+    @staticmethod
+    async def get_log_list() -> Sequence[Log]:
+        async with async_db_session() as db:
+            logs = await log_dao.get_all(db)
+            return logs
+
+    @staticmethod
+    async def create(*, obj: CreateLog) -> None:
+        async with async_db_session.begin() as db:
+            await log_dao.create(db, obj)
+
+    @staticmethod
+    async def update(*, pk: int, obj: LogSchemaBase) -> int:
+        async with async_db_session.begin() as db:
+            count = await log_dao.update(db, pk, obj)
+            return count
+
+    @staticmethod
+    async def delete(*, pk: list[int]) -> int:
+        async with async_db_session.begin() as db:
+            count = await log_dao.delete(db, pk)
+            return count
+
+
+log_service: LogService = LogService()

toolserve/toolserve/server/utils/timezone.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+import zoneinfo
+
+from datetime import datetime
+
+from toolserve.server.core.conf import settings
+
+
+class TimeZone:
+    def __init__(self, tz: str = settings.DATETIME_TIMEZONE):
+        self.tz_info = zoneinfo.ZoneInfo(tz)
+
+    def now(self) -> datetime:
+        """
+        获取时区时间
+
+        :return:
+        """
+        return datetime.now(self.tz_info)
+
+    def f_datetime(self, dt: datetime) -> datetime:
+        """
+        datetime 时间转时区时间
+
+        :param dt:
+        :return:
+        """
+        return dt.astimezone(self.tz_info)
+
+    def f_str(self, date_str: str, format_str: str = settings.DATETIME_FORMAT) -> datetime:
+        """
+        时间字符串转时区时间
+
+        :param date_str:
+        :param format_str:
+        :return:
+        """
+        return datetime.strptime(date_str, format_str).replace(tzinfo=self.tz_info)
+
+
+timezone = TimeZone()

toolserve/toolserve/utils/openai_tool.py

@@ -1,11 +1,8 @@
 import json
-from typing import Any, Dict, Type, Union, List
-from pydantic import BaseModel, Field
-from datetime import datetime
+from typing import Any, Dict, Type
+from pydantic import BaseModel
 
-from toolserve.server.core.catalog import ParameterSchema
-
-# TODO clean this up
+from toolserve.server.core.catalog import ToolSchema
 
 PYTHON_TO_JSON_TYPES = {
     str: "string",
@@ -26,55 +23,47 @@ def python_type_to_json_type(python_type: Type) -> Dict[str, Any]:
     Returns:
         Dict[str, Any]: A dictionary representing the JSON schema for the given Python type.
     """
-    if hasattr(python_type, '__origin__'):  # For generic types like List[str] or Dict[str, int]
+    if hasattr(python_type, '__origin__'):
         origin = python_type.__origin__
         if origin is list:
             item_type = python_type_to_json_type(python_type.__args__[0])
-            return {'type': 'array', 'items': {'type': item_type}}
+            return {'type': 'array', 'items': item_type}
         elif origin is dict:
-            # Handle dictionary with specific key and value types
             key_type = python_type_to_json_type(python_type.__args__[0])
             value_type = python_type_to_json_type(python_type.__args__[1])
-            return {'type': 'object', 'additionalProperties': {'type': value_type}}
-    #elif issubclass(python_type, BaseModel):  # For Pydantic models
-    #    return model_to_json_schema(python_type)
+            return {'type': 'object', 'additionalProperties': value_type}
     return PYTHON_TO_JSON_TYPES.get(python_type, "string")
 
-def parameter_schema_to_json(parameter_schema: 'ParameterSchema') -> Dict[str, Any]:
-    """Convert a ParameterSchema to a JSON schema property."""
-    property_schema = {
-        "type": python_type_to_json_type(parameter_schema.dtype),
-        "description": parameter_schema.description,
-    }
-    if parameter_schema.default is not None:
-        property_schema["default"] = parameter_schema.default
-    return property_schema
-
 def model_to_json_schema(model: Type[BaseModel]) -> Dict[str, Any]:
-    """Convert a Pydantic model to a JSON schema."""
+    """
+    Convert a Pydantic model to a JSON schema.
+
+    Args:
+        model (Type[BaseModel]): The Pydantic model to convert.
+
+    Returns:
+        Dict[str, Any]: A dictionary representing the JSON schema for the given model.
+    """
     properties = {}
     required = []
     for field_name, model_field in model.model_fields.items():
-        field_schema = parameter_schema_to_json(
-            ParameterSchema(
-                name=field_name,
-                dtype=model_field.annotation,
-                description=model_field.description or "",
-                default=model_field.default,
-                required=model_field.required
-            )
-        )
-        properties[field_name] = field_schema
+        field_schema = {
+            "type": python_type_to_json_type(model_field.annotation),
+            "description": model_field.description or "",
+        }
+        if model_field.default is not None:
+            field_schema["default"] = model_field.default
         if model_field.is_required():
             required.append(field_name)
+        properties[field_name] = field_schema
     return {
         "type": "object",
         "properties": properties,
         "required": required,
     }
 
-def schema_to_openai_tool(action_schema: 'ActionSchema') -> str:
-    """Convert an ActionSchema object to a JSON schema string in the specified function format.
+def schema_to_openai_tool(tool_schema: 'ToolSchema') -> str:
+    """Convert an ToolSchema object to a JSON schema string in the specified function format.
 
     Example output format:
     {
@@ -100,30 +89,18 @@ def schema_to_openai_tool(action_schema: 'ActionSchema') -> str:
     }
 
     Args:
-        action_schema (ActionSchema): The action schema to convert.
+        tool_schema (ToolSchema): The tool schema to convert.
 
     Returns:
-        str: A JSON schema string representing the action in the specified format.
+        str: A JSON schema string representing the tool in the specified format.
     """
-    properties = {}
-    required = []
-    if action_schema.in_schema:
-        for input_param in action_schema.in_schema.inputs:
-            param_schema = parameter_schema_to_json(input_param)
-            properties[input_param.name] = param_schema
-            if input_param.required:
-                required.append(input_param.name)
-
+    input_model_schema = model_to_json_schema(tool_schema.input_model)
     function_schema = {
         "type": "function",
         "function": {
-            "name": action_schema.name,
-            "description": action_schema.description,
-            "parameters": {
-                "type": "object",
-                "properties": properties,
-                "required": required,
-            }
+            "name": tool_schema.name,
+            "description": tool_schema.description,
+            "parameters": input_model_schema,
         }
     }
     return json.dumps(function_schema, indent=2)