diff --git a/toolserve/toolserve/common/__init__.py b/toolserve/toolserve/builtin/default/__init__.py similarity index 100% rename from toolserve/toolserve/common/__init__.py rename to toolserve/toolserve/builtin/default/__init__.py diff --git a/toolserve/toolserve/builtin/default/query.py b/toolserve/toolserve/builtin/default/query.py new file mode 100644 index 00000000..3b9bd1cf --- /dev/null +++ b/toolserve/toolserve/builtin/default/query.py @@ -0,0 +1,119 @@ +from typing import Any, Dict, Optional +import io + +from toolserve.sdk.client import list_data, log +from toolserve.sdk.dataframe import get_df, save_df +from toolserve.sdk.tool import tool, Param +import duckdb +import pandas as pd + +@tool +async def list_data_sources() -> Dict[str, Dict[str, str]]: + """List all data sources. + + Returns: + Dict[str, str]: A dictionary mapping data source IDs to their details. + """ + data = await list_data() + return {str(item["id"]): { + "file_name": item["file_name"], + "created_at": item["created_time"], + "updated_at": item["updated_time"] + } for item in data} + +@tool +async def get_data_schema( + data_id: Param(int, "id of the data source"), + ) -> Param(str, "schema of the data source"): + """Get the schema of the data source by id. + + Args: + data_id (int): The id of the data source to get the schema of. + + Returns: + str: The schema of the data source. + """ + # TODO read in only a few lines + df = await get_df(data_id) + return get_df_info(df) + + +@tool +async def query_sql( + data_id: Param(int, "id of the data source"), + sql: Param(str, "parameterized SQL query to execute"), + params: Param(Optional[Dict[str, Any]], "parameters to pass to the SQL query") = None, + ) -> Param(str, "schema of the data source after executing the query"): + """Query a data source using SQL + + The SQL query should be parameterized with Python's named parameter syntax, + e.g. `SELECT * FROM table WHERE column = :value`. + + After the query, a new data source at a new id will be created with the results and + the schema of the data source will be returned. + + Args: + data_id (int): The id of the data source to query. + sql (str): The parameterized SQL query to execute. + params (Optional[Dict[str, Any]]): Parameters to pass to the SQL query. + + Returns: + str: The schema of the data source after executing the query. + """ + try: + # Retrieve the DataFrame and execute the SQL query using DuckDB + import duckdb + df = await get_df(data_id) + con = duckdb.connect(database=':memory:', read_only=False) + con.register('df_table', df) + if params: + result_df = con.execute(sql, params).fetchdf() + else: + result_df = con.execute(sql).fetchdf() + + # Save the resulting DataFrame and create a new data source + result = await save_df(result_df, f"query_result_{data_id}") + result_id = result["id"] + # Retrieve and return the schema of the new data source + return get_df_info(result_df, data_id=result_id) + + except Exception as e: + # Log the error and raise an exception + await log(f"Failed to execute query: {str(e)}", level="ERROR") + raise RuntimeError(f"Query execution failed: {str(e)}") + + +def get_df_info(df: pd.DataFrame, data_id: Optional[int]=None) -> str: + """ + Generate a compact string representation of a DataFrame including the count of columns, + rows, overall size, and details for each column such as name and datatype. + + Parameters: + df (pd.DataFrame): The Pandas DataFrame to describe. + + Returns: + str: A string that contains the compact representation of the DataFrame. + """ + + # Create an output stream to collect strings + output = io.StringIO() + + # Write general information about the DataFrame + if data_id: + output.write(f"Result Data ID: {data_id}\n") + output.write("Table Name: df\n") + output.write(f"Columns: {len(df.columns)}\n") + output.write(f"Rows: {len(df.index)}\n") + output.write(f"Size: {df.memory_usage(deep=True).sum()} bytes\n") + + # Iterate through each column to get details + for column in df.columns: + output.write("---\n") + output.write(f"Column: {column}\n") + output.write(f"type: {df[column].dtype}\n") + + # Get the complete string from the output stream + result = output.getvalue() + output.close() + + return result \ No newline at end of file diff --git a/toolserve/toolserve/cli/main.py b/toolserve/toolserve/cli/main.py index 0b6d3b45..351579e0 100644 --- a/toolserve/toolserve/cli/main.py +++ b/toolserve/toolserve/cli/main.py @@ -6,7 +6,6 @@ from pathlib import Path from rich.console import Console from rich.markup import escape -from toolserve.common.log import log from toolserve.server.core.conf import settings from toolserve.server.main import app from toolserve.apm.pack import Packer diff --git a/toolserve/toolserve/common/serializers.py b/toolserve/toolserve/common/serializers.py deleted file mode 100644 index affd6034..00000000 --- a/toolserve/toolserve/common/serializers.py +++ /dev/null @@ -1,15 +0,0 @@ -from decimal import Decimal -from typing import Any, Sequence, TypeVar - -import msgspec - -from starlette.responses import JSONResponse - - -class MsgSpecJSONResponse(JSONResponse): - """ - JSON response using the high-performance msgspec library to serialize data to JSON. - """ - - def render(self, content: Any) -> bytes: - return msgspec.json.encode(content) diff --git a/toolserve/toolserve/sdk/client.py b/toolserve/toolserve/sdk/client.py new file mode 100644 index 00000000..e87f19ec --- /dev/null +++ b/toolserve/toolserve/sdk/client.py @@ -0,0 +1,201 @@ + +import os +import json +import httpx + +from typing import Optional, Dict, Any, List +from contextlib import asynccontextmanager +from enum import Enum + + +class HttpClient: + """ + A simple HTTP client class to handle requests to a specified base URL with optional authentication. + """ + def __init__(self, base_url: str, auth_token: Optional[str] = None): + """ + Initializes the HttpClient with a base URL and an optional authentication token. + + Args: + base_url (str): The base URL for the HTTP requests. + auth_token (Optional[str]): Optional bearer token for authorization. + """ + self.base_url = base_url + self.auth_token = auth_token + self.client = httpx.AsyncClient() + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + await self.client.aclose() + + async def post(self, endpoint: str, data: Dict[str, Any], files: Optional[Dict[str, Any]] = None) -> Any: + """ + Sends a POST request to the specified endpoint with the provided data and files. + + Args: + endpoint (str): The endpoint to send the POST request to. + data (Dict[str, Any]): The data to send in the POST request. + files (Optional[Dict[str, Any]]): Optional files to send with the request. + + Returns: + Any: The JSON response from the server. + """ + headers = {"Authorization": f"Bearer {self.auth_token}"} if self.auth_token else {} + try: + if files: + response = await self.client.post(f"{self.base_url}{endpoint}", files=files, headers=headers) + else: + response = await self.client.post(f"{self.base_url}{endpoint}", json=data, headers=headers) + response.raise_for_status() + return response.json() + except httpx.HTTPStatusError as e: + raise RuntimeError(f"HTTP error occurred: {e.response.status_code}") + except httpx.RequestError as e: + raise RuntimeError(f"Request error occurred: {e}") + + async def get(self, endpoint: str, params: Optional[Dict[str, Any]] = None) -> Any: + """ + Sends a GET request to the specified endpoint with optional parameters. + + Args: + endpoint (str): The endpoint to send the GET request to. + params (Optional[Dict[str, Any]]): Optional parameters to include in the request. + + Returns: + Any: The JSON response from the server. + """ + headers = {"Authorization": f"Bearer {self.auth_token}"} if self.auth_token else {} + try: + response = await self.client.get(f"{self.base_url}{endpoint}", params=params, headers=headers) + response.raise_for_status() + return response.json() + except httpx.HTTPStatusError as e: + raise RuntimeError(f"HTTP error occurred: {e.response.status_code}") + except httpx.RequestError as e: + raise RuntimeError(f"Request error occurred: {e}") + +@asynccontextmanager +async def managed_http_client(base_url: str, auth_token: Optional[str] = None): + """ + Context manager to handle the lifecycle of HttpClient instances. + + Args: + base_url (str): The base URL for the HTTP requests. + auth_token (Optional[str]): Optional bearer token for authorization. + """ + client = HttpClient(base_url, auth_token) + try: + yield client + finally: + await client.__aexit__(None, None, None) + +def get_base_url() -> str: + return os.getenv('TOOLSERVE_URL', 'http://localhost:8000') + + +# ----- SDK Functions ----- + + +class LogLevel(Enum): + DEBUG = "DEBUG" + INFO = "INFO" + WARNING = "WARNING" + ERROR = "ERROR" + CRITICAL = "CRITICAL" + +async def log(message: str = "", level: LogLevel = LogLevel.INFO, auth_token: Optional[str] = None, endpoint: str = "/api/v1/log"): + """ + Asynchronously sends a log message to a specified endpoint. + + This function constructs a log entry with a message and a log level, then sends it to the server using the provided endpoint. It uses an HTTP POST request within a managed HTTP client context. + + Args: + message (str): The log message to send. Defaults to an empty string. + level (LogLevel): The severity level of the log message. Defaults to LogLevel.INFO. + auth_token (Optional[str]): An optional authorization token for the request. + endpoint (str): The API endpoint to which the log message is sent. Defaults to "/api/v1/log". + + Returns: + Any: The response from the server as a result of the log message post request. + """ + base_url = get_base_url() + if isinstance(level, str): + level = LogLevel(level) + async with managed_http_client(base_url, auth_token) as client: + log_data = {"msg": message, "level": level.value} + return await client.post(endpoint, data=log_data) + + +async def list_data(auth_token: Optional[str] = None, endpoint: str = "/api/v1/data") -> List[Dict[str, Any]]: + """ + Retrieve a list of data objects from a specified endpoint. + + Args: + auth_token (Optional[str]): Optional authorization token. + endpoint (str): API endpoint to send the request to. Defaults to "/api/v1/data". + + Returns: + Dict[str, Any]: The deserialized JSON data retrieved from the server. + """ + base_url = get_base_url() + async with managed_http_client(base_url, auth_token) as client: + response = await client.get(endpoint) + return response["data"] + +async def get_data(data_id: int, auth_token: Optional[str] = None, endpoint: str = "/api/v1/data/object") -> Any: + """ + Retrieve data object by its primary key from a specified endpoint. + + Args: + data_id (int): The primary key of the data object to retrieve. + auth_token (Optional[str]): Optional authorization token. + endpoint (str): API endpoint to send the request to. Defaults to "/api/v1/data/object". + + Returns: + Any: The deserialized JSON data retrieved from the server. + """ + base_url = get_base_url() + endpoint = f"{endpoint}/{str(data_id)}" # Append the data ID to the endpoint URL + async with managed_http_client(base_url, auth_token) as client: + response = await client.get(endpoint) + json_blob = response["data"].get('json_blob', '{}') + return json.loads(json_blob) + + +async def send_data(name: str, data: Dict[str, Any], auth_token: Optional[str] = None, endpoint: str = "/api/v1/data") -> Dict[str, Any]: + """ + Send data to a specified endpoint, serializing the data into JSON under the key 'json_blob'. + + Args: + data (Dict[str, Any]): Data to be serialized and sent. + auth_token (Optional[str]): Optional authorization token. + endpoint (str): API endpoint to send the data to. + + Returns: + Dict[str, Any]: The response from the server after sending the data. + """ + base_url = get_base_url() + json_blob = json.dumps(data) + payload = {'file_name': name, 'json_blob': json_blob} + async with managed_http_client(base_url, auth_token) as client: + response = await client.post(endpoint, data=payload) + if response["code"] != 200: + raise RuntimeError(f"Failed to send data: {response['msg']}") + else: + return { + "id": response["data"]["id"], + "file_path": response["data"]["file_path"] + } + + +async def save_artifact_from_file(file_path: str = "", auth_token: Optional[str] = None, endpoint: str = "/api/v1/artifact"): + base_url = get_base_url() + async with managed_http_client(base_url, auth_token) as client: + with open(file_path, 'rb') as file: + files = {'file': file} + return await client.post(endpoint, data={}, files=files) + + + diff --git a/toolserve/toolserve/sdk/dataframe.py b/toolserve/toolserve/sdk/dataframe.py new file mode 100644 index 00000000..795be3be --- /dev/null +++ b/toolserve/toolserve/sdk/dataframe.py @@ -0,0 +1,39 @@ + +try: + import pandas as pd +except ImportError: + raise ImportError("Pandas is required for this SDK component. Please install it using `pip install pandas`.") + +from typing import Any, Dict +from toolserve.sdk.client import get_data, send_data + + +async def save_df(df: pd.DataFrame, name: str) -> Dict[str, Any]: + """ + Asynchronously saves a DataFrame to the server by converting it to a dictionary and using the SDK's send_data function. + + Args: + df (pd.DataFrame): The DataFrame to save. + name (str): The name under which the DataFrame should be saved. + + Returns: + Dict[str, Any]: The server's response after saving the DataFrame. + """ + data_dict = df.to_dict(orient='records') + response = await send_data(name=name, data={"data": data_dict}) + return response + + +async def get_df(data_id: int) -> pd.DataFrame: + """ + Asynchronously retrieves a DataFrame from the server using its data ID. + + Args: + data_id (int): The unique identifier for the DataFrame to retrieve. + + Returns: + pd.DataFrame: The DataFrame retrieved from the server. + """ + response = await get_data(data_id=data_id) + df = pd.DataFrame(response['data']) + return df diff --git a/toolserve/toolserve/server/routes/secret.py b/toolserve/toolserve/server/common/__init__.py similarity index 100% rename from toolserve/toolserve/server/routes/secret.py rename to toolserve/toolserve/server/common/__init__.py diff --git a/toolserve/toolserve/server/common/exception/__init__.py b/toolserve/toolserve/server/common/exception/__init__.py new file mode 100644 index 00000000..56fafa58 --- /dev/null +++ b/toolserve/toolserve/server/common/exception/__init__.py @@ -0,0 +1,2 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- diff --git a/toolserve/toolserve/server/common/exception/errors.py b/toolserve/toolserve/server/common/exception/errors.py new file mode 100644 index 00000000..cd3072ac --- /dev/null +++ b/toolserve/toolserve/server/common/exception/errors.py @@ -0,0 +1,79 @@ + +from typing import Any + +from fastapi import HTTPException +from starlette.background import BackgroundTask + +from toolserve.server.common.response_code import CustomErrorCode, StandardResponseCode + + +class BaseExceptionMixin(Exception): + code: int + + def __init__(self, *, msg: str = None, data: Any = None, background: BackgroundTask | None = None): + self.msg = msg + self.data = data + # The original background task: https://www.starlette.io/background/ + self.background = background + + +class HTTPError(HTTPException): + def __init__(self, *, code: int, msg: Any = None, headers: dict[str, Any] | None = None): + super().__init__(status_code=code, detail=msg, headers=headers) + + +class CustomError(BaseExceptionMixin): + def __init__(self, *, error: CustomErrorCode, data: Any = None, background: BackgroundTask | None = None): + self.code = error.code + super().__init__(msg=error.msg, data=data, background=background) + + +class RequestError(BaseExceptionMixin): + code = StandardResponseCode.HTTP_400 + + def __init__(self, *, msg: str = 'Bad Request', data: Any = None, background: BackgroundTask | None = None): + super().__init__(msg=msg, data=data, background=background) + + +class ForbiddenError(BaseExceptionMixin): + code = StandardResponseCode.HTTP_403 + + def __init__(self, *, msg: str = 'Forbidden', data: Any = None, background: BackgroundTask | None = None): + super().__init__(msg=msg, data=data, background=background) + + +class NotFoundError(BaseExceptionMixin): + code = StandardResponseCode.HTTP_404 + + def __init__(self, *, msg: str = 'Not Found', data: Any = None, background: BackgroundTask | None = None): + super().__init__(msg=msg, data=data, background=background) + + +class ServerError(BaseExceptionMixin): + code = StandardResponseCode.HTTP_500 + + def __init__( + self, *, msg: str = 'Internal Server Error', data: Any = None, background: BackgroundTask | None = None + ): + super().__init__(msg=msg, data=data, background=background) + + +class GatewayError(BaseExceptionMixin): + code = StandardResponseCode.HTTP_502 + + def __init__(self, *, msg: str = 'Bad Gateway', data: Any = None, background: BackgroundTask | None = None): + super().__init__(msg=msg, data=data, background=background) + + +class AuthorizationError(BaseExceptionMixin): + code = StandardResponseCode.HTTP_401 + + def __init__(self, *, msg: str = 'Permission Denied', data: Any = None, background: BackgroundTask | None = None): + super().__init__(msg=msg, data=data, background=background) + + +class TokenError(HTTPError): + code = StandardResponseCode.HTTP_401 + + def __init__(self, *, msg: str = 'Not Authenticated', headers: dict[str, Any] | None = None): + super().__init__(code=self.code, msg=msg, headers=headers or {'WWW-Authenticate': 'Bearer'}) diff --git a/toolserve/toolserve/server/common/exception/exception_handler.py b/toolserve/toolserve/server/common/exception/exception_handler.py new file mode 100644 index 00000000..d2a0c38c --- /dev/null +++ b/toolserve/toolserve/server/common/exception/exception_handler.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +from fastapi import FastAPI, Request +from fastapi.exceptions import RequestValidationError +from pydantic import ValidationError +from pydantic.errors import PydanticUserError +from starlette.exceptions import HTTPException +from starlette.middleware.cors import CORSMiddleware + +from toolserve.server.common.exception.errors import BaseExceptionMixin +from toolserve.server.common.log import log +from toolserve.server.common.response_code import CustomResponseCode, StandardResponseCode +from toolserve.server.common.response_code import response_base +from toolserve.server.core.conf import settings +from toolserve.server.schemas.base import ( + CUSTOM_USAGE_ERROR_MESSAGES, + CUSTOM_VALIDATION_ERROR_MESSAGES, +) +from toolserve.server.utils.serializers import MsgSpecJSONResponse + + +async def _validation_exception_handler(request: Request, e: RequestValidationError | ValidationError): + """ + Data validation exception handling + + :param e: + :return: + """ + error = e.errors()[0] + if error.get('type') == 'json_invalid': + message = 'JSON parsing failed' + else: + error_input = error.get('input') + field = str(error.get('loc')[-1]) + error_msg = error.get('msg') + message = f'{field} {error_msg}, input: {error_input}' + msg = f'Invalid request parameters: {message}' + data = {'errors': error} if settings.ENVIRONMENT == 'dev' else None + content = { + 'code': StandardResponseCode.HTTP_422, + 'msg': msg, + 'data': data, + } + request.state.__request_validation_exception__ = content # For obtaining exception information in middleware + return MsgSpecJSONResponse(status_code=422, content=content) + + +def register_exception(app: FastAPI): + @app.exception_handler(HTTPException) + async def http_exception_handler(request: Request, exc: HTTPException): + """ + Global HTTP exception handling + + :param request: + :param exc: + :return: + """ + if settings.ENVIRONMENT == 'dev': + content = { + 'code': exc.status_code, + 'msg': exc.detail, + 'data': None, + } + else: + res = await response_base.fail(res=CustomResponseCode.HTTP_400) + content = res.model_dump() + request.state.__request_http_exception__ = content # For obtaining exception information in middleware + return MsgSpecJSONResponse( + status_code=StandardResponseCode.HTTP_400, + content=content, + headers=exc.headers, + ) + + @app.exception_handler(RequestValidationError) + async def fastapi_validation_exception_handler(request: Request, exc: RequestValidationError): + """ + FastAPI data validation exception handling + + :param request: + :param exc: + :return: + """ + return await _validation_exception_handler(request, exc) + + @app.exception_handler(ValidationError) + async def pydantic_validation_exception_handler(request: Request, exc: ValidationError): + """ + Pydantic data validation exception handling + + :param request: + :param exc: + :return: + """ + return await _validation_exception_handler(request, exc) + + @app.exception_handler(PydanticUserError) + async def pydantic_user_error_handler(request: Request, exc: PydanticUserError): + """ + Pydantic user exception handling + + :param request: + :param exc: + :return: + """ + return MsgSpecJSONResponse( + status_code=StandardResponseCode.HTTP_500, + content={ + 'code': StandardResponseCode.HTTP_500, + 'msg': CUSTOM_USAGE_ERROR_MESSAGES.get(exc.code), + 'data': None, + }, + ) + + @app.exception_handler(AssertionError) + async def assertion_error_handler(request: Request, exc: AssertionError): + """ + Assertion error handling + + :param request: + :param exc: + :return: + """ + if settings.ENVIRONMENT == 'dev': + content = { + 'code': StandardResponseCode.HTTP_500, + 'msg': str(''.join(exc.args) if exc.args else exc.__doc__), + 'data': None, + } + else: + res = await response_base.fail(res=CustomResponseCode.HTTP_500) + content = res.model_dump() + return MsgSpecJSONResponse( + status_code=StandardResponseCode.HTTP_500, + content=content, + ) + + @app.exception_handler(Exception) + async def all_exception_handler(request: Request, exc: Exception): + """ + Global exception handling + + :param request: + :param exc: + :return: + """ + if isinstance(exc, BaseExceptionMixin): + return MsgSpecJSONResponse( + status_code=StandardResponseCode.HTTP_400, + content={ + 'code': exc.code, + 'msg': str(exc.msg), + 'data': exc.data if exc.data else None, + }, + background=exc.background, + ) + else: + import traceback + + log.error(f'Unknown exception: {exc}') + log.error(traceback.format_exc()) + if settings.ENVIRONMENT == 'dev': + content = { + 'code': 500, + 'msg': str(exc), + 'data': None, + } + else: + res = await response_base.fail(res=CustomResponseCode.HTTP_500) + content = res.model_dump() + return MsgSpecJSONResponse(status_code=StandardResponseCode.HTTP_500, content=content) + + if settings.MIDDLEWARE_CORS: + + @app.exception_handler(StandardResponseCode.HTTP_500) + async def cors_status_code_500_exception_handler(request, exc): + """ + CORS 500 exception handling + + `Related issue `_ + + :param request: + :param exc: + :return: + """ + if isinstance(exc, BaseExceptionMixin): + content = { + 'code': exc.code, + 'msg': exc.msg, + 'data': exc.data, + } + else: + if settings.ENVIRONMENT == 'dev': + content = { + 'code': StandardResponseCode.HTTP_500, + 'msg': str(exc), + 'data': None, + } + else: + res = await response_base.fail(res=CustomResponseCode.HTTP_500) + content = res.model_dump() + response = MsgSpecJSONResponse( + status_code=exc.code if isinstance(exc, BaseExceptionMixin) else StandardResponseCode.HTTP_500, + content=content, + background=exc.background if isinstance(exc, BaseExceptionMixin) else None, + ) + origin = request.headers.get('origin') + if origin: + cors = CORSMiddleware( + app=app, + allow_origins=['*'], + allow_credentials=True, + allow_methods=['*'], + allow_headers=['*'], + ) + response.headers.update(cors.simple_headers) + has_cookie = 'cookie' in request.headers + if cors.allow_all_origins and has_cookie: + response.headers['Access-Control-Allow-Origin'] = origin + elif not cors.allow_all_origins and cors.is_allowed_origin(origin=origin): + response.headers['Access-Control-Allow-Origin'] = origin + response.headers.add_vary_header('Origin') + return response diff --git a/toolserve/toolserve/common/log.py b/toolserve/toolserve/server/common/log.py similarity index 100% rename from toolserve/toolserve/common/log.py rename to toolserve/toolserve/server/common/log.py diff --git a/toolserve/toolserve/common/response.py b/toolserve/toolserve/server/common/response.py similarity index 96% rename from toolserve/toolserve/common/response.py rename to toolserve/toolserve/server/common/response.py index e158cc40..642129df 100644 --- a/toolserve/toolserve/common/response.py +++ b/toolserve/toolserve/server/common/response.py @@ -5,7 +5,7 @@ from typing import Any from pydantic import BaseModel, ConfigDict -from toolserve.common.response_code import CustomResponse, CustomResponseCode +from toolserve.server.common.response_code import CustomResponse, CustomResponseCode from toolserve.server.core.conf import settings _ExcludeData = set[int | str] | dict[int | str, Any] diff --git a/toolserve/toolserve/common/response_code.py b/toolserve/toolserve/server/common/response_code.py similarity index 100% rename from toolserve/toolserve/common/response_code.py rename to toolserve/toolserve/server/common/response_code.py diff --git a/toolserve/toolserve/server/common/serializers.py b/toolserve/toolserve/server/common/serializers.py new file mode 100644 index 00000000..0f4b928c --- /dev/null +++ b/toolserve/toolserve/server/common/serializers.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +from decimal import Decimal +from typing import Any, Sequence, TypeVar + +import msgspec + +from asgiref.sync import sync_to_async +from sqlalchemy import Row, RowMapping +from starlette.responses import JSONResponse + +RowData = Row | RowMapping | Any + +R = TypeVar('R', bound=RowData) + + +@sync_to_async +def select_columns_serialize(row: R) -> dict: + """ + Serialize SQLAlchemy select table columns, does not contain relational columns + + :param row: + :return: + """ + obj_dict = {} + for column in row.__table__.columns.keys(): + val = getattr(row, column) + if isinstance(val, Decimal): + if val % 1 == 0: + val = int(val) + val = float(val) + obj_dict[column] = val + return obj_dict + + +async def select_list_serialize(row: Sequence[R]) -> list: + """ + Serialize SQLAlchemy select list + + :param row: + :return: + """ + ret_list = [await select_columns_serialize(_) for _ in row] + return ret_list + + +@sync_to_async +def select_as_dict(row: R) -> dict: + """ + Converting SQLAlchemy select to dict, which can contain relational data, + depends on the properties of the select object itself + + :param row: + :return: + """ + obj_dict = row.__dict__ + if '_sa_instance_state' in obj_dict: + del obj_dict['_sa_instance_state'] + return obj_dict + + +class MsgSpecJSONResponse(JSONResponse): + """ + JSON response using the high-performance msgspec library to serialize data to JSON. + """ + + def render(self, content: Any) -> bytes: + return msgspec.json.encode(content) diff --git a/toolserve/toolserve/server/core/catalog.py b/toolserve/toolserve/server/core/catalog.py index 291a10bb..710c24ec 100644 --- a/toolserve/toolserve/server/core/catalog.py +++ b/toolserve/toolserve/server/core/catalog.py @@ -13,8 +13,8 @@ from pydantic import BaseModel, ValidationError, Field, create_model from importlib import import_module from toolserve.server.core.conf import settings -from toolserve.common.response_code import CustomResponseCode -from toolserve.common.response import ResponseModel, response_base +from toolserve.server.common.response_code import CustomResponseCode +from toolserve.server.common.response import ResponseModel, response_base from toolserve.apm.base import ToolPack from toolserve.sdk import Param from toolserve.utils import snake_to_camel @@ -41,6 +41,7 @@ class ToolSchema(BaseModel): class ToolCatalog: def __init__(self, tools_dir: str = settings.TOOLS_DIR): self.tools = self.read_tools(tools_dir) + self.tools.update(self.__get_builitin_tools()) @staticmethod def read_tools(directory: str) -> List[ToolSchema]: @@ -76,10 +77,39 @@ class ToolCatalog: return tools + def __get_builitin_tools(self) -> Dict[str, ToolSchema]: + tools = {} + sys.path.append(str(settings.BUILTIN_TOOLS_DIR)) + + for tool_spec in settings.BUILTIN_TOOLS: + print(tool_spec) + + module_name, versioned_tool = tool_spec.split('.', 1) + func_name, version = versioned_tool.split('@') + + module = import_module(module_name) + tool = getattr(module, func_name) + + input_model, output_model = create_func_models(tool) + response_model = create_response_model(func_name, output_model) + tool_schema = ToolSchema( + name=func_name, + description=tool.__doc__, + version='builtin', + tool=tool, + input_model=input_model, + output_model=response_model, + meta=ToolMeta(module=module_name, path=module.__file__) + ) + tools[func_name] = tool_schema + + return tools + + def __getitem__(self, name: str) -> Optional[ToolSchema]: #TODO error handling - for tool in self.tools: - if tool.name == name: + for tool_name, tool in self.tools.items(): + if tool_name == name: return tool return None @@ -90,7 +120,7 @@ class ToolCatalog: return None def list_tools(self) -> List[Dict[str, str]]: - return [{'name': t.name, 'description': t.description} for t in self.tools] + return [{'name': t.name, 'description': t.description} for t in self.tools.values()] @@ -160,10 +190,11 @@ def determine_output_model(func: Callable) -> Type[BaseModel]: if return_annotation is inspect.Signature.empty: return create_model(f"{snake_to_camel(func.__name__)}Output") elif hasattr(return_annotation, '__origin__'): - field_type = Optional[return_annotation.__args__[0]] - description = return_annotation.__metadata__[0] if return_annotation.__metadata__ else "" - if description: - return create_model(f"{snake_to_camel(func.__name__)}Output", result=(field_type, Field(description=str(description)))) + if hasattr(return_annotation, '__metadata__'): + field_type = Optional[return_annotation.__args__[0]] + description = return_annotation.__metadata__[0] if return_annotation.__metadata__ else "" + if description: + return create_model(f"{snake_to_camel(func.__name__)}Output", result=(field_type, Field(description=str(description)))) else: return create_model(f"{snake_to_camel(func.__name__)}Output", result=(return_annotation, Field(description="No description provided."))) @@ -173,7 +204,7 @@ def create_response_model(name: str, output_model: Type[BaseModel]) -> Type[Resp """ # Create a new response model response_model = create_model( - f"{name}Response", + f"{snake_to_camel(name)}Response", code=(int, CustomResponseCode.HTTP_200.code), msg=(str, CustomResponseCode.HTTP_200.msg), data=(Optional[output_model], None) diff --git a/toolserve/toolserve/server/core/conf.py b/toolserve/toolserve/server/core/conf.py index 5b461ece..ba12b0ec 100644 --- a/toolserve/toolserve/server/core/conf.py +++ b/toolserve/toolserve/server/core/conf.py @@ -14,8 +14,17 @@ from pydantic_settings import BaseSettings, SettingsConfigDict class Settings(BaseSettings): model_config = SettingsConfigDict(env_file='.env') - WORK_DIR: Path = Path.home() / '.darkstar' + WORK_DIR: Path = Path.home() / '.arcade' TOOLS_DIR: Path = os.getcwd() + ARTIFACTS_DIR: Path = WORK_DIR / 'artifacts' + DATA_DIR: Path = WORK_DIR / 'data' + + BUILTIN_TOOLS_DIR: Path = Path(__file__).parent.parent.parent / 'builtin' / 'default' + BUILTIN_TOOLS: list[str] = [ + "query.list_data_sources@builtin", + "query.get_data_schema@builtin", + "query.query_sql@builtin", + ] # Env Config ENVIRONMENT: Literal['dev', 'pro'] = 'dev' @@ -34,10 +43,10 @@ class Settings(BaseSettings): # FastAPI API_V1_STR: str = '/api/v1' - API_ACTION_STR: str = '/action' - TITLE: str = 'Darkstar Toolserver' + API_ACTION_STR: str = '/tool' + TITLE: str = 'Arcade AI Toolserver' VERSION: str = '0.1.0' - DESCRIPTION: str = 'Darkstar Toolserver API' + DESCRIPTION: str = 'Arcade AI Toolserver API' DOCS_URL: str | None = f'{API_V1_STR}/docs' REDOCS_URL: str | None = f'{API_V1_STR}/redocs' OPENAPI_URL: str | None = f'{API_V1_STR}/openapi' @@ -88,7 +97,15 @@ class Settings(BaseSettings): TOKEN_SECRET_KEY: str = "secret" OPERA_LOG_ENCRYPT_SECRET_KEY: str = "secret" + # SQL Database + DB_HOST: str = "localhost" + DB_PORT: int = "3306" + DB_USER: str = "arcade" + DB_PASSWORD: str = "arcade" + DB_ECHO: bool = False + DB_DATABASE: str = 'arcade' + DB_CHARSET: str = 'utf8mb4' @lru_cache def get_settings(): diff --git a/toolserve/toolserve/server/core/generate.py b/toolserve/toolserve/server/core/generate.py index e13e5c74..7efe7f04 100644 --- a/toolserve/toolserve/server/core/generate.py +++ b/toolserve/toolserve/server/core/generate.py @@ -1,5 +1,6 @@ import os import sys +import traceback import inspect from textwrap import dedent from typing import List, Optional, Type, Annotated, Dict @@ -11,8 +12,8 @@ from importlib import import_module from toolserve.server.core.catalog import ToolSchema from toolserve.server.core.conf import settings -from toolserve.common.response_code import CustomResponseCode -from toolserve.common.response import ResponseModel, response_base +from toolserve.server.common.response_code import CustomResponseCode +from toolserve.server.common.response import ResponseModel, response_base def create_endpoint_function(name, description, func, input_model, output_model): @@ -28,7 +29,6 @@ def create_endpoint_function(name, description, func, input_model, output_model) except ValidationError as e: return await response_base.error(res=CustomResponseCode.HTTP_400, msg=str(e)) except Exception as e: - import traceback print(traceback.format_exc()) return await response_base.error(res=CustomResponseCode.HTTP_500, msg=str(e)) diff --git a/toolserve/toolserve/server/core/registrar.py b/toolserve/toolserve/server/core/registrar.py index 36cb5617..73e834dd 100644 --- a/toolserve/toolserve/server/core/registrar.py +++ b/toolserve/toolserve/server/core/registrar.py @@ -5,8 +5,22 @@ from contextlib import asynccontextmanager from fastapi import Depends, FastAPI from toolserve.server.routes import v1 +from toolserve.server.database.db_sqlite import create_table from toolserve.server.core.conf import settings -from toolserve.common.serializers import MsgSpecJSONResponse +from toolserve.server.common.serializers import MsgSpecJSONResponse + + +@asynccontextmanager +async def register_init(app: FastAPI): + """ + + :return: + """ + # create database tables + await create_table() + + yield + def register_app(): # FastAPI @@ -18,6 +32,7 @@ def register_app(): redoc_url=settings.REDOCS_URL, openapi_url=settings.OPENAPI_URL, default_response_class=MsgSpecJSONResponse, + lifespan=register_init, ) register_static_file(app) diff --git a/toolserve/toolserve/server/crud/__init__.py b/toolserve/toolserve/server/crud/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/toolserve/toolserve/server/crud/base.py b/toolserve/toolserve/server/crud/base.py new file mode 100644 index 00000000..433bee96 --- /dev/null +++ b/toolserve/toolserve/server/crud/base.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +from typing import Any, Dict, Generic, Type, TypeVar + +from pydantic import BaseModel +from sqlalchemy import and_, delete, select, update +from sqlalchemy.ext.asyncio import AsyncSession + +from toolserve.server.models.base import MappedBase + +ModelType = TypeVar('ModelType', bound=MappedBase) +CreateSchemaType = TypeVar('CreateSchemaType', bound=BaseModel) +UpdateSchemaType = TypeVar('UpdateSchemaType', bound=BaseModel) + + +class CRUDBase(Generic[ModelType, CreateSchemaType, UpdateSchemaType]): + def __init__(self, model: Type[ModelType]): + self.model = model + + async def get_( + self, + db: AsyncSession, + *, + pk: int | None = None, + name: str | None = None, + status: int | None = None, + del_flag: int | None = None, + ) -> ModelType | None: + """ + Get a record by primary key id or name + + :param db: + :param pk: + :param name: + :param status: + :param del_flag: + :return: + """ + assert pk is not None or name is not None, 'Query error, pk and name parameters cannot be empty at the same time' + assert pk is None or name is None, 'Query error, pk and name parameters cannot exist at the same time' + where_list = [self.model.id == pk] if pk is not None else [self.model.name == name] + if status is not None: + assert status in (0, 1), 'Query error, status parameter can only be 0 or 1' + where_list.append(self.model.status == status) + if del_flag is not None: + assert del_flag in (0, 1), 'Query error, del_flag parameter can only be 0 or 1' + where_list.append(self.model.del_flag == del_flag) + + result = await db.execute(select(self.model).where(and_(*where_list))) + return result.scalars().first() + + async def create_(self, db: AsyncSession, obj_in: CreateSchemaType, user_id: int | None = None) -> None: + """ + Add a new record + + :param db: + :param obj_in: Pydantic model class + :param user_id: + :return: + """ + if user_id: + create_data = self.model(**obj_in.model_dump(), create_user=user_id) + else: + create_data = self.model(**obj_in.model_dump()) + db.add(create_data) + + async def update_( + self, db: AsyncSession, pk: int, obj_in: UpdateSchemaType | Dict[str, Any], user_id: int | None = None + ) -> int: + """ + Update a record by primary key id + + :param db: + :param pk: + :param obj_in: Pydantic model class or dictionary corresponding to database fields + :param user_id: + :return: + """ + if isinstance(obj_in, dict): + update_data = obj_in + else: + update_data = obj_in.model_dump(exclude_unset=True) + if user_id: + update_data.update({'update_user': user_id}) + result = await db.execute(update(self.model).where(self.model.id == pk).values(**update_data)) + return result.rowcount + + async def delete_(self, db: AsyncSession, pk: int, *, del_flag: int | None = None) -> int: + """ + Delete a record by primary key id + + :param db: + :param pk: + :param del_flag: + :return: + """ + if del_flag is None: + result = await db.execute(delete(self.model).where(self.model.id == pk)) + else: + assert del_flag == 1, 'Delete error, del_flag parameter can only be 1' + result = await db.execute(update(self.model).where(self.model.id == pk).values(del_flag=del_flag)) + return result.rowcount diff --git a/toolserve/toolserve/server/crud/crud_artifact.py b/toolserve/toolserve/server/crud/crud_artifact.py new file mode 100644 index 00000000..021a6e46 --- /dev/null +++ b/toolserve/toolserve/server/crud/crud_artifact.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +from typing import Sequence + +from sqlalchemy import Select, and_, delete, desc, select +from sqlalchemy.ext.asyncio import AsyncSession + +from toolserve.server.crud.base import CRUDBase +from toolserve.server.models.sys_artifact import Artifact +from toolserve.server.schemas.artifact import CreateArtifactParam, UpdateArtifactParam + + +class CRUDArtifact(CRUDBase[Artifact, CreateArtifactParam, UpdateArtifactParam]): + async def get(self, db: AsyncSession, pk: int) -> Artifact | None: + return await self.get_(db, pk=pk) + + async def get_list(self, name: str = None, file_path: str = None) -> Select: + se = select(self.model).order_by(desc(self.model.created_time)) + where_list = [] + if name: + where_list.append(self.model.name.like(f'%{name}%')) + if file_path: + where_list.append(self.model.file_path.like(f'%{file_path}%', escape='/')) + if where_list: + se = se.where(and_(*where_list)) + return se + + async def get_all(self, db: AsyncSession) -> Sequence[Artifact]: + artifacts = await db.execute(select(self.model)) + return artifacts.scalars().all() + + async def get_by_name(self, db: AsyncSession, name: str) -> Artifact | None: + artifact = await db.execute(select(self.model).where(self.model.name == name)) + return artifact.scalars().first() + + async def create(self, db: AsyncSession, obj_in: CreateArtifactParam) -> None: + await self.create_(db, obj_in) + + async def update(self, db: AsyncSession, pk: int, obj_in: UpdateArtifactParam) -> int: + return await self.update_(db, pk, obj_in) + + async def delete(self, db: AsyncSession, pk: list[int]) -> int: + artifacts = await db.execute(delete(self.model).where(self.model.id.in_(pk))) + return artifacts.rowcount + + +artifact_dao: CRUDArtifact = CRUDArtifact(Artifact) diff --git a/toolserve/toolserve/server/crud/crud_data.py b/toolserve/toolserve/server/crud/crud_data.py new file mode 100644 index 00000000..bf0ecf32 --- /dev/null +++ b/toolserve/toolserve/server/crud/crud_data.py @@ -0,0 +1,51 @@ +import datetime +from typing import Sequence +from sqlalchemy import Select, and_, delete, desc, select +from sqlalchemy.ext.asyncio import AsyncSession + +from toolserve.server.crud.base import CRUDBase +from toolserve.server.models.sys_data import Data +from toolserve.server.schemas.data import CreateDataParam, DataSchemaBase + +class CRUDData(CRUDBase[Data, CreateDataParam, DataSchemaBase]): + async def get(self, db: AsyncSession, pk: int) -> Data | None: + return await self.get_(db, pk=pk) + + async def get_list(self, file_name: str = None, file_path: str = None) -> Select: + query = select(self.model).order_by(desc(self.model.created_time)) + conditions = [] + if file_name: + conditions.append(self.model.file_name.like(f'%{file_name}%')) + if file_path: + conditions.append(self.model.file_path.like(f'%{file_path}%')) + if conditions: + query = query.where(and_(*conditions)) + return query + + async def get_all(self, db: AsyncSession) -> Sequence[Data]: + result = await db.execute(select(self.model)) + return result.scalars().all() + + async def get_by_file_name(self, db: AsyncSession, file_name: str) -> Data | None: + result = await db.execute(select(self.model).where(self.model.file_name == file_name)) + return result.scalars().first() + + async def create(self, db: AsyncSession, obj_in: CreateDataParam) -> Data: + existing_data = await self.get_by_file_name(db, obj_in.file_name) + if existing_data: + existing_data.updated_time = datetime.datetime.now() + db.add(existing_data) + return existing_data + else: + obj = self.model(**obj_in.dict()) + db.add(obj) + return obj + + async def update(self, db: AsyncSession, pk: int, obj_in: DataSchemaBase) -> int: + return await self.update_(db, pk, obj_in) + + async def delete(self, db: AsyncSession, pk: list[int]) -> int: + result = await db.execute(delete(self.model).where(self.model.id.in_(pk))) + return result.rowcount + +data_dao: CRUDData = CRUDData(Data) diff --git a/toolserve/toolserve/server/crud/crud_log.py b/toolserve/toolserve/server/crud/crud_log.py new file mode 100644 index 00000000..3167ddd8 --- /dev/null +++ b/toolserve/toolserve/server/crud/crud_log.py @@ -0,0 +1,45 @@ +from typing import Sequence + +from sqlalchemy import Select, and_, delete, desc, select +from sqlalchemy.ext.asyncio import AsyncSession + +from toolserve.server.crud.base import CRUDBase +from toolserve.server.models.sys_log import Log +from toolserve.server.schemas.log import CreateLog, LogSchemaBase + + +class CRUDLog(CRUDBase[Log, CreateLog, LogSchemaBase]): + async def get(self, db: AsyncSession, pk: int) -> Log | None: + return await self.get_(db, pk=pk) + + async def get_list(self, level: str = None, msg: str = None) -> Select: + query = select(self.model).order_by(desc(self.model.created_time)) + conditions = [] + if level: + conditions.append(self.model.level == level) + if msg: + conditions.append(self.model.msg.like(f'%{msg}%')) + if conditions: + query = query.where(and_(*conditions)) + return query + + async def get_all(self, db: AsyncSession) -> Sequence[Log]: + result = await db.execute(select(self.model)) + return result.scalars().all() + + async def get_by_level(self, db: AsyncSession, level: str) -> Log | None: + result = await db.execute(select(self.model).where(self.model.level == level)) + return result.scalars().first() + + async def create(self, db: AsyncSession, obj_in: CreateLog) -> None: + await self.create_(db, obj_in) + + async def update(self, db: AsyncSession, pk: int, obj_in: LogSchemaBase) -> int: + return await self.update_(db, pk, obj_in) + + async def delete(self, db: AsyncSession, pk: list[int]) -> int: + result = await db.execute(delete(self.model).where(self.model.id.in_(pk))) + return result.rowcount + + +log_dao: CRUDLog = CRUDLog(Log) diff --git a/toolserve/toolserve/server/database/__init__.py b/toolserve/toolserve/server/database/__init__.py new file mode 100644 index 00000000..56fafa58 --- /dev/null +++ b/toolserve/toolserve/server/database/__init__.py @@ -0,0 +1,2 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- diff --git a/toolserve/toolserve/server/database/db_sqlite.py b/toolserve/toolserve/server/database/db_sqlite.py new file mode 100644 index 00000000..09882786 --- /dev/null +++ b/toolserve/toolserve/server/database/db_sqlite.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +import sys + +from typing import Annotated +from uuid import uuid4 + +from fastapi import Depends +from sqlalchemy import URL +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine + +from toolserve.server.common.log import log +from toolserve.server.core.conf import settings +from toolserve.server.models import ( + MappedBase, + Log, + Artifact, + Data, +) + + +def create_engine_and_session(url: str | URL): + try: + engine = create_async_engine(url, echo=settings.DB_ECHO, future=True, pool_pre_ping=True) + except Exception as e: + log.error('❌ Error starting db session {}', e) + sys.exit() + else: + db_session = async_sessionmaker(bind=engine, autoflush=False, expire_on_commit=False) + return engine, db_session + + +SQLALCHEMY_DATABASE_URL = ( + f'sqlite+aiosqlite:///{settings.WORK_DIR}/{settings.DB_DATABASE}.db' +) + +async_engine, async_db_session = create_engine_and_session(SQLALCHEMY_DATABASE_URL) + + +async def get_db() -> AsyncSession: + """Provide a database session for a single request.""" + session = async_db_session() + try: + yield session + except Exception as se: + await session.rollback() + raise se + finally: + await session.close() + + +# Session Annotated +CurrentSession = Annotated[AsyncSession, Depends(get_db)] + + +async def create_table(): + """Create the database tables if they do not exist.""" + async with async_engine.begin() as conn: + try: + await conn.run_sync(MappedBase.metadata.create_all) + except Exception as e: + log.error('❌ Error creating tables {}', e) + raise e + + +def uuid4_str() -> str: + """Generate a UUID4 string.""" + return str(uuid4()) diff --git a/toolserve/toolserve/server/main.py b/toolserve/toolserve/server/main.py index a407e838..3069bb77 100644 --- a/toolserve/toolserve/server/main.py +++ b/toolserve/toolserve/server/main.py @@ -2,7 +2,7 @@ import uvicorn from pathlib import Path import asyncio -from toolserve.common.log import log +from toolserve.server.common.log import log from toolserve.server.core.conf import settings from toolserve.server.core.registrar import register_app @@ -12,7 +12,7 @@ app = register_app() if __name__ == '__main__': try: log.info( - "Darkstar Toolserve is starting..." + "Arcade AI Toolserve is starting..." ) uvicorn.run( app=f'{Path(__file__).stem}:app', diff --git a/toolserve/toolserve/server/models/__init__.py b/toolserve/toolserve/server/models/__init__.py new file mode 100644 index 00000000..300d0090 --- /dev/null +++ b/toolserve/toolserve/server/models/__init__.py @@ -0,0 +1,7 @@ + +from toolserve.server.models.base import MappedBase +from toolserve.server.models.sys_data import Data +from toolserve.server.models.sys_log import Log +from toolserve.server.models.sys_artifact import Artifact + +__all__ = ['MappedBase', 'Data', 'Log', 'Artifact'] diff --git a/toolserve/toolserve/server/models/base.py b/toolserve/toolserve/server/models/base.py new file mode 100644 index 00000000..29177498 --- /dev/null +++ b/toolserve/toolserve/server/models/base.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +from datetime import datetime +from typing import Annotated + +from sqlalchemy.orm import ( + DeclarativeBase, + Mapped, + MappedAsDataclass, + declared_attr, + mapped_column, +) + +from toolserve.server.utils.timezone import timezone + +# Common Mapped type primary key, manual addition required, refer to the following usage +# MappedBase -> id: Mapped[id_key] +# DataClassBase && Base -> id: Mapped[id_key] = mapped_column(init=False) +id_key = Annotated[ + int, + mapped_column( + primary_key=True, + index=True, + autoincrement=True, + sort_order=-999, + comment="Primary key id", + ), +] + + +# Mixin: An object-oriented programming concept, makes the structure clearer, `Wiki `__ +class UserMixin(MappedAsDataclass): + """User Mixin data class""" + + create_user: Mapped[int] = mapped_column(sort_order=998, comment="Creator") + update_user: Mapped[int | None] = mapped_column( + init=False, default=None, sort_order=998, comment="Modifier" + ) + + +class DateTimeMixin(MappedAsDataclass): + """Datetime Mixin data class""" + + created_time: Mapped[datetime] = mapped_column( + init=False, + default_factory=timezone.now, + sort_order=999, + comment="Creation time", + ) + updated_time: Mapped[datetime | None] = mapped_column( + init=False, onupdate=timezone.now, sort_order=999, comment="Update time" + ) + + +class MappedBase(DeclarativeBase): + """ + Declarative base class, the original DeclarativeBase class, serves as the parent class for all base or data model classes + + `DeclarativeBase `__ + `mapped_column() `__ + """ + + @declared_attr.directive + def __tablename__(cls) -> str: + return cls.__name__.lower() + + +class DataClassBase(MappedAsDataclass, MappedBase): + """ + Declarative data class base class, integrates with data classes, allows for advanced configurations, but you must be aware of its characteristics, especially when used with DeclarativeBase + + `MappedAsDataclass `__ + """ # noqa: E501 + + __abstract__ = True + + +class Base(DataClassBase, DateTimeMixin): + """ + Declarative Mixin data class base class, integrates data classes, includes the Mixin data class basic table structure, you can simply understand it as a data class base class with basic table structure + """ # noqa: E501 + + __abstract__ = True diff --git a/toolserve/toolserve/server/models/sys_artifact.py b/toolserve/toolserve/server/models/sys_artifact.py new file mode 100644 index 00000000..8197b014 --- /dev/null +++ b/toolserve/toolserve/server/models/sys_artifact.py @@ -0,0 +1,13 @@ +from sqlalchemy import String +from sqlalchemy.orm import Mapped, mapped_column + +from toolserve.server.models.base import Base, id_key + + +class Artifact(Base): + + __tablename__ = 'sys_artifact' + + id: Mapped[id_key] = mapped_column(init=False) + name: Mapped[str] = mapped_column(String(255), comment='Artifact name') + file_path: Mapped[str] = mapped_column(String(255), comment='File path') diff --git a/toolserve/toolserve/server/models/sys_data.py b/toolserve/toolserve/server/models/sys_data.py new file mode 100644 index 00000000..25acf0fa --- /dev/null +++ b/toolserve/toolserve/server/models/sys_data.py @@ -0,0 +1,14 @@ + +from sqlalchemy import String +from sqlalchemy.orm import Mapped, mapped_column + +from toolserve.server.models.base import Base, id_key + + +class Data(Base): + + __tablename__ = 'sys_data' + + id: Mapped[id_key] = mapped_column(init=False) + file_name: Mapped[str] = mapped_column(String(255), comment='File name') + file_path: Mapped[str] = mapped_column(String(255), comment='File path') diff --git a/toolserve/toolserve/server/models/sys_log.py b/toolserve/toolserve/server/models/sys_log.py new file mode 100644 index 00000000..665f7e6c --- /dev/null +++ b/toolserve/toolserve/server/models/sys_log.py @@ -0,0 +1,15 @@ + + +from sqlalchemy import String +from sqlalchemy.orm import Mapped, mapped_column + +from toolserve.server.models.base import Base, id_key + + +class Log(Base): + + __tablename__ = 'sys_log' + + id: Mapped[id_key] = mapped_column(init=False) + level: Mapped[str] = mapped_column(String(50), comment='Log level') + msg: Mapped[str] = mapped_column(String(500), comment='Log message') diff --git a/toolserve/toolserve/server/routes/__init__.py b/toolserve/toolserve/server/routes/__init__.py index 71332435..011d56ca 100644 --- a/toolserve/toolserve/server/routes/__init__.py +++ b/toolserve/toolserve/server/routes/__init__.py @@ -1,6 +1,15 @@ from fastapi import APIRouter from toolserve.server.core.conf import settings -from toolserve.server.routes.action import router as action_router +from toolserve.server.routes.tool import router as tool_router +from toolserve.server.routes.data import router as data_router +from toolserve.server.routes.artifact import router as artifact_router +from toolserve.server.routes.log import router as log_router + v1 = APIRouter(prefix=settings.API_V1_STR) -v1.include_router(action_router, tags=["action"]) +v1.include_router(tool_router, prefix="/tools", tags=["Tool Catalog"]) +v1.include_router(data_router, prefix="/data", tags=["Data Management"]) +v1.include_router(artifact_router, prefix="/artifact", tags=["Artifact Management"]) +v1.include_router(log_router, prefix="/log", tags=["Tool Logging API"]) + + diff --git a/toolserve/toolserve/server/routes/artifact.py b/toolserve/toolserve/server/routes/artifact.py new file mode 100644 index 00000000..1bdaffb2 --- /dev/null +++ b/toolserve/toolserve/server/routes/artifact.py @@ -0,0 +1,45 @@ + +from typing import Annotated +from fastapi import APIRouter, Path, Query +from toolserve.server.schemas.artifact import ( + ArtifactSchemaBase, + CreateArtifactParam, + DeleteArtifactParam, + GetArtifactDetails +) +from toolserve.server.services.artifact_service import artifact_service +from toolserve.server.common.response import ResponseModel, response_base +from toolserve.server.common.serializers import select_as_dict + +router = APIRouter() + +# Get artifact details by artifact id +@router.get('/{pk}', summary='Get artifact details') +async def get_artifact( + pk: Annotated[int, Path(...)], +) -> ResponseModel: + artifact = await artifact_service.get(pk=pk) + data = GetArtifactDetails(**await select_as_dict(artifact)) + return await response_base.success(data=data) + +# Create a new artifact +@router.post('', summary='Create artifact') +async def create_artifact( + obj: CreateArtifactParam +) -> ResponseModel: + await artifact_service.create(obj=obj) + return await response_base.success() + +# Delete artifact +@router.delete('', summary='Delete artifact') +async def delete_artifact(pk: Annotated[list[int], Query(...)]) -> ResponseModel: + count = await artifact_service.delete(pk=pk) + if count > 0: + return await response_base.success() + return await response_base.fail() + +# Get all artifacts +@router.get('', summary='Get all artifacts') +async def get_all_artifacts() -> ResponseModel: + data = await artifact_service.get_all_artifacts() + return await response_base.success(data=data) diff --git a/toolserve/toolserve/server/routes/data.py b/toolserve/toolserve/server/routes/data.py new file mode 100644 index 00000000..36ab2484 --- /dev/null +++ b/toolserve/toolserve/server/routes/data.py @@ -0,0 +1,79 @@ + +from typing import Annotated +from fastapi import APIRouter, Path, Query +from toolserve.server.schemas.data import ( + CreateDataParam, + DataSchemaBase, + GetDataDetails, + GetDataObject +) +from toolserve.server.services.data_service import data_service +from toolserve.server.common.response import ResponseModel, response_base +from toolserve.server.common.serializers import select_as_dict + +router = APIRouter() + +# Get data details by data id +@router.get('/{pk}', summary='Get data details') +async def get_data( + pk: Annotated[int, Path(...)], +) -> ResponseModel: + data_entry = await data_service.get(pk=pk) + data = GetDataDetails(**await select_as_dict(data_entry)) + return await response_base.success(data=data) + +@router.get('', summary='Get all data files') +async def get_all_data() -> ResponseModel: + data = await data_service.get_all_data() + all_data = [GetDataDetails(**await select_as_dict(data_entry)) for data_entry in data] + return await response_base.success(data=all_data) + +# Create a new data entry +@router.post('', summary='Create data') +async def create_data( + obj: CreateDataParam +) -> ResponseModel: + data_obj = await data_service.create(obj=obj) + data = { + "id": data_obj.id, + "file_path": data_obj.file_path + } + return await response_base.success(data=data) + +# Delete data +@router.delete('', summary='Delete data') +async def delete_data(pk: Annotated[list[int], Query(...)]) -> ResponseModel: + count = await data_service.delete(pk=pk) + if count > 0: + return await response_base.success() + return await response_base.fail() + +@router.get('/object/{pk}', summary='Get data object') +async def get_data_object( + pk: Annotated[int, Path(...)], +) -> ResponseModel: + """ + Retrieve a data object by its primary key using the GetDataObject schema. + + Args: + pk (int): The primary key of the data object to retrieve. + + Returns: + ResponseModel: The response model containing the data object or an error message. + """ + data_entry = await data_service.get(pk=pk) + obj = await select_as_dict(data_entry) + try: + with open(obj["file_path"], 'r', encoding='utf-8') as file: + json_data = file.read() + except FileNotFoundError: + raise HTTPException(status_code=404, detail="File not found") + except IOError: + raise HTTPException(status_code=500, detail="File read error") + + data_object = GetDataObject( + file_name=obj["file_name"], + file_path=obj["file_path"], + json_blob=json_data + ) + return await response_base.success(data=data_object) diff --git a/toolserve/toolserve/server/routes/log.py b/toolserve/toolserve/server/routes/log.py new file mode 100644 index 00000000..863b51a7 --- /dev/null +++ b/toolserve/toolserve/server/routes/log.py @@ -0,0 +1,40 @@ + +from typing import Annotated +from fastapi import APIRouter, Path, Query +from toolserve.server.schemas.log import CreateLog, LogSchemaBase, GetLogDetails +from toolserve.server.services.log_service import log_service +from toolserve.server.common.response import ResponseModel, response_base +from toolserve.server.common.serializers import select_as_dict + +router = APIRouter() + +# Get log details by log id +@router.get('/{pk}', summary='Get log details') +async def get_log( + pk: Annotated[int, Path(...)], +) -> ResponseModel: + log = await log_service.get(pk=pk) + data = GetLogDetails(**await select_as_dict(log)) + return await response_base.success(data=data) + +# Create a new log +@router.post('', summary='Create log') +async def create_log( + obj: CreateLog +) -> ResponseModel: + await log_service.create(obj=obj) + return await response_base.success() + +# Delete log +@router.delete('', summary='Delete log') +async def delete_log(pk: Annotated[list[int], Query(...)]) -> ResponseModel: + count = await log_service.delete(pk=pk) + if count > 0: + return await response_base.success() + return await response_base.fail() + +# Get all logs +@router.get('', summary='Get all logs') +async def get_all_logs() -> ResponseModel: + data = await log_service.get_log_list() + return await response_base.success(data=data) diff --git a/toolserve/toolserve/server/routes/action.py b/toolserve/toolserve/server/routes/tool.py similarity index 64% rename from toolserve/toolserve/server/routes/action.py rename to toolserve/toolserve/server/routes/tool.py index 39c78c3e..ab8ed190 100644 --- a/toolserve/toolserve/server/routes/action.py +++ b/toolserve/toolserve/server/routes/tool.py @@ -6,9 +6,9 @@ from pydantic import ValidationError from toolserve.server.core.conf import settings from toolserve.server.core.depends import get_catalog -from toolserve.common.response_code import CustomResponseCode -from toolserve.common.response import ResponseModel, response_base -#from toolserve.utils.openai_tool import schema_to_openai_tool +from toolserve.server.common.response_code import CustomResponseCode +from toolserve.server.common.response import ResponseModel, response_base +from toolserve.utils.openai_tool import schema_to_openai_tool router = APIRouter() @@ -17,25 +17,25 @@ router = APIRouter() summary='List available tools', ) async def list_tools(catalog=Depends(get_catalog)) -> ResponseModel: - """List all available actions""" + """List all available tools""" tools = catalog.list_tools() return await response_base.success(data=tools) @router.get( '/oai_function', - summary="Get the OpenAI function format of an action" + summary="Get the OpenAI function format of an tool" ) async def get_oai_function( - action_name: str = Query(..., title="Action Name", description="The name of the action"), + tool_name: str = Query(..., title="Tool Name", description="The name of the tool"), catalog=Depends(get_catalog) ) -> ResponseModel: - """Get the OpenAI function format of an action""" + """Get the OpenAI function format of an tool""" try: # TODO handle keyerror - action = catalog[action_name] - json_data = schema_to_openai_tool(action) + tool = catalog[tool_name] + json_data = schema_to_openai_tool(tool) return await response_base.success(data=json_data) except ValidationError as e: diff --git a/toolserve/toolserve/server/schemas/__init__.py b/toolserve/toolserve/server/schemas/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/toolserve/toolserve/server/schemas/artifact.py b/toolserve/toolserve/server/schemas/artifact.py new file mode 100644 index 00000000..5c431fca --- /dev/null +++ b/toolserve/toolserve/server/schemas/artifact.py @@ -0,0 +1,29 @@ +from datetime import datetime + +from pydantic import ConfigDict, Field + +from toolserve.server.schemas.base import SchemaBase + + +class ArtifactSchemaBase(SchemaBase): + name: str + file_path: str + + +class CreateArtifactParam(ArtifactSchemaBase): + data: bytes + media_type: str + + +class DeleteArtifactParam(ArtifactSchemaBase): + pass + +class UpdateArtifactParam(ArtifactSchemaBase): + pass + +class GetArtifactDetails(ArtifactSchemaBase): + model_config = ConfigDict(from_attributes=True) + + id: int + created_time: datetime + updated_time: datetime | None = None diff --git a/toolserve/toolserve/server/schemas/base.py b/toolserve/toolserve/server/schemas/base.py new file mode 100644 index 00000000..f3a53832 --- /dev/null +++ b/toolserve/toolserve/server/schemas/base.py @@ -0,0 +1,148 @@ +# -*- coding: utf-8 -*- + +from pydantic import BaseModel, ConfigDict, EmailStr, validate_email + +# Custom validation error messages do not include the expected content of validation (i.e., input content). For supported expected content fields, refer to the following link: +# https://github.com/pydantic/pydantic-core/blob/a5cb7382643415b716b1a7a5392914e50f726528/tests/test_errors.py#L266 +# For replacing expected content fields, refer to the following link: +# https://github.com/pydantic/pydantic/blob/caa78016433ec9b16a973f92f187a7b6bfde6cb5/docs/errors/errors.md?plain=1#L232 +CUSTOM_VALIDATION_ERROR_MESSAGES = { + 'arguments_type': 'Incorrect argument type input', + 'assertion_error': 'Assertion execution error', + 'bool_parsing': 'Boolean value parsing error', + 'bool_type': 'Boolean type input error', + 'bytes_too_long': 'Byte length input too long', + 'bytes_too_short': 'Byte length input too short', + 'bytes_type': 'Byte type input error', + 'callable_type': 'Callable object type input error', + 'dataclass_exact_type': 'Dataclass instance type input error', + 'dataclass_type': 'Dataclass type input error', + 'date_from_datetime_inexact': 'Non-zero date component input', + 'date_from_datetime_parsing': 'Date input parsing error', + 'date_future': 'Date input is not in the future', + 'date_parsing': 'Date input validation error', + 'date_past': 'Date input is not in the past', + 'date_type': 'Date type input error', + 'datetime_future': 'Datetime input is not in the future', + 'datetime_object_invalid': 'Datetime input object invalid', + 'datetime_parsing': 'Datetime input parsing error', + 'datetime_past': 'Datetime input is not in the past', + 'datetime_type': 'Datetime type input error', + 'decimal_max_digits': 'Decimal input has too many digits', + 'decimal_max_places': 'Decimal places input error', + 'decimal_parsing': 'Decimal input parsing error', + 'decimal_type': 'Decimal type input error', + 'decimal_whole_digits': 'Decimal whole digits input error', + 'dict_type': 'Dictionary type input error', + 'enum': 'Enum member input error, allowed {expected}', + 'extra_forbidden': 'Extra fields input forbidden', + 'finite_number': 'Finite value input error', + 'float_parsing': 'Float parsing error', + 'float_type': 'Float type input error', + 'frozen_field': 'Frozen field input error', + 'frozen_instance': 'Modification of frozen instance forbidden', + 'frozen_set_type': 'Frozen set type input forbidden', + 'get_attribute_error': 'Attribute retrieval error', + 'greater_than': 'Input value too large', + 'greater_than_equal': 'Input value too large or equal', + 'int_from_float': 'Integer type input error', + 'int_parsing': 'Integer input parsing error', + 'int_parsing_size': 'Integer input parsing size error', + 'int_type': 'Integer type input error', + 'invalid_key': 'Invalid key input', + 'is_instance_of': 'Instance type input error', + 'is_subclass_of': 'Subclass type input error', + 'iterable_type': 'Iterable type input error', + 'iteration_error': 'Iteration value input error', + 'json_invalid': 'JSON string input error', + 'json_type': 'JSON type input error', + 'less_than': 'Input value too small', + 'less_than_equal': 'Input value too small or equal', + 'list_type': 'List type input error', + 'literal_error': 'Literal input error', + 'mapping_type': 'Mapping type input error', + 'missing': 'Missing required field', + 'missing_argument': 'Missing argument', + 'missing_keyword_only_argument': 'Missing keyword-only argument', + 'missing_positional_only_argument': 'Missing positional-only argument', + 'model_attributes_type': 'Model attributes type input error', + 'model_type': 'Model instance input error', + 'multiple_argument_values': 'Multiple argument values input', + 'multiple_of': 'Input value not a multiple', + 'no_such_attribute': 'Invalid attribute assignment', + 'none_required': 'Input value must be None', + 'recursion_loop': 'Recursion loop in input', + 'set_type': 'Set type input error', + 'string_pattern_mismatch': 'String pattern mismatch input', + 'string_sub_type': 'String subtype (non-strict instance) input error', + 'string_too_long': 'String input too long', + 'string_too_short': 'String input too short', + 'string_type': 'String type input error', + 'string_unicode': 'String input not Unicode', + 'time_delta_parsing': 'Time delta parsing error', + 'time_delta_type': 'Time delta type input error', + 'time_parsing': 'Time input parsing error', + 'time_type': 'Time type input error', + 'timezone_aware': 'Missing timezone input', + 'timezone_naive': 'Timezone input forbidden', + 'too_long': 'Input too long', + 'too_short': 'Input too short', + 'tuple_type': 'Tuple type input error', + 'unexpected_keyword_argument': 'Unexpected keyword argument input', + 'unexpected_positional_argument': 'Unexpected positional argument input', + 'union_tag_invalid': 'Union tag literal input error', + 'union_tag_not_found': 'Union tag argument not found', + 'url_parsing': 'URL input parsing error', + 'url_scheme': 'URL scheme input error', + 'url_syntax_violation': 'URL syntax violation', + 'url_too_long': 'URL input too long', + 'url_type': 'URL type input error', + 'uuid_parsing': 'UUID parsing error', + 'uuid_type': 'UUID type input error', + 'uuid_version': 'UUID version type input error', + 'value_error': 'Value input error', +} + +CUSTOM_USAGE_ERROR_MESSAGES = { + 'class-not-fully-defined': 'Class attributes type not fully defined', + 'custom-json-schema': '__modify_schema__ method deprecated in V2', + 'decorator-missing-field': 'Invalid field validator defined', + 'discriminator-no-field': 'Discriminator field not fully defined', + 'discriminator-alias-type': 'Discriminator field defined using non-string type', + 'discriminator-needs-literal': 'Discriminator field requires literal definition', + 'discriminator-alias': 'Inconsistent discriminator field alias definition', + 'discriminator-validator': 'Field validator forbidden on discriminator field', + 'model-field-overridden': 'Typeless field override forbidden', + 'model-field-missing-annotation': 'Missing field type definition', + 'config-both': 'Duplicate configuration item defined', + 'removed-kwargs': 'Removed keyword configuration parameter called', + 'invalid-for-json-schema': 'Invalid JSON type present', + 'base-model-instantiated': 'Instantiation of base model forbidden', + 'undefined-annotation': 'Missing type definition', + 'schema-for-unknown-type': 'Unknown type definition', + 'create-model-field-definitions': 'Field definition error', + 'create-model-config-base': 'Configuration item definition error', + 'validator-no-fields': 'Field validator without specified fields', + 'validator-invalid-fields': 'Field validator fields definition error', + 'validator-instance-method': 'Field validator must be a class method', + 'model-serializer-instance-method': 'Serializer must be an instance method', + 'validator-v1-signature': 'V1 field validator error deprecated', + 'validator-signature': 'Field validator signature error', + 'field-serializer-signature': 'Field serializer signature unrecognized', + 'model-serializer-signature': 'Model serializer signature unrecognized', + 'multiple-field-serializers': 'Field serializers defined multiple times', + 'invalid_annotated_type': 'Invalid type definition', + 'type-adapter-config-unused': 'Type adapter configuration item definition error', + 'root-model-extra': 'Extra fields on root model forbidden', +} + + + +class CustomEmailStr(EmailStr): + @classmethod + def _validate(cls, __input_value: str) -> str: + return None if __input_value == '' else validate_email(__input_value)[1] + + +class SchemaBase(BaseModel): + model_config = ConfigDict(use_enum_values=True) diff --git a/toolserve/toolserve/server/schemas/data.py b/toolserve/toolserve/server/schemas/data.py new file mode 100644 index 00000000..3d5ecb5b --- /dev/null +++ b/toolserve/toolserve/server/schemas/data.py @@ -0,0 +1,46 @@ + +from datetime import datetime + +from pydantic import BaseModel, Field + +from toolserve.server.schemas.base import SchemaBase + + +class DataSchemaBase(SchemaBase): + """ + Base schema for Data model. + """ + file_name: str = Field(..., title="File Name", description="Name of the file") + + +class CreateDataParam(DataSchemaBase): + """ + Parameters required to create a Data entry. + """ + json_blob: str = Field(..., title="JSON Blob", description="JSON blob containing the data") + file_path: str | None = Field(default=None, title="File Path", description="Path of the file") + + +class DeleteDataParam(DataSchemaBase): + """ + Parameters required to delete a Data entry. + """ + pass + + +class GetDataObject(DataSchemaBase): + """ + Schema to retrieve details of a Data entry. + """ + json_blob: str = Field(..., title="JSON Blob", description="JSON blob containing the data") + + +class GetDataDetails(DataSchemaBase): + """ + Schema to retrieve details of a Data entry. + """ + id: int = Field(..., title="ID", description="Unique identifier of the Data entry") + file_path: str = Field(..., title="File Path", description="Path of the file") + + created_time: datetime = Field(..., title="Creation Time", description="Time when the Data entry was created") + updated_time: datetime | None = Field(default=None, title="Updated Time", description="Time when the Data entry was last updated") diff --git a/toolserve/toolserve/server/schemas/log.py b/toolserve/toolserve/server/schemas/log.py new file mode 100644 index 00000000..78bdc71f --- /dev/null +++ b/toolserve/toolserve/server/schemas/log.py @@ -0,0 +1,22 @@ +from datetime import datetime + +from pydantic import ConfigDict, Field + +from toolserve.server.schemas.base import SchemaBase + + +class LogSchemaBase(SchemaBase): + level: str = Field(..., title='Log level', description='Log level') + msg: str = Field(..., title='Log message', description='Log message') + + +class CreateLog(LogSchemaBase): + pass + + +class GetLogDetails(LogSchemaBase): + model_config = ConfigDict(from_attributes=True) + + id: int + created_time: datetime + updated_time: datetime | None = None diff --git a/toolserve/toolserve/server/services/__init__.py b/toolserve/toolserve/server/services/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/toolserve/toolserve/server/services/artifact_service.py b/toolserve/toolserve/server/services/artifact_service.py new file mode 100644 index 00000000..a21942ac --- /dev/null +++ b/toolserve/toolserve/server/services/artifact_service.py @@ -0,0 +1,47 @@ +from typing import Sequence + +from sqlalchemy import select +from toolserve.server.common.exception import errors +from toolserve.server.crud.crud_artifact import artifact_dao +from toolserve.server.database.db_sqlite import async_db_session +from toolserve.server.models.sys_artifact import Artifact +from toolserve.server.schemas.artifact import CreateArtifactParam, ArtifactSchemaBase +from toolserve.server.core.conf import settings +import os + +class ArtifactService: + @staticmethod + async def get(*, pk: int) -> Artifact: + async with async_db_session() as db: + artifact = await artifact_dao.get(db, pk) + if not artifact: + raise errors.NotFoundError(msg='Artifact not found') + return artifact + + @staticmethod + async def get_all_artifacts() -> Sequence[Artifact]: + async with async_db_session() as db: + artifacts = await artifact_dao.get_all(db) + return artifacts + + @staticmethod + async def create(*, obj: CreateArtifactParam) -> None: + async with async_db_session.begin() as db: + artifact = Artifact(name=obj.name, file_path=os.path.join(settings.ARTIFACTS_DIR, obj.name)) + await artifact_dao.create(db, artifact) # This line persists the artifact in the database. + with open(artifact.file_path, 'wb') as file: + file.write(obj.data) + + @staticmethod + async def update(*, pk: int, obj: ArtifactSchemaBase) -> int: + async with async_db_session.begin() as db: + count = await artifact_dao.update(db, pk, obj) + return count + + @staticmethod + async def delete(*, pk: list[int]) -> int: + async with async_db_session.begin() as db: + count = await artifact_dao.delete(db, pk) + return count + +artifact_service: ArtifactService = ArtifactService() diff --git a/toolserve/toolserve/server/services/data_service.py b/toolserve/toolserve/server/services/data_service.py new file mode 100644 index 00000000..62e09272 --- /dev/null +++ b/toolserve/toolserve/server/services/data_service.py @@ -0,0 +1,60 @@ +from typing import Sequence + +from sqlalchemy import select +from toolserve.server.common.exception import errors +from toolserve.server.crud.crud_data import data_dao +from toolserve.server.database.db_sqlite import async_db_session +from toolserve.server.models.sys_data import Data +from toolserve.server.schemas.data import CreateDataParam, DataSchemaBase +from toolserve.server.core.conf import settings +import os +import json + +class DataService: + @staticmethod + async def get(*, pk: int) -> Data: + async with async_db_session() as db: + data = await data_dao.get(db, pk) + if not data: + raise errors.NotFoundError(msg='Data not found') + return data + + @staticmethod + async def get_all_data() -> Sequence[Data]: + async with async_db_session() as db: + data_entries = await data_dao.get_all(db) + return data_entries + + @staticmethod + async def create(*, obj: CreateDataParam) -> Data: + async with async_db_session.begin() as db: + file_name = obj.file_name + ".json" if not obj.file_name.endswith(".json") else obj.file_name + obj.file_path = os.path.join(settings.DATA_DIR, file_name) + + data = obj.copy(exclude={'json_blob'}) # Save everything but the data + db_model = await data_dao.create(db, data) # This now returns the ID + + await db.commit() + + # TODO figure out how to save it so it doesnt overwrite the existing file + os.makedirs(os.path.dirname(obj.file_path), exist_ok=True) + with open(obj.file_path, 'w') as file: + file.write(obj.json_blob) + + return db_model + + + @staticmethod + async def update(*, pk: int, obj: DataSchemaBase) -> int: + async with async_db_session.begin() as db: + count = await data_dao.update(db, pk, obj) + return count + + @staticmethod + async def delete(*, pk: list[int]) -> int: + async with async_db_session.begin() as db: + count = await data_dao.delete(db, pk) + return count + + +data_service: DataService = DataService() diff --git a/toolserve/toolserve/server/services/log_service.py b/toolserve/toolserve/server/services/log_service.py new file mode 100644 index 00000000..72ec8321 --- /dev/null +++ b/toolserve/toolserve/server/services/log_service.py @@ -0,0 +1,47 @@ + +from typing import Sequence + +from sqlalchemy import Select + +from toolserve.server.common.exception import errors +from toolserve.server.crud.crud_log import log_dao +from toolserve.server.database.db_sqlite import async_db_session +from toolserve.server.models.sys_log import Log +from toolserve.server.schemas.log import CreateLog, LogSchemaBase + + +class LogService: + @staticmethod + async def get(*, pk: int) -> Log: + async with async_db_session() as db: + log = await log_dao.get(db, pk) + if not log: + raise errors.NotFoundError(msg='Log entry not found') + return log + + + @staticmethod + async def get_log_list() -> Sequence[Log]: + async with async_db_session() as db: + logs = await log_dao.get_all(db) + return logs + + @staticmethod + async def create(*, obj: CreateLog) -> None: + async with async_db_session.begin() as db: + await log_dao.create(db, obj) + + @staticmethod + async def update(*, pk: int, obj: LogSchemaBase) -> int: + async with async_db_session.begin() as db: + count = await log_dao.update(db, pk, obj) + return count + + @staticmethod + async def delete(*, pk: list[int]) -> int: + async with async_db_session.begin() as db: + count = await log_dao.delete(db, pk) + return count + + +log_service: LogService = LogService() diff --git a/toolserve/toolserve/server/utils/__init__.py b/toolserve/toolserve/server/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/toolserve/toolserve/server/utils/timezone.py b/toolserve/toolserve/server/utils/timezone.py new file mode 100644 index 00000000..38a00b8f --- /dev/null +++ b/toolserve/toolserve/server/utils/timezone.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +import zoneinfo + +from datetime import datetime + +from toolserve.server.core.conf import settings + + +class TimeZone: + def __init__(self, tz: str = settings.DATETIME_TIMEZONE): + self.tz_info = zoneinfo.ZoneInfo(tz) + + def now(self) -> datetime: + """ + 获取时区时间 + + :return: + """ + return datetime.now(self.tz_info) + + def f_datetime(self, dt: datetime) -> datetime: + """ + datetime 时间转时区时间 + + :param dt: + :return: + """ + return dt.astimezone(self.tz_info) + + def f_str(self, date_str: str, format_str: str = settings.DATETIME_FORMAT) -> datetime: + """ + 时间字符串转时区时间 + + :param date_str: + :param format_str: + :return: + """ + return datetime.strptime(date_str, format_str).replace(tzinfo=self.tz_info) + + +timezone = TimeZone() diff --git a/toolserve/toolserve/utils/openai_tool.py b/toolserve/toolserve/utils/openai_tool.py index 7fb34b62..47f0405b 100644 --- a/toolserve/toolserve/utils/openai_tool.py +++ b/toolserve/toolserve/utils/openai_tool.py @@ -1,11 +1,8 @@ import json -from typing import Any, Dict, Type, Union, List -from pydantic import BaseModel, Field -from datetime import datetime +from typing import Any, Dict, Type +from pydantic import BaseModel -from toolserve.server.core.catalog import ParameterSchema - -# TODO clean this up +from toolserve.server.core.catalog import ToolSchema PYTHON_TO_JSON_TYPES = { str: "string", @@ -26,55 +23,47 @@ def python_type_to_json_type(python_type: Type) -> Dict[str, Any]: Returns: Dict[str, Any]: A dictionary representing the JSON schema for the given Python type. """ - if hasattr(python_type, '__origin__'): # For generic types like List[str] or Dict[str, int] + if hasattr(python_type, '__origin__'): origin = python_type.__origin__ if origin is list: item_type = python_type_to_json_type(python_type.__args__[0]) - return {'type': 'array', 'items': {'type': item_type}} + return {'type': 'array', 'items': item_type} elif origin is dict: - # Handle dictionary with specific key and value types key_type = python_type_to_json_type(python_type.__args__[0]) value_type = python_type_to_json_type(python_type.__args__[1]) - return {'type': 'object', 'additionalProperties': {'type': value_type}} - #elif issubclass(python_type, BaseModel): # For Pydantic models - # return model_to_json_schema(python_type) + return {'type': 'object', 'additionalProperties': value_type} return PYTHON_TO_JSON_TYPES.get(python_type, "string") -def parameter_schema_to_json(parameter_schema: 'ParameterSchema') -> Dict[str, Any]: - """Convert a ParameterSchema to a JSON schema property.""" - property_schema = { - "type": python_type_to_json_type(parameter_schema.dtype), - "description": parameter_schema.description, - } - if parameter_schema.default is not None: - property_schema["default"] = parameter_schema.default - return property_schema - def model_to_json_schema(model: Type[BaseModel]) -> Dict[str, Any]: - """Convert a Pydantic model to a JSON schema.""" + """ + Convert a Pydantic model to a JSON schema. + + Args: + model (Type[BaseModel]): The Pydantic model to convert. + + Returns: + Dict[str, Any]: A dictionary representing the JSON schema for the given model. + """ properties = {} required = [] for field_name, model_field in model.model_fields.items(): - field_schema = parameter_schema_to_json( - ParameterSchema( - name=field_name, - dtype=model_field.annotation, - description=model_field.description or "", - default=model_field.default, - required=model_field.required - ) - ) - properties[field_name] = field_schema + field_schema = { + "type": python_type_to_json_type(model_field.annotation), + "description": model_field.description or "", + } + if model_field.default is not None: + field_schema["default"] = model_field.default if model_field.is_required(): required.append(field_name) + properties[field_name] = field_schema return { "type": "object", "properties": properties, "required": required, } -def schema_to_openai_tool(action_schema: 'ActionSchema') -> str: - """Convert an ActionSchema object to a JSON schema string in the specified function format. +def schema_to_openai_tool(tool_schema: 'ToolSchema') -> str: + """Convert an ToolSchema object to a JSON schema string in the specified function format. Example output format: { @@ -100,30 +89,18 @@ def schema_to_openai_tool(action_schema: 'ActionSchema') -> str: } Args: - action_schema (ActionSchema): The action schema to convert. + tool_schema (ToolSchema): The tool schema to convert. Returns: - str: A JSON schema string representing the action in the specified format. + str: A JSON schema string representing the tool in the specified format. """ - properties = {} - required = [] - if action_schema.in_schema: - for input_param in action_schema.in_schema.inputs: - param_schema = parameter_schema_to_json(input_param) - properties[input_param.name] = param_schema - if input_param.required: - required.append(input_param.name) - + input_model_schema = model_to_json_schema(tool_schema.input_model) function_schema = { "type": "function", "function": { - "name": action_schema.name, - "description": action_schema.description, - "parameters": { - "type": "object", - "properties": properties, - "required": required, - } + "name": tool_schema.name, + "description": tool_schema.description, + "parameters": input_model_schema, } } return json.dumps(function_schema, indent=2)