db948125d5
1. New Eval SDK (`arcade/sdk/eval.py`): - Introduces `EvalSuite`, `EvalCase`, and `EvalRubric` classes for structured evaluation. - Implements various Critic classes (Binary, Numeric, Similarity) for flexible scoring. - Adds a `tool_eval` decorator for easy integration with existing tools. 2. CLI Integration (`arcade/cli/main.py` and `arcade/cli/utils.py`): - Adds an `evals` command to run evaluation suites from the CLI. - Implements result display functionality for evaluation outcomes. 3. Toolkit Updates: - Adds evaluation scripts for Gmail ([toolkits/gmail/evals/eval_gmail_tools.py](file:///Users/spartee/Dropbox/Arcade/platform/Team/arcade-ai/toolkits/gmail/evals/eval_gmail_tools.py#1%2C1-1%2C1)) and Slack ([toolkits/slack/evals/eval_slack_messaging.py](file:///Users/spartee/Dropbox/Arcade/platform/Team/arcade-ai/toolkits/slack/evals/eval_slack_messaging.py#1%2C1-1%2C1)) toolkits. - Demonstrates practical usage of the Eval SDK with real-world scenarios. 4. Miscellaneous: - Updates `arcade/cli/new.py` to optionally generate an `evals` directory for new toolkits. --------- Co-authored-by: Nate Barbettini <nate@arcade-ai.com>
44 lines
1.2 KiB
Python
44 lines
1.2 KiB
Python
import os
|
|
|
|
from modal import App, Image, asgi_app
|
|
|
|
os.environ["WORK_DIR"] = "/root"
|
|
|
|
# Define the FastAPI app
|
|
app = App("arcade-ai-actor")
|
|
|
|
|
|
image = (
|
|
Image.debian_slim()
|
|
.copy_local_dir("./dist", "/root/dist")
|
|
.pip_install("/root/dist/arcade_ai-0.1.0-py3-none-any.whl")
|
|
.pip_install("/root/dist/arcade_gmail-0.1.0-py3-none-any.whl")
|
|
.pip_install("/root/dist/arcade_websearch-0.1.0-py3-none-any.whl")
|
|
.pip_install("/root/dist/arcade_github-0.1.0-py3-none-any.whl")
|
|
.pip_install("/root/dist/arcade_slack-0.1.0-py3-none-any.whl")
|
|
.pip_install("fastapi>=0.110.0")
|
|
.pip_install("uvicorn>=0.24.0")
|
|
.pip_install("pydantic>=2.7.0")
|
|
.copy_local_file("./arcade.toml", "/root/arcade.toml")
|
|
)
|
|
|
|
|
|
@app.function(image=image)
|
|
@asgi_app()
|
|
def fastapi_app():
|
|
from fastapi import FastAPI
|
|
|
|
from arcade.actor.fastapi.actor import FastAPIActor
|
|
from arcade.core.toolkit import Toolkit
|
|
|
|
web_app = FastAPI()
|
|
|
|
# Initialize app and Arcade FastAPIActor
|
|
actor = FastAPIActor(web_app)
|
|
|
|
# Register toolkits we've installed
|
|
toolkits = Toolkit.find_all_arcade_toolkits()
|
|
for toolkit in toolkits:
|
|
actor.register_toolkit(toolkit)
|
|
|
|
return web_app
|