admin-valentin/arcade-mcp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

arcade evals Default to localhost (#172)

Browse source 
# PR Description
* `arcade evals` now run evaluations against Arcade Engine at
`http://localhost:9099` by default.
* Added optional flag `--cloud` to run evaluations against Arcade's
Cloud Engine at `https://api.arcade-ai.com`. Overrides `-h` flag.
* Always print the Engine that the evaluations are using. Previously
this was reserved for `-d` flag.
This commit is contained in:
Eric Gustin 2024-12-17 10:41:32 -08:00 committed by GitHub
parent d8c8b060af
commit b12ceec4b5
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 22 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
arcade/arcade/cli/constants.py
View file

@ -1,5 +1,6 @@
DEFAULT_CLOUD_HOST = "cloud.arcade-ai.com" DEFAULT_CLOUD_HOST = "cloud.arcade-ai.com"
DEFAULT_ENGINE_HOST = "api.arcade-ai.com" DEFAULT_ENGINE_HOST = "api.arcade-ai.com"
LOCALHOST = "localhost"
_style_block = b""" _style_block = b"""
<link rel="icon" href="https://cdn.arcade-ai.com/favicons/favicon.ico" sizes="any"> <link rel="icon" href="https://cdn.arcade-ai.com/favicons/favicon.ico" sizes="any">

22
arcade/arcade/cli/main.py
View file

@ -14,7 +14,7 @@ from rich.markup import escape
from rich.text import Text from rich.text import Text
from arcade.cli.authn import LocalAuthCallbackServer, check_existing_login from arcade.cli.authn import LocalAuthCallbackServer, check_existing_login
from arcade.cli.constants import DEFAULT_CLOUD_HOST, DEFAULT_ENGINE_HOST from arcade.cli.constants import DEFAULT_CLOUD_HOST, DEFAULT_ENGINE_HOST, LOCALHOST
from arcade.cli.display import ( from arcade.cli.display import (
display_arcade_chat_header, display_arcade_chat_header,
display_eval_results, display_eval_results,
@ -345,11 +345,16 @@ def evals(
help="The models to use for evaluation (default: gpt-4o)", help="The models to use for evaluation (default: gpt-4o)",
), ),
host: str = typer.Option( host: str = typer.Option(
DEFAULT_ENGINE_HOST, LOCALHOST,
"-h", "-h",
"--host", "--host",
help="The Arcade Engine address to send chat requests to.", help="The Arcade Engine address to send chat requests to.",
), ),
cloud: bool = typer.Option(
False,
"--cloud",
help="Whether to run evaluations against the Arcade Cloud Engine. Overrides the 'host' option.",
),
port: int = typer.Option( port: int = typer.Option(
None, None,
"-p", "-p",
@ -372,6 +377,8 @@ def evals(
execute any functions decorated with @tool_eval, and display the results. execute any functions decorated with @tool_eval, and display the results.
""" """
config = validate_and_get_config() config = validate_and_get_config()
host = DEFAULT_ENGINE_HOST if cloud else host
base_url = compute_engine_base_url(force_tls, force_no_tls, host, port) base_url = compute_engine_base_url(force_tls, force_no_tls, host, port)
models_list = models.split(",") # Use 'models_list' to avoid shadowing models_list = models.split(",") # Use 'models_list' to avoid shadowing
@ -380,13 +387,12 @@ def evals(
if not eval_files: if not eval_files:
return return
if show_details: console.print(
console.print( Text.assemble(
Text.assemble( ("\nRunning evaluations against Arcade Engine at ", "bold"),
("\nRunning evaluations against Arcade Engine at ", "bold"), (base_url, "bold blue"),
(base_url, "bold blue"),
)
) )
)
# Try to hit /health endpoint on engine and warn if it is down # Try to hit /health endpoint on engine and warn if it is down
with Arcade(api_key=config.api.key, base_url=base_url) as client: with Arcade(api_key=config.api.key, base_url=base_url) as client:

13
arcade/arcade/cli/utils.py
View file

@ -24,6 +24,7 @@ from rich.text import Text
from typer.core import TyperGroup from typer.core import TyperGroup
from typer.models import Context from typer.models import Context
from arcade.cli.constants import LOCALHOST
from arcade.core.config_model import Config from arcade.core.config_model import Config
from arcade.core.errors import ToolkitLoadError from arcade.core.errors import ToolkitLoadError
from arcade.core.schema import ToolDefinition from arcade.core.schema import ToolDefinition
@ -116,7 +117,7 @@ def compute_engine_base_url(
str: The fully constructed URL for the Arcade Engine. str: The fully constructed URL for the Arcade Engine.
""" """
# "Use 127.0.0.1" and "0.0.0.0" as aliases for "localhost" # "Use 127.0.0.1" and "0.0.0.0" as aliases for "localhost"
host = "localhost" if host in ["127.0.0.1", "0.0.0.0"] else host # noqa: S104 host = LOCALHOST if host in ["127.0.0.1", "0.0.0.0"] else host # noqa: S104
# Determine TLS setting based on input flags # Determine TLS setting based on input flags
if force_no_tls: if force_no_tls:
@ -124,10 +125,10 @@ def compute_engine_base_url(
elif force_tls: elif force_tls:
is_tls = True is_tls = True
else: else:
is_tls = host != "localhost" is_tls = host != LOCALHOST
# "localhost" defaults to dev port if not specified # "localhost" defaults to dev port if not specified
if host == "localhost" and port is None: if host == LOCALHOST and port is None:
port = 9099 port = 9099
protocol = "https" if is_tls else "http" protocol = "https" if is_tls else "http"
@ -170,14 +171,14 @@ def compute_login_url(host: str, state: str, port: int | None) -> str:
""" """
Compute the full URL for the CLI login endpoint. Compute the full URL for the CLI login endpoint.
""" """
callback_uri = "http://localhost:9905/callback" callback_uri = f"http://{LOCALHOST}:9905/callback"
params = urlencode({"callback_uri": callback_uri, "state": state}) params = urlencode({"callback_uri": callback_uri, "state": state})
port = port if port else 8000 port = port if port else 8000
login_base_url = ( login_base_url = (
f"http://localhost:{port}" f"http://{LOCALHOST}:{port}"
if host in ["localhost", "127.0.0.1", "0.0.0.0"] # noqa: S104 if host in [LOCALHOST, "127.0.0.1", "0.0.0.0"] # noqa: S104
else f"https://{host}" else f"https://{host}"
) )
endpoint = "/api/v1/auth/cli_login" endpoint = "/api/v1/auth/cli_login"