From b12ceec4b5b69ecfcd6dc96762f41dc6a571a97e Mon Sep 17 00:00:00 2001
From: Eric Gustin <34000337+EricGustin@users.noreply.github.com>
Date: Tue, 17 Dec 2024 10:41:32 -0800
Subject: [PATCH] `arcade evals` Default to localhost (#172)

# PR Description
* `arcade evals` now run evaluations against Arcade Engine at
`http://localhost:9099` by default.
* Added optional flag `--cloud` to run evaluations against Arcade's
Cloud Engine at `https://api.arcade-ai.com`. Overrides `-h` flag.
* Always print the Engine that the evaluations are using. Previously
this was reserved for `-d` flag.
---
 arcade/arcade/cli/constants.py |  1 +
 arcade/arcade/cli/main.py      | 22 ++++++++++++++--------
 arcade/arcade/cli/utils.py     | 13 +++++++------
 3 files changed, 22 insertions(+), 14 deletions(-)

diff --git a/arcade/arcade/cli/constants.py b/arcade/arcade/cli/constants.py
index aeaa6bb0..9586202d 100644
--- a/arcade/arcade/cli/constants.py
+++ b/arcade/arcade/cli/constants.py
@@ -1,5 +1,6 @@
 DEFAULT_CLOUD_HOST = "cloud.arcade-ai.com"
 DEFAULT_ENGINE_HOST = "api.arcade-ai.com"
+LOCALHOST = "localhost"
 
 _style_block = b"""
 <link rel="icon" href="https://cdn.arcade-ai.com/favicons/favicon.ico" sizes="any">
diff --git a/arcade/arcade/cli/main.py b/arcade/arcade/cli/main.py
index 51ad9ceb..885c66f8 100644
--- a/arcade/arcade/cli/main.py
+++ b/arcade/arcade/cli/main.py
@@ -14,7 +14,7 @@ from rich.markup import escape
 from rich.text import Text
 
 from arcade.cli.authn import LocalAuthCallbackServer, check_existing_login
-from arcade.cli.constants import DEFAULT_CLOUD_HOST, DEFAULT_ENGINE_HOST
+from arcade.cli.constants import DEFAULT_CLOUD_HOST, DEFAULT_ENGINE_HOST, LOCALHOST
 from arcade.cli.display import (
     display_arcade_chat_header,
     display_eval_results,
@@ -345,11 +345,16 @@ def evals(
         help="The models to use for evaluation (default: gpt-4o)",
     ),
     host: str = typer.Option(
-        DEFAULT_ENGINE_HOST,
+        LOCALHOST,
         "-h",
         "--host",
         help="The Arcade Engine address to send chat requests to.",
     ),
+    cloud: bool = typer.Option(
+        False,
+        "--cloud",
+        help="Whether to run evaluations against the Arcade Cloud Engine. Overrides the 'host' option.",
+    ),
     port: int = typer.Option(
         None,
         "-p",
@@ -372,6 +377,8 @@ def evals(
     execute any functions decorated with @tool_eval, and display the results.
     """
     config = validate_and_get_config()
+
+    host = DEFAULT_ENGINE_HOST if cloud else host
     base_url = compute_engine_base_url(force_tls, force_no_tls, host, port)
 
     models_list = models.split(",")  # Use 'models_list' to avoid shadowing
@@ -380,13 +387,12 @@ def evals(
     if not eval_files:
         return
 
-    if show_details:
-        console.print(
-            Text.assemble(
-                ("\nRunning evaluations against Arcade Engine at ", "bold"),
-                (base_url, "bold blue"),
-            )
+    console.print(
+        Text.assemble(
+            ("\nRunning evaluations against Arcade Engine at ", "bold"),
+            (base_url, "bold blue"),
         )
+    )
 
     # Try to hit /health endpoint on engine and warn if it is down
     with Arcade(api_key=config.api.key, base_url=base_url) as client:
diff --git a/arcade/arcade/cli/utils.py b/arcade/arcade/cli/utils.py
index 16a7cccf..6ea4c666 100644
--- a/arcade/arcade/cli/utils.py
+++ b/arcade/arcade/cli/utils.py
@@ -24,6 +24,7 @@ from rich.text import Text
 from typer.core import TyperGroup
 from typer.models import Context
 
+from arcade.cli.constants import LOCALHOST
 from arcade.core.config_model import Config
 from arcade.core.errors import ToolkitLoadError
 from arcade.core.schema import ToolDefinition
@@ -116,7 +117,7 @@ def compute_engine_base_url(
         str: The fully constructed URL for the Arcade Engine.
     """
     # "Use 127.0.0.1" and "0.0.0.0" as aliases for "localhost"
-    host = "localhost" if host in ["127.0.0.1", "0.0.0.0"] else host  # noqa: S104
+    host = LOCALHOST if host in ["127.0.0.1", "0.0.0.0"] else host  # noqa: S104
 
     # Determine TLS setting based on input flags
     if force_no_tls:
@@ -124,10 +125,10 @@ def compute_engine_base_url(
     elif force_tls:
         is_tls = True
     else:
-        is_tls = host != "localhost"
+        is_tls = host != LOCALHOST
 
     # "localhost" defaults to dev port if not specified
-    if host == "localhost" and port is None:
+    if host == LOCALHOST and port is None:
         port = 9099
 
     protocol = "https" if is_tls else "http"
@@ -170,14 +171,14 @@ def compute_login_url(host: str, state: str, port: int | None) -> str:
     """
     Compute the full URL for the CLI login endpoint.
     """
-    callback_uri = "http://localhost:9905/callback"
+    callback_uri = f"http://{LOCALHOST}:9905/callback"
     params = urlencode({"callback_uri": callback_uri, "state": state})
 
     port = port if port else 8000
 
     login_base_url = (
-        f"http://localhost:{port}"
-        if host in ["localhost", "127.0.0.1", "0.0.0.0"]  # noqa: S104
+        f"http://{LOCALHOST}:{port}"
+        if host in [LOCALHOST, "127.0.0.1", "0.0.0.0"]  # noqa: S104
         else f"https://{host}"
     )
     endpoint = "/api/v1/auth/cli_login"