diff --git a/toolkits/code_sandbox/arcade_code_sandbox/__init__.py b/toolkits/code_sandbox/arcade_code_sandbox/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/toolkits/code_sandbox/arcade_code_sandbox/tools/__init__.py b/toolkits/code_sandbox/arcade_code_sandbox/tools/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/toolkits/code_sandbox/arcade_code_sandbox/tools/e2b.py b/toolkits/code_sandbox/arcade_code_sandbox/tools/e2b.py new file mode 100644 index 00000000..be48cc8d --- /dev/null +++ b/toolkits/code_sandbox/arcade_code_sandbox/tools/e2b.py @@ -0,0 +1,49 @@ +from typing import Annotated + +from e2b_code_interpreter import Sandbox + +from arcade.sdk import tool +from arcade_code_sandbox.tools.models import E2BSupportedLanguage +from arcade_code_sandbox.tools.utils import get_secret + +# See https://e2b.dev/docs to learn more about E2B + + +@tool +def run_code( + code: Annotated[str, "The code to run"], + language: Annotated[ + E2BSupportedLanguage, "The language of the code" + ] = E2BSupportedLanguage.PYTHON, +) -> Annotated[str, "The sandbox execution as a JSON string"]: + """ + Run code in a sandbox and return the output. + """ + api_key = get_secret("E2B_API_KEY") + + with Sandbox(api_key=api_key) as sbx: + execution = sbx.run_code(code=code, language=language) + + return execution.to_json() + + +# Note: Not recommended to use tool_choice='generate' with this tool since it contains base64 encoded image. +@tool +def create_static_matplotlib_chart( + code: Annotated[str, "The Python code to run"], +) -> Annotated[dict, "A dictionary with the following keys: base64_image, logs, error"]: + """ + Run the provided Python code to generate a static matplotlib chart. The resulting chart is returned as a base64 encoded image. + """ + api_key = get_secret("E2B_API_KEY") + + with Sandbox(api_key=api_key) as sbx: + execution = sbx.run_code(code=code) + + result = { + "base64_image": execution.results[0].png if execution.results else None, + "logs": execution.logs.to_json(), + "error": execution.error.to_json() if execution.error else None, + } + + return result diff --git a/toolkits/code_sandbox/arcade_code_sandbox/tools/models.py b/toolkits/code_sandbox/arcade_code_sandbox/tools/models.py new file mode 100644 index 00000000..34ff05bc --- /dev/null +++ b/toolkits/code_sandbox/arcade_code_sandbox/tools/models.py @@ -0,0 +1,10 @@ +from enum import Enum + + +# Models and enums for the e2b code interpreter +class E2BSupportedLanguage(str, Enum): + PYTHON = "python" + JAVASCRIPT = "js" + R = "r" + JAVA = "java" + BASH = "bash" diff --git a/toolkits/code_sandbox/arcade_code_sandbox/tools/utils.py b/toolkits/code_sandbox/arcade_code_sandbox/tools/utils.py new file mode 100644 index 00000000..c03b10da --- /dev/null +++ b/toolkits/code_sandbox/arcade_code_sandbox/tools/utils.py @@ -0,0 +1,9 @@ +import os +from typing import Any, Optional + + +def get_secret(name: str, default: Optional[Any] = None) -> Any: + secret = os.getenv(name) + if secret is None and default is not None: + return default + return secret diff --git a/toolkits/code_sandbox/evals/eval_e2b.py b/toolkits/code_sandbox/evals/eval_e2b.py new file mode 100644 index 00000000..9eb5c9b0 --- /dev/null +++ b/toolkits/code_sandbox/evals/eval_e2b.py @@ -0,0 +1,117 @@ +import arcade_code_sandbox +from arcade_code_sandbox.tools.e2b import create_static_matplotlib_chart, run_code +from arcade_code_sandbox.tools.models import E2BSupportedLanguage + +from arcade.core.catalog import ToolCatalog +from arcade.sdk.eval import ( + EvalRubric, + EvalSuite, + tool_eval, +) +from arcade.sdk.eval.critic import BinaryCritic, SimilarityCritic + +merge_sort_code = """ +def merge_sort(arr): + if len(arr) <= 1: + return arr + + mid = len(arr) // 2 + left = merge_sort(arr[:mid]) + right = merge_sort(arr[mid:]) + + return merge(left, right) + +def merge(left, right): + result = [] + i, j = 0, 0 + + while i < len(left) and j < len(right): + if left[i] < right[j]: + result.append(left[i]) + i += 1 + else: + result.append(right[j]) + j += 1 + + result.extend(left[i:]) + result.extend(right[j:]) + + return result + +sample_list = ["banana", "apple", "cherry", "date", "elderberry"] + +sorted_list = merge_sort(sample_list) +print("Sorted list:", sorted_list) +""" + +matplotlib_chart_code = """ +import matplotlib.pyplot as plt + +labels = ['Apples', 'Bananas', 'Cherries', 'Dates'] +sizes = [30, 25, 20, 25] +colors = ['red', 'yellow', 'purple', 'brown'] + +plt.pie(sizes, labels=labels, colors=colors, autopct='%1.1f%%', startangle=90) + +plt.axis('equal') + +plt.title('Fruit Distribution') + +plt.savefig('fruit_pie_chart.png') +""" + +# Evaluation rubric +rubric = EvalRubric( + fail_threshold=0.85, + warn_threshold=0.95, +) + + +catalog = ToolCatalog() +catalog.add_module(arcade_code_sandbox) + + +@tool_eval() +def code_sandbox_eval_suite(): + suite = EvalSuite( + name="code_sandbox Tools Evaluation", + system_message="You are an AI assistant with access to code_sandbox tools. Use them to help the user with their tasks.", + catalog=catalog, + rubric=rubric, + ) + + suite.add_case( + name="Run code", + user_message=f"Can you please run my merge sort algo?\n\n{merge_sort_code}", + expected_tool_calls=[ + ( + run_code, + { + "code": merge_sort_code, + "language": E2BSupportedLanguage.PYTHON, + }, + ) + ], + critics=[ + SimilarityCritic(critic_field="code", weight=0.8), + BinaryCritic(critic_field="language", weight=0.2), + ], + ) + + suite.add_case( + name="Create static matplotlib chart", + user_message=f"Run this code:\n\n{matplotlib_chart_code}", + expected_tool_calls=[ + ( + create_static_matplotlib_chart, + { + "code": matplotlib_chart_code, + }, + ) + ], + critics=[ + SimilarityCritic(critic_field="code", weight=1.0), + ], + ) + + return suite diff --git a/toolkits/code_sandbox/pyproject.toml b/toolkits/code_sandbox/pyproject.toml new file mode 100644 index 00000000..59263563 --- /dev/null +++ b/toolkits/code_sandbox/pyproject.toml @@ -0,0 +1,17 @@ +[tool.poetry] +name = "arcade_code_sandbox" +version = "0.1.0" +description = "LLM tools for running code in a sandbox" +authors = ["Arcade AI "] + +[tool.poetry.dependencies] +python = "^3.10" +arcade-ai = "0.1.*" +e2b-code-interpreter = "^1.0.1" + +[tool.poetry.dev-dependencies] +pytest = "^8.3.0" + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api"