From 154017c62f006617c2cfd46627b9217e6de140e7 Mon Sep 17 00:00:00 2001 From: Madhu Date: Thu, 30 Jan 2025 04:32:21 +0530 Subject: [PATCH] browser use to complete demo --- .../{main.py => ai_3dpygame_r1.py} | 47 +++++++++- ai_agent_tutorials/ai_3dpygame_r1/test.py | 87 +++++++++++++++++++ 2 files changed, 131 insertions(+), 3 deletions(-) rename ai_agent_tutorials/ai_3dpygame_r1/{main.py => ai_3dpygame_r1.py} (61%) create mode 100644 ai_agent_tutorials/ai_3dpygame_r1/test.py diff --git a/ai_agent_tutorials/ai_3dpygame_r1/main.py b/ai_agent_tutorials/ai_3dpygame_r1/ai_3dpygame_r1.py similarity index 61% rename from ai_agent_tutorials/ai_3dpygame_r1/main.py rename to ai_agent_tutorials/ai_3dpygame_r1/ai_3dpygame_r1.py index d534b25..eb93c85 100644 --- a/ai_agent_tutorials/ai_3dpygame_r1/main.py +++ b/ai_agent_tutorials/ai_3dpygame_r1/ai_3dpygame_r1.py @@ -1,7 +1,10 @@ import streamlit as st from openai import OpenAI -from phi.agent import Agent +from phi.agent import Agent as PhiAgent from phi.model.anthropic import Claude +import asyncio +from browser_use import Agent as BrowserAgent, SystemPrompt +from langchain_anthropic import ChatAnthropic st.set_page_config(page_title="PyGame Code Generator", layout="wide") @@ -68,8 +71,8 @@ if generate_btn and query: with st.expander("R1's Reasoning"): st.write(reasoning_content) - # Initialize Claude agent - claude_agent = Agent( + # Initialize Claude agent (using PhiAgent) + claude_agent = PhiAgent( model=Claude( id="claude-3-5-sonnet-20241022", api_key=st.session_state.api_keys["claude"] @@ -90,6 +93,44 @@ if generate_btn and query: with st.expander("Generated PyGame Code"): st.code(extracted_code, language="python") + # Initialize browser agent for Trinket interaction + async def run_pygame_on_trinket(code: str) -> None: + task_description = ( + "You are a Trinket.io PyGame expert. Follow these steps precisely:\n" + "1. Navigate to https://trinket.io/features/pygame\n" + "2. In the main.py file, clear any existing code in the editor\n" + "3. Paste this code into the editor:\n{0}\n" + "4. Click the Run button on the right to execute the code\n" + "5. Wait for the pygame visualisation to appear, once it does, view it for 10 seconds and then Quit the pygame window\n" + ).format(code) + + browser_agent = BrowserAgent( + task=task_description, + llm=ChatAnthropic( + model="claude-3-5-sonnet-20240620", + api_key=st.session_state.api_keys["claude"] + ), + max_actions_per_step=5, + max_failures=3 + ) + + with st.spinner("Running code on Trinket..."): + try: + result = await browser_agent.run() + if result and hasattr(result, 'final_response'): + share_url = result.final_response + st.success("Code is running on Trinket!") + st.write("You can view the visualization here:") + st.write(share_url) + else: + st.error("Failed to get a sharing URL from Trinket") + except Exception as e: + st.error(f"Error running code on Trinket: {str(e)}") + st.info("You can still copy the code above and run it manually on Trinket") + + # Run the async function + asyncio.run(run_pygame_on_trinket(extracted_code)) + except Exception as e: st.error(f"An error occurred: {str(e)}") diff --git a/ai_agent_tutorials/ai_3dpygame_r1/test.py b/ai_agent_tutorials/ai_3dpygame_r1/test.py new file mode 100644 index 0000000..9db9b5a --- /dev/null +++ b/ai_agent_tutorials/ai_3dpygame_r1/test.py @@ -0,0 +1,87 @@ +extracted_code = """import pygame +import random + +pygame.init() + +screen_width = 800 +screen_height = 600 +screen = pygame.display.set_mode((screen_width, screen_height)) +pygame.display.set_caption("Bouncing Rectangle") + +clock = pygame.time.Clock() + +# Rectangle properties +rect_width = 50 +rect_height = 30 +x = screen_width // 2 - rect_width // 2 +y = screen_height // 2 - rect_height // 2 +dx = 5 +dy = 5 + +colors = [(255, 0, 0), (0, 255, 0), (0, 0, 255), (255, 255, 0), (0, 255, 255), (255, 0, 255)] +current_color = random.choice(colors) + +running = True +while running: + for event in pygame.event.get(): + if event.type == pygame.QUIT: + running = False + + # Update position + x += dx + y += dy + + # Check collisions and change direction/color + if x <= 0 or x + rect_width >= screen_width: + dx = -dx + current_color = random.choice(colors) + if y <= 0 or y + rect_height >= screen_height: + dy = -dy + current_color = random.choice(colors) + + # Fill the screen with a background color + screen.fill((0, 0, 0)) + + # Draw the rectangle + pygame.draw.rect(screen, current_color, (x, y, rect_width, rect_height)) + + pygame.display.flip() + clock.tick(60) + +pygame.quit()""" + +import asyncio +from browser_use import Agent as BrowserAgent, SystemPrompt +from langchain_anthropic import ChatAnthropic +from langchain_openai import ChatOpenAI +import os +from dotenv import load_dotenv + +load_dotenv() + + +async def run_pygame_on_trinket(code: str) -> None: + task_description = ( + "You are a Trinket.io PyGame expert. Follow these steps precisely:\n" + "1. Navigate to https://trinket.io/features/pygame\n" + "2. In the main.py file, clear and delete the existing code. If you dont know how to do it, copy the whole code by command + A and delete it.\n" + "3. Paste this code into the editor:\n{0} by control\n" + "4. Click the Run button on the right to execute the code\n" + "5. Wait for the pygame visualisation to appear, once it does, view it for 10 seconds and then Quit the pygame window\n" + "6. If you have any issues, try to fix them by yourself. If you cannot fix them, ask the user for help.\n" + ).format(code) + + browser_agent = BrowserAgent( + task=task_description, + llm=ChatOpenAI( + model="gpt-4o", + api_key=os.getenv("OPENAI_API_KEY") + ), + max_actions_per_step=10, + max_failures=25 + ) + + result = await browser_agent.run() + +# Run the async function +asyncio.run(run_pygame_on_trinket(extracted_code)) \ No newline at end of file