code is perfect

This commit is contained in:
Madhu 2025-01-05 17:22:09 +05:30
parent 8cdb9bd663
commit 9fd044b81e
2 changed files with 195 additions and 362 deletions

View file

@ -1,202 +1,214 @@
import os
import json
import re
import sys
import io
import contextlib
import warnings
from typing import Optional, List, Any, Tuple
from dotenv import load_dotenv
from PIL import Image
import streamlit as st import streamlit as st
import pandas as pd import pandas as pd
import tempfile
import re
from together import Together
import csv
from dotenv import load_dotenv
import base64 import base64
import matplotlib.pyplot as plt from io import BytesIO
import io from together import Together
import seaborn as sns from e2b_code_interpreter import Sandbox
# Load environment variables # Suppress Pydantic warnings globally
load_dotenv() warnings.filterwarnings("ignore", category=UserWarning, module="pydantic")
# Function to preprocess and save the uploaded file to a temporary file # Regex pattern to extract code from LLM response
def preprocess_and_save(file): pattern = re.compile(r"```python\n(.*?)\n```", re.DOTALL)
try:
# Read the uploaded file into a DataFrame
if file.name.endswith('.csv'):
df = pd.read_csv(file, encoding='utf-8', na_values=['NA', 'N/A', 'missing'])
elif file.name.endswith('.xlsx'):
df = pd.read_excel(file, na_values=['NA', 'N/A', 'missing'])
else:
st.error("Unsupported file format. Please upload a CSV or Excel file.")
return None, None, None
def code_interpret(e2b_code_interpreter: Sandbox, code: str) -> Optional[List[Any]]:
"""
Runs the given Python code in the E2B sandbox.
# Ensure string columns are properly quoted Args:
for col in df.select_dtypes(include=['object']): e2b_code_interpreter: The E2B sandbox instance
df[col] = df[col].astype(str).replace({r'"': '""'}, regex=True) code: Python code to execute
# Parse dates and numeric columns Returns:
for col in df.columns: Optional[List[Any]]: Results from code execution
if 'date' in col.lower(): """
df[col] = pd.to_datetime(df[col], errors='coerce') with st.spinner('Executing code in E2B sandbox...'):
elif df[col].dtype == 'object': # Capture stdout and stderr
try: stdout_capture = io.StringIO()
# Handle columns with values like "4.1/5" stderr_capture = io.StringIO()
if df[col].str.contains('/').any():
# Split the values and take the first part (e.g., "4.1/5" -> 4.1)
df[col] = df[col].str.split('/').str[0]
# Convert to numeric, coerce errors to NaN
df[col] = pd.to_numeric(df[col], errors='coerce')
except (ValueError, TypeError):
# Keep as is if conversion fails
st.warning(f"Could not convert column '{col}' to numeric. Keeping as string.")
pass
# Drop rows with all NaN values with contextlib.redirect_stdout(stdout_capture), contextlib.redirect_stderr(stderr_capture):
df.dropna(how='all', inplace=True) # Suppress warnings during code execution
with warnings.catch_warnings():
warnings.simplefilter("ignore")
exec = e2b_code_interpreter.run_code(code)
# Create a temporary file to save the preprocessed data # Log stderr (warnings and errors) to the terminal
with tempfile.NamedTemporaryFile(delete=False, suffix=".csv") as temp_file: if stderr_capture.getvalue():
temp_path = temp_file.name print("[Code Interpreter Warnings/Errors]", file=sys.stderr)
# Save the DataFrame to the temporary CSV file print(stderr_capture.getvalue(), file=sys.stderr)
df.to_csv(temp_path, index=False, quoting=csv.QUOTE_ALL)
return temp_path, df.columns.tolist(), df # Return the DataFrame as well # Log stdout (normal output) to the terminal
except Exception as e: if stdout_capture.getvalue():
st.error(f"Error processing file: {e}") print("[Code Interpreter Output]", file=sys.stdout)
return None, None, None print(stdout_capture.getvalue(), file=sys.stdout)
# Function to execute Python code and generate plots if exec.error:
def execute_code(code: str, df): print(f"[Code Interpreter ERROR] {exec.error}", file=sys.stderr)
try:
# Define locals with necessary imports and the DataFrame
local_env = {
'pd': pd,
'df': df,
'plt': plt,
'sns': sns # if seaborn is needed
}
# Execute the code in the local environment
exec(code, globals(), local_env)
# Check if a plot was generated
if 'plt' in local_env:
# Save the plot to a BytesIO object
buf = io.BytesIO()
plt.savefig(buf, format='png')
plt.close()
buf.seek(0)
# Encode the plot as base64
base64_image = base64.b64encode(buf.read()).decode('utf-8')
return base64_image
else:
st.warning("No plot generated. Ensure the data being plotted is numeric.")
return None return None
except Exception as e: return exec.results
st.error(f"Error executing code: {e}")
return None
# Function to communicate with Together AI def match_code_blocks(llm_response: str) -> str:
def chat_with_llm(user_message, file_path, columns, df): """
print(f"\n{'='*50}\nUser message: {user_message}\n{'='*50}") Extracts Python code blocks from the LLM response.
# Update the system prompt with the file path, columns, and plot path Args:
system_prompt = SYSTEM_PROMPT.format( llm_response: The response from the LLM
file_path=file_path,
columns=columns,
)
# Add a hint to include a plot if the user asks for visualization Returns:
if "plot" in user_message.lower(): str: Extracted Python code or empty string
system_prompt += " Include a plot in your response and output the base64 string of the plot image." """
match = pattern.search(llm_response)
if match:
code = match.group(1)
return code
return ""
def chat_with_llm(e2b_code_interpreter: Sandbox, user_message: str, dataset_path: str) -> Tuple[Optional[List[Any]], str]:
"""
Sends the user message to the LLM and executes the generated code.
Args:
e2b_code_interpreter: The E2B sandbox instance
user_message: User's query message
dataset_path: Path to the uploaded dataset
Returns:
Tuple[Optional[List[Any]], str]: Code execution results and LLM response
"""
# Update system prompt to include dataset path information
system_prompt = f"""You're a Python data scientist and data visualization expert. You are given a dataset at path '{dataset_path}' and also the user's query.
You need to analyze the dataset and answer the user's query with a response and you run Python code to solve them.
IMPORTANT: Always use the dataset path variable '{dataset_path}' in your code when reading the CSV file."""
messages = [ messages = [
{"role": "system", "content": system_prompt}, {"role": "system", "content": system_prompt},
{"role": "user", "content": user_message}, {"role": "user", "content": user_message},
] ]
# Use the Together API key from session state with st.spinner('Getting response from Together AI LLM model...'):
response = client.chat.completions.create( client = Together(api_key=st.session_state.together_api_key)
model="meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo", response = client.chat.completions.create(
messages=messages, model=st.session_state.model_name,
) messages=messages,
)
response_message = response.choices[0].message.content response_message = response.choices[0].message
print("LLM Response:", response_message) # Debug: Print the LLM's response python_code = match_code_blocks(response_message.content)
python_code = match_code_blocks(response_message)
print("Extracted Python Code:", python_code) # Debug: Print the extracted code
if python_code: if python_code:
# Execute the code and generate the plot code_interpreter_results = code_interpret(e2b_code_interpreter, python_code)
base64_image = execute_code(python_code, df) return code_interpreter_results, response_message.content
return response_message, base64_image else:
else: st.warning(f"Failed to match any Python code in model's response")
print(f"Failed to match any Python code in model's response {response_message}") return None, response_message.content
return response_message, None
# Set up Streamlit app def upload_dataset(code_interpreter: Sandbox, uploaded_file) -> str:
st.title("AI Data Visualisation Agent") """
Uploads the dataset to the E2B sandbox.
# Sidebar for API keys and file upload Args:
st.sidebar.header("API Keys") code_interpreter: The E2B sandbox instance
together_api_key = st.sidebar.text_input("Together AI API Key", type="password") uploaded_file: Streamlit uploaded file
# Store API key in session state Returns:
if 'together_api_key' not in st.session_state: str: Path where file was uploaded
st.session_state.together_api_key = None """
dataset_path = f"./{uploaded_file.name}"
uploaded_file = st.sidebar.file_uploader("Upload CSV or Excel File", type=['csv', 'xlsx']) try:
code_interpreter.files.write(dataset_path, uploaded_file)
return dataset_path
except Exception as error:
st.error(f"Error during file upload: {error}")
raise error
# System prompt (dynamic based on the uploaded file)
SYSTEM_PROMPT = """
You are a Python data scientist and Visualisation expert. You have access to a CSV file located at '{file_path}'.
The dataset has the following columns: {columns}.
You can read this file into a DataFrame using `df = pd.read_csv('{file_path}')` and perform data analysis tasks based on user queries.
Make sure to handle missing values and data type inconsistencies. When generating plots,
use matplotlib or seaborn and output the plot as a base64 string.
Always check if the data being plotted is numeric. If the data is not numeric, preprocess it to convert it to numeric values.
Always respond with the Python code to answer the user's query, and include visualizations only if explicitly requested.
"""
# Function to match Python code blocks def main():
pattern = re.compile(r"```python\n(.*?)\n```", re.DOTALL) """Main Streamlit application."""
st.title("AI Data Visualization Agent")
st.write("Upload your dataset and ask questions about it!")
def match_code_blocks(llm_response): # Sidebar for API keys and model selection
match = pattern.search(llm_response) with st.sidebar:
if match: st.header("API Keys and Model Configuration")
code = match.group(1) st.session_state.together_api_key = st.text_input("Enter Together API Key", type="password")
# Remove comments and extra text st.session_state.e2b_api_key = st.text_input("Enter E2B API Key", type="password")
code = "\n".join([line for line in code.split("\n") if not line.strip().startswith("#")])
return code
return ""
# Main app logic # Add model selection dropdown
if uploaded_file: model_options = {
if not together_api_key: "Meta-Llama 3.1 405B": "meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo",
st.warning("Please provide the Together AI API key.") "DeepSeek V3": "deepseek-ai/DeepSeek-V3",
else: "Qwen 2.5 7B": "Qwen/Qwen2.5-7B-Instruct-Turbo",
# Update session state with API key "Meta-Llama 3.3 70B": "meta-llama/Llama-3.3-70B-Instruct-Turbo"
st.session_state.together_api_key = together_api_key }
selected_model = st.selectbox(
"Select Model",
options=list(model_options.keys()),
index=0 # Default to first option
)
st.session_state.model_name = model_options[selected_model]
# Initialize Together AI client only after confirming API key exists uploaded_file = st.file_uploader("Choose a CSV file", type="csv")
try:
client = Together(api_key=together_api_key)
# Preprocess and save the uploaded file if uploaded_file is not None:
temp_path, columns, df = preprocess_and_save(uploaded_file) # Display dataset with toggle
if temp_path: df = pd.read_csv(uploaded_file)
# Rest of your code for user query handling st.write("Dataset:")
user_query = st.text_input("Ask a query about the data:") show_full = st.checkbox("Show full dataset")
if st.button("Submit Query"): if show_full:
response_message, base64_image = chat_with_llm(user_query, temp_path, columns, df) st.dataframe(df)
else:
st.write("Preview (first 5 rows):")
st.dataframe(df.head())
# Query input
query = st.text_area("What would you like to know about your data?",
"Can you compare the average cost for two people between different categories?")
# Display AI's response if st.button("Analyze"):
st.write("AI's Response:") if not st.session_state.together_api_key or not st.session_state.e2b_api_key:
st.write(response_message) st.error("Please enter both API keys in the sidebar.")
# Display the plot if generated
if base64_image:
st.image(base64.b64decode(base64_image), use_container_width=True)
else:
st.write("No plot generated.")
else: else:
st.error("Failed to preprocess and save the data.") with Sandbox(api_key=st.session_state.e2b_api_key) as code_interpreter:
except Exception as e: # Upload the dataset
st.error(f"Error initializing Together AI client: {str(e)}") dataset_path = upload_dataset(code_interpreter, uploaded_file)
else:
st.warning("Please upload a file.") # Pass dataset_path to chat_with_llm
code_results, llm_response = chat_with_llm(code_interpreter, query, dataset_path)
# Display LLM's text response
st.write("AI Response:")
st.write(llm_response)
# Display results/visualizations
if code_results:
for result in code_results:
if hasattr(result, 'png') and result.png: # Check if PNG data is available
# Decode the base64-encoded PNG data
png_data = base64.b64decode(result.png)
# Convert PNG data to an image and display it
image = Image.open(BytesIO(png_data))
st.image(image, caption="Generated Visualization", use_container_width=False)
elif hasattr(result, 'figure'): # For matplotlib figures
fig = result.figure # Extract the matplotlib figure
st.pyplot(fig) # Display using st.pyplot
elif hasattr(result, 'show'): # For plotly figures
st.plotly_chart(result)
elif isinstance(result, (pd.DataFrame, pd.Series)):
st.dataframe(result)
else:
st.write(result)
if __name__ == "__main__":
main()

View file

@ -1,179 +0,0 @@
import os
import json
import re
from typing import Optional, List, Any, Tuple
from dotenv import load_dotenv
from PIL import Image
import io
import streamlit as st
import pandas as pd
import base64
from io import BytesIO
from PIL import Image
from together import Together
from e2b_code_interpreter import Sandbox
# Load environment variables
load_dotenv()
# Regex pattern to extract code from LLM response
pattern = re.compile(r"```python\n(.*?)\n```", re.DOTALL)
def code_interpret(e2b_code_interpreter: Sandbox, code: str) -> Optional[List[Any]]:
"""
Runs the given Python code in the E2B sandbox.
Args:
e2b_code_interpreter: The E2B sandbox instance
code: Python code to execute
Returns:
Optional[List[Any]]: Results from code execution
"""
with st.spinner('Executing code in E2B sandbox...'):
exec = e2b_code_interpreter.run_code(code,
on_stderr=lambda stderr: st.error(f"[Code Interpreter] {stderr}"),
on_stdout=lambda stdout: st.info(f"[Code Interpreter] {stdout}"))
if exec.error:
st.error(f"[Code Interpreter ERROR] {exec.error}")
return None
return exec.results
def match_code_blocks(llm_response: str) -> str:
"""
Extracts Python code blocks from the LLM response.
Args:
llm_response: The response from the LLM
Returns:
str: Extracted Python code or empty string
"""
match = pattern.search(llm_response)
if match:
code = match.group(1)
return code
return ""
def chat_with_llm(e2b_code_interpreter: Sandbox, user_message: str, dataset_path: str) -> Tuple[Optional[List[Any]], str]:
"""
Sends the user message to the LLM and executes the generated code.
Args:
e2b_code_interpreter: The E2B sandbox instance
user_message: User's query message
dataset_path: Path to the uploaded dataset
Returns:
Tuple[Optional[List[Any]], str]: Code execution results and LLM response
"""
# Update system prompt to include dataset path information
system_prompt = f"""You're a Python data scientist and data visualization expert. You are given a dataset at path '{dataset_path}' and also the user's query.
You need to analyze the dataset and answer the user's query with a response and you run Python code to solve them.
IMPORTANT: Always use the dataset path variable '{dataset_path}' in your code when reading the CSV file."""
messages = [
{"role": "system", "content": system_prompt},
{"role": "user", "content": user_message},
]
with st.spinner('Getting response from together AI...'):
client = Together(api_key=st.session_state.together_api_key)
response = client.chat.completions.create(
model=st.session_state.model_name,
messages=messages,
)
response_message = response.choices[0].message
python_code = match_code_blocks(response_message.content)
if python_code:
code_interpreter_results = code_interpret(e2b_code_interpreter, python_code)
return code_interpreter_results, response_message.content
else:
st.warning(f"Failed to match any Python code in model's response")
return None, response_message.content
def upload_dataset(code_interpreter: Sandbox, uploaded_file) -> str:
"""
Uploads the dataset to the E2B sandbox.
Args:
code_interpreter: The E2B sandbox instance
uploaded_file: Streamlit uploaded file
Returns:
str: Path where file was uploaded
"""
dataset_path = f"./{uploaded_file.name}"
try:
code_interpreter.files.write(dataset_path, uploaded_file)
return dataset_path
except Exception as error:
st.error(f"Error during file upload: {error}")
raise error
def main():
"""Main Streamlit application."""
st.title("AI Data Visualization Assistant")
st.write("Upload your dataset and ask questions about it!")
# Sidebar for API keys and model name
with st.sidebar:
st.header("API Keys and Model Configuration")
st.session_state.together_api_key = st.text_input("Enter Together API Key", type="password")
st.session_state.e2b_api_key = st.text_input("Enter E2B API Key", type="password")
st.session_state.model_name = st.text_input("Enter Model Name", value="meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo")
uploaded_file = st.file_uploader("Choose a CSV file", type="csv")
if uploaded_file is not None:
# Display dataset preview
df = pd.read_csv(uploaded_file)
st.write("Dataset Preview:")
st.dataframe(df.head())
# Query input
query = st.text_area("What would you like to know about your data?",
"Can you compare the average cost for two people between different categories?")
if st.button("Analyze"):
if not st.session_state.together_api_key or not st.session_state.e2b_api_key:
st.error("Please enter both API keys in the sidebar.")
else:
with Sandbox(api_key=st.session_state.e2b_api_key) as code_interpreter:
# Upload the dataset
dataset_path = upload_dataset(code_interpreter, uploaded_file)
# Pass dataset_path to chat_with_llm
code_results, llm_response = chat_with_llm(code_interpreter, query, dataset_path)
# Display LLM's text response
st.write("AI Response:")
st.write(llm_response)
# Display results/visualizations
if code_results:
for result in code_results:
if hasattr(result, 'png') and result.png: # Check if PNG data is available
# Decode the base64-encoded PNG data
png_data = base64.b64decode(result.png)
# Convert PNG data to an image and display it
image = Image.open(BytesIO(png_data))
st.image(image, caption="Generated Visualization", use_container_width=False)
elif hasattr(result, 'figure'): # For matplotlib figures
fig = result.figure # Extract the matplotlib figure
st.pyplot(fig) # Display using st.pyplot
elif hasattr(result, 'show'): # For plotly figures
st.plotly_chart(result)
elif isinstance(result, (pd.DataFrame, pd.Series)):
st.dataframe(result)
else:
st.write(result)
if __name__ == "__main__":
main()