From 2ab1ac8cd4beb05e7eeb73488bcefdc4c59695bc Mon Sep 17 00:00:00 2001 From: Madhu Date: Tue, 25 Mar 2025 02:58:25 +0530 Subject: [PATCH 1/5] Add AI agent tutorials files --- .../ai_voice_agent_openaisdk/README.md | 0 .../ai_voice_agent_docs.py | 360 ++++++++++++++++++ .../ai_voice_agent_openaisdk/requirements.txt | 0 3 files changed, 360 insertions(+) create mode 100644 ai_agent_tutorials/ai_voice_agent_openaisdk/README.md create mode 100644 ai_agent_tutorials/ai_voice_agent_openaisdk/ai_voice_agent_docs.py create mode 100644 ai_agent_tutorials/ai_voice_agent_openaisdk/requirements.txt diff --git a/ai_agent_tutorials/ai_voice_agent_openaisdk/README.md b/ai_agent_tutorials/ai_voice_agent_openaisdk/README.md new file mode 100644 index 0000000..e69de29 diff --git a/ai_agent_tutorials/ai_voice_agent_openaisdk/ai_voice_agent_docs.py b/ai_agent_tutorials/ai_voice_agent_openaisdk/ai_voice_agent_docs.py new file mode 100644 index 0000000..329a833 --- /dev/null +++ b/ai_agent_tutorials/ai_voice_agent_openaisdk/ai_voice_agent_docs.py @@ -0,0 +1,360 @@ +from typing import List, Dict, Optional +from dataclasses import dataclass +from pathlib import Path +import os +from firecrawl import FirecrawlApp +from qdrant_client import QdrantClient +from qdrant_client.http import models +from qdrant_client.http.models import Distance, VectorParams +from fastembed import TextEmbedding +from agents import Agent, ModelSettings, function_tool, Runner +from openai import OpenAI, AsyncOpenAI +from openai.helpers import LocalAudioPlayer +import textwrap +import tempfile +import uuid +import numpy as np +from typing import Callable +from urllib.parse import urlparse +from dotenv import load_dotenv +import asyncio +import json +from datetime import datetime +import time + +load_dotenv() + + + +def setup_qdrant_collection(qdrant_url: str, qdrant_api_key: str, collection_name: str = "docs_embeddings"): + print("\n--- Step 1: Setting up Qdrant Collection ---") + try: + client = QdrantClient(url=qdrant_url, api_key=qdrant_api_key) + print("✓ Connected to Qdrant") + + embedding_model = TextEmbedding() + test_embedding = list(embedding_model.embed(["test"]))[0] + embedding_dim = len(test_embedding) + print(f"✓ Embedding model ready (dimension: {embedding_dim})") + + client.create_collection( + collection_name=collection_name, + vectors_config=VectorParams(size=embedding_dim, distance=Distance.COSINE) + ) + print(f"✓ Created collection: {collection_name}") + + return client, embedding_model + + except Exception as e: + if "already exists" in str(e): + print(f"✓ Collection {collection_name} already exists") + return client, embedding_model + raise e + +def crawl_documentation(firecrawl_api_key: str, url: str, output_dir: Optional[str] = None): + print("\n--- Step 2: Crawling Documentation ---") + try: + firecrawl = FirecrawlApp(api_key=firecrawl_api_key) + print(f"✓ Initialized Firecrawl") + + if output_dir: + os.makedirs(output_dir, exist_ok=True) + print(f"✓ Created output directory: {output_dir}") + + print(f"Starting crawl of {url}...") + + pages = [] + + response = firecrawl.crawl_url( + url, + params={ + 'limit': 5, + 'scrapeOptions': { + 'formats': ['markdown', 'html'] + } + } + ) + + while True: + if response.get('status') == 'scraping': + print(f"Progress: {response.get('completed', 0)}/{response.get('total', 0)} pages") + print(f"Credits used: {response.get('creditsUsed', 0)}") + + for page in response.get('data', []): + content = page.get('markdown') or page.get('html', '') + metadata = page.get('metadata', {}) + source_url = metadata.get('sourceURL', '') + + if output_dir and content: + filename = f"{uuid.uuid4()}.md" + filepath = os.path.join(output_dir, filename) + with open(filepath, 'w', encoding='utf-8') as f: + f.write(content) + + pages.append({ + "content": content, + "url": source_url, + "metadata": { + "title": metadata.get('title', ''), + "description": metadata.get('description', ''), + "language": metadata.get('language', 'en'), + "crawl_date": datetime.now().isoformat() + } + }) + + print(f"✓ Processed page: {metadata.get('title', 'Untitled')}") + + next_url = response.get('next') + if not next_url: + break + + response = firecrawl.get(next_url) + time.sleep(1) + + print(f"✓ Crawled {len(pages)} pages") + return pages + + except Exception as e: + print(f"Error crawling documentation: {str(e)}") + raise e + +def store_embeddings(client: QdrantClient, embedding_model: TextEmbedding, pages: List[Dict], collection_name: str): + print("\n--- Step 3: Generating and Storing Embeddings ---") + try: + for page in pages: + embedding = list(embedding_model.embed([page["content"]]))[0] + + client.upsert( + collection_name=collection_name, + points=[ + models.PointStruct( + id=str(uuid.uuid4()), + vector=embedding.tolist(), + payload={ + "content": page["content"], + "url": page["url"], + **page["metadata"] + } + ) + ] + ) + print(f"✓ Stored embedding for: {page['metadata']['title'] or page['url']}") + + print(f"✓ Stored {len(pages)} embeddings in Qdrant") + + except Exception as e: + print(f"Error storing embeddings: {str(e)}") + raise e + +def setup_agents(openai_api_key: str): + print("\n--- Step 4: Setting up OpenAI Agents ---") + try: + # Set OpenAI API key in environment + os.environ["OPENAI_API_KEY"] = openai_api_key + print("✓ Set OpenAI API key in environment") + + processor_agent = Agent( + name="Documentation Processor", + instructions="""You are a helpful documentation assistant. Your task is to: + 1. Analyze the provided documentation content + 2. Answer the user's question clearly and concisely + 3. Include relevant examples when available + 4. Cite the source URLs when referencing specific content + 5. Keep responses natural and conversational + 6. Format your response in a way that's easy to speak out loud""", + model="gpt-4o" + ) + print("✓ Set up Documentation Processor Agent") + + tts_agent = Agent( + name="Text-to-Speech Agent", + instructions="""You are a text-to-speech agent. Your task is to: + 1. Convert the processed documentation response into natural speech + 2. Maintain proper pacing and emphasis + 3. Handle technical terms clearly + 4. Keep the tone professional but friendly + 5. Use appropriate pauses for better comprehension + 6. Ensure the speech is clear and well-articulated""", + model="gpt-4o-mini-tts" + ) + print("✓ Set up TTS Agent") + + return processor_agent, tts_agent + + except Exception as e: + print(f"Error setting up agents: {str(e)}") + raise e + +async def process_query( + query: str, + client: QdrantClient, + embedding_model: TextEmbedding, + processor_agent: Agent, + tts_agent: Agent, + collection_name: str, + openai_api_key: str +): + print("\n--- Step 5: Processing Query ---") + try: + # Generate query embedding + print("Generating query embedding...") + query_embedding = list(embedding_model.embed([query]))[0] + print(f"✓ Generated query embedding with shape: {len(query_embedding)}") + print(f"Vector sample (first 5 elements): {query_embedding[:5]}") + + # Try to get collection info first + print("\nVerifying collection status...") + try: + collection_info = client.get_collection(collection_name) + print(f"Collection exists with {collection_info.points_count} points") + except Exception as e: + print(f"Warning: Could not get collection info: {str(e)}") + + # Attempt search with query parameter (confirmed working) + print("\nAttempting vector search...") + try: + print("Querying with 'query' parameter...") + search_response = client.query_points( + collection_name=collection_name, + query=query_embedding.tolist(), + limit=3, + with_payload=True + ) + print("✓ Query successful") + + # Debug search response + print("\nSearch Response Debug:") + print(f"Response type: {type(search_response)}") + + # Get points from the response + if hasattr(search_response, 'points'): + search_results = search_response.points + else: + search_results = [] + + print(f"\n✓ Found {len(search_results)} relevant documents") + + if not search_results: + raise Exception("No relevant documents found in the vector database") + + # Build context from search results + context = "Based on the following documentation:\n\n" + for result in search_results: + payload = result.payload + if not payload: + print(f"Warning: Result missing payload") + continue + + url = payload.get('url', 'Unknown URL') + content = payload.get('content', '') + score = getattr(result, 'score', 'N/A') + + print(f"\nDocument from {url}") + print(f"Relevance score: {score}") + context += f"From {url}:\n{content}\n\n" + + context += f"\nUser Question: {query}\n\n" + context += "Please provide a clear, concise answer that can be easily spoken out loud." + + # Process response with agents + print("\nProcessing with Documentation Agent...") + processor_result = await Runner.run(processor_agent, context) + processor_response = processor_result.final_output + print("✓ Generated text response") + + print("\nProcessing with TTS Agent...") + tts_result = await Runner.run(tts_agent, processor_response) + tts_response = tts_result.final_output + print("✓ Generated TTS instructions") + + # Generate and play audio + print("\nGenerating audio response...") + async_openai = AsyncOpenAI(api_key=openai_api_key) + async with async_openai.audio.speech.with_streaming_response.create( + model="tts-1", + voice="alloy", + input=processor_response, + instructions=tts_response, + response_format="pcm" + ) as response: + print("✓ Streaming audio response") + await LocalAudioPlayer().play(response) + + return { + "status": "success", + "text_response": processor_response, + "tts_instructions": tts_response, + "sources": [r.payload.get("url", "Unknown URL") for r in search_results if r.payload], + "query_details": { + "vector_size": len(query_embedding), + "results_found": len(search_results), + "collection_name": collection_name + } + } + + except Exception as e: + print(f"Error during vector search: {str(e)}") + print("Full error details:") + import traceback + traceback.print_exc() + raise + + except Exception as e: + print(f"\nError processing query: {str(e)}") + print("Full error details:") + import traceback + traceback.print_exc() + + return { + "status": "error", + "error": str(e), + "error_details": traceback.format_exc(), + "query": query + } + +async def main(): + try: + env_vars = get_env_vars() + print("✓ Loaded environment variables") + + client, embedding_model = setup_qdrant_collection( + env_vars["QDRANT_URL"], + env_vars["QDRANT_API_KEY"] + ) + + pages = crawl_documentation( + env_vars["FIRECRAWL_API_KEY"], + "https://docs.agentmail.to/api-reference", + "crawled_docs" + ) + + store_embeddings(client, embedding_model, pages, "docs_embeddings") + + processor_agent, tts_agent = setup_agents(env_vars["OPENAI_API_KEY"]) + + query = "What are the required parameters for List Threads API of Agent Mail?" + result = await process_query( + query, + client, + embedding_model, + processor_agent, + tts_agent, + "docs_embeddings", + env_vars["OPENAI_API_KEY"] + ) + + print("\n--- Final Results ---") + print(json.dumps(result, indent=2)) + + except ValueError as e: + print(f"\nConfiguration Error: {str(e)}") + print("\nPlease ensure your .env file contains all required variables:") + print("FIRECRAWL_API_KEY=your_key") + print("QDRANT_URL=your_qdrant_url") + print("QDRANT_API_KEY=your_qdrant_key") + print("OPENAI_API_KEY=your_openai_key") + except Exception as e: + print(f"\nError: {str(e)}") + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/ai_agent_tutorials/ai_voice_agent_openaisdk/requirements.txt b/ai_agent_tutorials/ai_voice_agent_openaisdk/requirements.txt new file mode 100644 index 0000000..e69de29 From dc76c43d79fb3df0ee4c7b1d9ad7ac0b9490aa38 Mon Sep 17 00:00:00 2001 From: Madhu Date: Tue, 25 Mar 2025 03:06:51 +0530 Subject: [PATCH 2/5] requirements + readme --- ai_agent_tutorials/ai_voice_agent_openaisdk/README.md | 1 + .../ai_voice_agent_openaisdk/requirements.txt | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/ai_agent_tutorials/ai_voice_agent_openaisdk/README.md b/ai_agent_tutorials/ai_voice_agent_openaisdk/README.md index e69de29..912dbf2 100644 --- a/ai_agent_tutorials/ai_voice_agent_openaisdk/README.md +++ b/ai_agent_tutorials/ai_voice_agent_openaisdk/README.md @@ -0,0 +1 @@ +## AI Customer Support Voice Agent Team with OpenAI Agents SDK \ No newline at end of file diff --git a/ai_agent_tutorials/ai_voice_agent_openaisdk/requirements.txt b/ai_agent_tutorials/ai_voice_agent_openaisdk/requirements.txt index e69de29..18b68f3 100644 --- a/ai_agent_tutorials/ai_voice_agent_openaisdk/requirements.txt +++ b/ai_agent_tutorials/ai_voice_agent_openaisdk/requirements.txt @@ -0,0 +1,7 @@ +firecrawl-py +qdrant-client +streamlit +fastembed +openai>=1.0.0 +python-dotenv +openai-agents From d16e2b757d1c5558b23e91c8aafc92107a13499d Mon Sep 17 00:00:00 2001 From: Madhu Date: Tue, 25 Mar 2025 04:02:38 +0530 Subject: [PATCH 3/5] streamlit inclusion --- .../ai_voice_agent_docs.py | 401 +++++++++++------- 1 file changed, 255 insertions(+), 146 deletions(-) diff --git a/ai_agent_tutorials/ai_voice_agent_openaisdk/ai_voice_agent_docs.py b/ai_agent_tutorials/ai_voice_agent_openaisdk/ai_voice_agent_docs.py index 329a833..895de99 100644 --- a/ai_agent_tutorials/ai_voice_agent_openaisdk/ai_voice_agent_docs.py +++ b/ai_agent_tutorials/ai_voice_agent_openaisdk/ai_voice_agent_docs.py @@ -21,10 +21,129 @@ import asyncio import json from datetime import datetime import time +import streamlit as st load_dotenv() +def init_session_state(): + """Initialize session state variables for storing API keys and configurations.""" + defaults = { + "initialized": False, + "qdrant_url": "", + "qdrant_api_key": "", + "firecrawl_api_key": "", + "openai_api_key": "", + "doc_url": "", + "setup_complete": False, + "client": None, + "embedding_model": None, + "processor_agent": None, + "tts_agent": None, + "selected_voice": "coral" # Default voice + } + + for key, value in defaults.items(): + if key not in st.session_state: + st.session_state[key] = value +def sidebar_config(): + """Render and handle the configuration sidebar.""" + with st.sidebar: + st.title("🔑 Configuration") + st.markdown("---") + + # API Keys and URLs + st.session_state.qdrant_url = st.text_input( + "Qdrant URL", + value=st.session_state.qdrant_url, + type="password" + ) + st.session_state.qdrant_api_key = st.text_input( + "Qdrant API Key", + value=st.session_state.qdrant_api_key, + type="password" + ) + st.session_state.firecrawl_api_key = st.text_input( + "Firecrawl API Key", + value=st.session_state.firecrawl_api_key, + type="password" + ) + st.session_state.openai_api_key = st.text_input( + "OpenAI API Key", + value=st.session_state.openai_api_key, + type="password" + ) + + st.markdown("---") + st.session_state.doc_url = st.text_input( + "Documentation URL", + value=st.session_state.doc_url, + placeholder="https://docs.example.com" + ) + + # Voice selection + st.markdown("---") + st.markdown("### 🎤 Voice Settings") + voices = ["alloy", "ash", "ballad", "coral", "echo", "fable", "onyx", "nova", "sage", "shimmer", "verse"] + st.session_state.selected_voice = st.selectbox( + "Select Voice", + options=voices, + index=voices.index(st.session_state.selected_voice), + help="Choose the voice for the audio response" + ) + + # Setup button + if st.button("Initialize System", type="primary"): + if all([ + st.session_state.qdrant_url, + st.session_state.qdrant_api_key, + st.session_state.firecrawl_api_key, + st.session_state.openai_api_key, + st.session_state.doc_url + ]): + progress_placeholder = st.empty() + with progress_placeholder.container(): + try: + # Setup Qdrant + st.markdown("🔄 Setting up Qdrant connection...") + client, embedding_model = setup_qdrant_collection( + st.session_state.qdrant_url, + st.session_state.qdrant_api_key + ) + st.session_state.client = client + st.session_state.embedding_model = embedding_model + st.markdown("✅ Qdrant setup complete!") + + # Crawl documentation + st.markdown("🔄 Crawling documentation pages...") + pages = crawl_documentation( + st.session_state.firecrawl_api_key, + st.session_state.doc_url + ) + st.markdown(f"✅ Crawled {len(pages)} documentation pages!") + + # Store embeddings + store_embeddings( + client, + embedding_model, + pages, + "docs_embeddings" + ) + + # Setup agents + processor_agent, tts_agent = setup_agents( + st.session_state.openai_api_key + ) + st.session_state.processor_agent = processor_agent + st.session_state.tts_agent = tts_agent + + st.session_state.setup_complete = True + st.success("✅ System initialized successfully!") + + except Exception as e: + st.error(f"Error during setup: {str(e)}") + else: + st.error("Please fill in all the required fields!") def setup_qdrant_collection(qdrant_url: str, qdrant_api_key: str, collection_name: str = "docs_embeddings"): print("\n--- Step 1: Setting up Qdrant Collection ---") @@ -194,167 +313,157 @@ async def process_query( collection_name: str, openai_api_key: str ): - print("\n--- Step 5: Processing Query ---") try: # Generate query embedding - print("Generating query embedding...") query_embedding = list(embedding_model.embed([query]))[0] - print(f"✓ Generated query embedding with shape: {len(query_embedding)}") - print(f"Vector sample (first 5 elements): {query_embedding[:5]}") - # Try to get collection info first - print("\nVerifying collection status...") - try: - collection_info = client.get_collection(collection_name) - print(f"Collection exists with {collection_info.points_count} points") - except Exception as e: - print(f"Warning: Could not get collection info: {str(e)}") + # Search in Qdrant + search_response = client.query_points( + collection_name=collection_name, + query=query_embedding.tolist(), + limit=3, + with_payload=True + ) - # Attempt search with query parameter (confirmed working) - print("\nAttempting vector search...") - try: - print("Querying with 'query' parameter...") - search_response = client.query_points( - collection_name=collection_name, - query=query_embedding.tolist(), - limit=3, - with_payload=True - ) - print("✓ Query successful") - - # Debug search response - print("\nSearch Response Debug:") - print(f"Response type: {type(search_response)}") - - # Get points from the response - if hasattr(search_response, 'points'): - search_results = search_response.points - else: - search_results = [] + search_results = search_response.points if hasattr(search_response, 'points') else [] + + if not search_results: + raise Exception("No relevant documents found in the vector database") + + # Build context from search results + context = "Based on the following documentation:\n\n" + for result in search_results: + payload = result.payload + if not payload: + continue + url = payload.get('url', 'Unknown URL') + content = payload.get('content', '') + context += f"From {url}:\n{content}\n\n" + + context += f"\nUser Question: {query}\n\n" + context += "Please provide a clear, concise answer that can be easily spoken out loud." + + # Process response with agents + processor_result = await Runner.run(processor_agent, context) + processor_response = processor_result.final_output + + tts_result = await Runner.run(tts_agent, processor_response) + tts_response = tts_result.final_output + + # Generate audio + async_openai = AsyncOpenAI(api_key=openai_api_key) + audio_response = await async_openai.audio.speech.create( + model="gpt-4o-mini-tts", + voice=st.session_state.selected_voice, + input=processor_response, + instructions=tts_response, + response_format="mp3" + ) + + # Save audio to a temporary file + temp_dir = tempfile.gettempdir() + audio_path = os.path.join(temp_dir, f"response_{uuid.uuid4()}.mp3") + + # Write the audio content to the file + with open(audio_path, "wb") as f: + f.write(audio_response.content) - print(f"\n✓ Found {len(search_results)} relevant documents") - - if not search_results: - raise Exception("No relevant documents found in the vector database") - - # Build context from search results - context = "Based on the following documentation:\n\n" - for result in search_results: - payload = result.payload - if not payload: - print(f"Warning: Result missing payload") - continue - - url = payload.get('url', 'Unknown URL') - content = payload.get('content', '') - score = getattr(result, 'score', 'N/A') - - print(f"\nDocument from {url}") - print(f"Relevance score: {score}") - context += f"From {url}:\n{content}\n\n" - - context += f"\nUser Question: {query}\n\n" - context += "Please provide a clear, concise answer that can be easily spoken out loud." - - # Process response with agents - print("\nProcessing with Documentation Agent...") - processor_result = await Runner.run(processor_agent, context) - processor_response = processor_result.final_output - print("✓ Generated text response") - - print("\nProcessing with TTS Agent...") - tts_result = await Runner.run(tts_agent, processor_response) - tts_response = tts_result.final_output - print("✓ Generated TTS instructions") - - # Generate and play audio - print("\nGenerating audio response...") - async_openai = AsyncOpenAI(api_key=openai_api_key) - async with async_openai.audio.speech.with_streaming_response.create( - model="tts-1", - voice="alloy", - input=processor_response, - instructions=tts_response, - response_format="pcm" - ) as response: - print("✓ Streaming audio response") - await LocalAudioPlayer().play(response) - - return { - "status": "success", - "text_response": processor_response, - "tts_instructions": tts_response, - "sources": [r.payload.get("url", "Unknown URL") for r in search_results if r.payload], - "query_details": { - "vector_size": len(query_embedding), - "results_found": len(search_results), - "collection_name": collection_name - } + return { + "status": "success", + "text_response": processor_response, + "tts_instructions": tts_response, + "audio_path": audio_path, + "sources": [r.payload.get("url", "Unknown URL") for r in search_results if r.payload], + "query_details": { + "vector_size": len(query_embedding), + "results_found": len(search_results), + "collection_name": collection_name } - - except Exception as e: - print(f"Error during vector search: {str(e)}") - print("Full error details:") - import traceback - traceback.print_exc() - raise + } except Exception as e: print(f"\nError processing query: {str(e)}") - print("Full error details:") - import traceback - traceback.print_exc() - return { "status": "error", "error": str(e), - "error_details": traceback.format_exc(), "query": query } -async def main(): - try: - env_vars = get_env_vars() - print("✓ Loaded environment variables") - - client, embedding_model = setup_qdrant_collection( - env_vars["QDRANT_URL"], - env_vars["QDRANT_API_KEY"] - ) - - pages = crawl_documentation( - env_vars["FIRECRAWL_API_KEY"], - "https://docs.agentmail.to/api-reference", - "crawled_docs" - ) - - store_embeddings(client, embedding_model, pages, "docs_embeddings") - - processor_agent, tts_agent = setup_agents(env_vars["OPENAI_API_KEY"]) - - query = "What are the required parameters for List Threads API of Agent Mail?" - result = await process_query( - query, - client, - embedding_model, - processor_agent, - tts_agent, - "docs_embeddings", - env_vars["OPENAI_API_KEY"] - ) - - print("\n--- Final Results ---") - print(json.dumps(result, indent=2)) - - except ValueError as e: - print(f"\nConfiguration Error: {str(e)}") - print("\nPlease ensure your .env file contains all required variables:") - print("FIRECRAWL_API_KEY=your_key") - print("QDRANT_URL=your_qdrant_url") - print("QDRANT_API_KEY=your_qdrant_key") - print("OPENAI_API_KEY=your_openai_key") - except Exception as e: - print(f"\nError: {str(e)}") +def run_streamlit(): + """Main Streamlit application.""" + st.set_page_config( + page_title="AI Voice Documentation Agent Team", + page_icon="🎙️", + layout="wide" + ) + + init_session_state() + sidebar_config() + + # Main content area + st.title("🎙️ AI Voice Documentation Agent Team") + st.markdown(""" + Get OpenAI SDK voice-powered answers to your documentation questions! Simply: + 1. Configure your API keys in the sidebar + 2. Enter the documentation URL you want to learn about or have questions about + 3. Ask your question below and get both text and voice responses + """) + + # Query input and processing + query = st.text_input( + "What would you like to know about the documentation?", + placeholder="e.g., How do I authenticate API requests?", + disabled=not st.session_state.setup_complete + ) + + if query and st.session_state.setup_complete: + with st.status("Processing your query...", expanded=True) as status: + try: + st.markdown("🔄 Searching documentation and generating response...") + result = asyncio.run(process_query( + query, + st.session_state.client, + st.session_state.embedding_model, + st.session_state.processor_agent, + st.session_state.tts_agent, + "docs_embeddings", + st.session_state.openai_api_key + )) + + if result["status"] == "success": + status.update(label="✅ Query processed!", state="complete") + + st.markdown("### Response:") + st.write(result["text_response"]) + + if "audio_path" in result: + st.markdown(f"### 🔊 Audio Response (Voice: {st.session_state.selected_voice})") + # Pass the file path directly to st.audio + st.audio(result["audio_path"], format="audio/mp3", start_time=0) + + # For download button, we still need to read the bytes + with open(result["audio_path"], "rb") as audio_file: + audio_bytes = audio_file.read() + st.download_button( + label="📥 Download Audio Response", + data=audio_bytes, + file_name=f"voice_response_{st.session_state.selected_voice}.mp3", + mime="audio/mp3" + ) + + st.markdown("### Sources:") + for source in result["sources"]: + st.markdown(f"- {source}") + else: + status.update(label="❌ Error processing query", state="error") + st.error(f"Error: {result.get('error', 'Unknown error occurred')}") + + except Exception as e: + status.update(label="❌ Error processing query", state="error") + st.error(f"Error processing query: {str(e)}") + + elif not st.session_state.setup_complete: + st.info("👈 Please configure the system using the sidebar first!") if __name__ == "__main__": - asyncio.run(main()) \ No newline at end of file + run_streamlit() \ No newline at end of file From 6f0dc1fb35ef505952643d45dc6467cf3b6a08e5 Mon Sep 17 00:00:00 2001 From: Madhu Date: Tue, 25 Mar 2025 04:14:05 +0530 Subject: [PATCH 4/5] Final push --- .../ai_voice_agent_openaisdk/README.md | 69 ++++++++++++++++++- 1 file changed, 68 insertions(+), 1 deletion(-) diff --git a/ai_agent_tutorials/ai_voice_agent_openaisdk/README.md b/ai_agent_tutorials/ai_voice_agent_openaisdk/README.md index 912dbf2..ac04b8d 100644 --- a/ai_agent_tutorials/ai_voice_agent_openaisdk/README.md +++ b/ai_agent_tutorials/ai_voice_agent_openaisdk/README.md @@ -1 +1,68 @@ -## AI Customer Support Voice Agent Team with OpenAI Agents SDK \ No newline at end of file +# 🎙️ AI Customer Support Voice Agent Team for Documentations + +An OpenAISDK powered Agent application that transforms technical documentation into interactive voice-powered responses using OpenAI's GPT-4o-mini and TTS capabilities. The system crawls through documentation websites with firecrawl's Crawl Endpoint, processes the content and embeds them in qdrant, and provides both text and voice responses to user queries, making documentation more accessible and interactive. + +## Features + +- **Documentation Processing** + - Crawls documentation websites using Firecrawl + - Stores and indexes content using Qdrant vector database + - Generates embeddings for semantic search capabilities + +- **AI Agent Team** + - **Documentation Processor**: Analyzes documentation content and generates clear, concise responses to user queries + - **TTS Agent**: Converts text responses into natural-sounding speech with appropriate pacing and emphasis + - **Voice Customization**: Supports multiple OpenAI TTS voices: + - alloy, ash, ballad, coral, echo, fable, onyx, nova, sage, shimmer, verse + +- **Interactive Interface** + - Clean Streamlit UI with sidebar configuration + - Real-time documentation search and response generation + - Built-in audio player with download capability + - Progress indicators for system initialization and query processing + +## How to Run + +1. **Setup Environment** + ```bash + # Clone the repository + git clone https://github.com/Shubhamsaboo/awesome-llm-apps.git + cd awesome-llm-apps/ai_agent_tutorials/ai_voice_agent_openaisdk + + # Install dependencies + pip install -r requirements.txt + ``` + +2. **Configure API Keys** + - Get OpenAI API key from [OpenAI Platform](https://platform.openai.com) + - Get Qdrant API key and URL from [Qdrant Cloud](https://cloud.qdrant.io) + - Get Firecrawl API key for documentation crawling + +3. **Run the Application** + ```bash + streamlit run ai_voice_agent_docs.py + ``` + +4. **Use the Interface** + - Enter API credentials in the sidebar + - Input the documentation URL you want to learn about + - Select your preferred voice from the dropdown + - Click "Initialize System" to process the documentation + - Ask questions and receive both text and voice responses + +## Features in Detail + +- **Documentation Crawling** + - Automatically extracts content from documentation websites + - Preserves document structure and metadata + - Supports multiple page crawling + +- **Vector Search** + - Uses FastEmbed for generating embeddings + - Semantic search capabilities for finding relevant content + - Efficient document retrieval using Qdrant + +- **Voice Generation** + - High-quality text-to-speech using OpenAI's TTS models + - Multiple voice options for customization + - Natural speech patterns with proper pacing and emphasis From 0bec4eef9793c5131447e34c361c69a4c5a8616d Mon Sep 17 00:00:00 2001 From: Madhu Date: Tue, 25 Mar 2025 04:19:32 +0530 Subject: [PATCH 5/5] Final commit - working fine --- .../ai_voice_agent_docs.py | 280 +++++++----------- 1 file changed, 102 insertions(+), 178 deletions(-) diff --git a/ai_agent_tutorials/ai_voice_agent_openaisdk/ai_voice_agent_docs.py b/ai_agent_tutorials/ai_voice_agent_openaisdk/ai_voice_agent_docs.py index 895de99..1ecaaba 100644 --- a/ai_agent_tutorials/ai_voice_agent_openaisdk/ai_voice_agent_docs.py +++ b/ai_agent_tutorials/ai_voice_agent_openaisdk/ai_voice_agent_docs.py @@ -1,5 +1,4 @@ from typing import List, Dict, Optional -from dataclasses import dataclass from pathlib import Path import os from firecrawl import FirecrawlApp @@ -7,26 +6,19 @@ from qdrant_client import QdrantClient from qdrant_client.http import models from qdrant_client.http.models import Distance, VectorParams from fastembed import TextEmbedding -from agents import Agent, ModelSettings, function_tool, Runner -from openai import OpenAI, AsyncOpenAI -from openai.helpers import LocalAudioPlayer -import textwrap +from agents import Agent, Runner +from openai import AsyncOpenAI import tempfile import uuid -import numpy as np -from typing import Callable -from urllib.parse import urlparse -from dotenv import load_dotenv -import asyncio -import json from datetime import datetime import time import streamlit as st +from dotenv import load_dotenv +import asyncio load_dotenv() def init_session_state(): - """Initialize session state variables for storing API keys and configurations.""" defaults = { "initialized": False, "qdrant_url": "", @@ -39,7 +31,7 @@ def init_session_state(): "embedding_model": None, "processor_agent": None, "tts_agent": None, - "selected_voice": "coral" # Default voice + "selected_voice": "coral" } for key, value in defaults.items(): @@ -47,12 +39,10 @@ def init_session_state(): st.session_state[key] = value def sidebar_config(): - """Render and handle the configuration sidebar.""" with st.sidebar: st.title("🔑 Configuration") st.markdown("---") - # API Keys and URLs st.session_state.qdrant_url = st.text_input( "Qdrant URL", value=st.session_state.qdrant_url, @@ -81,7 +71,6 @@ def sidebar_config(): placeholder="https://docs.example.com" ) - # Voice selection st.markdown("---") st.markdown("### 🎤 Voice Settings") voices = ["alloy", "ash", "ballad", "coral", "echo", "fable", "onyx", "nova", "sage", "shimmer", "verse"] @@ -92,7 +81,6 @@ def sidebar_config(): help="Choose the voice for the audio response" ) - # Setup button if st.button("Initialize System", type="primary"): if all([ st.session_state.qdrant_url, @@ -104,7 +92,6 @@ def sidebar_config(): progress_placeholder = st.empty() with progress_placeholder.container(): try: - # Setup Qdrant st.markdown("🔄 Setting up Qdrant connection...") client, embedding_model = setup_qdrant_collection( st.session_state.qdrant_url, @@ -114,7 +101,6 @@ def sidebar_config(): st.session_state.embedding_model = embedding_model st.markdown("✅ Qdrant setup complete!") - # Crawl documentation st.markdown("🔄 Crawling documentation pages...") pages = crawl_documentation( st.session_state.firecrawl_api_key, @@ -122,7 +108,6 @@ def sidebar_config(): ) st.markdown(f"✅ Crawled {len(pages)} documentation pages!") - # Store embeddings store_embeddings( client, embedding_model, @@ -130,7 +115,6 @@ def sidebar_config(): "docs_embeddings" ) - # Setup agents processor_agent, tts_agent = setup_agents( st.session_state.openai_api_key ) @@ -146,163 +130,117 @@ def sidebar_config(): st.error("Please fill in all the required fields!") def setup_qdrant_collection(qdrant_url: str, qdrant_api_key: str, collection_name: str = "docs_embeddings"): - print("\n--- Step 1: Setting up Qdrant Collection ---") + client = QdrantClient(url=qdrant_url, api_key=qdrant_api_key) + embedding_model = TextEmbedding() + test_embedding = list(embedding_model.embed(["test"]))[0] + embedding_dim = len(test_embedding) + try: - client = QdrantClient(url=qdrant_url, api_key=qdrant_api_key) - print("✓ Connected to Qdrant") - - embedding_model = TextEmbedding() - test_embedding = list(embedding_model.embed(["test"]))[0] - embedding_dim = len(test_embedding) - print(f"✓ Embedding model ready (dimension: {embedding_dim})") - client.create_collection( collection_name=collection_name, vectors_config=VectorParams(size=embedding_dim, distance=Distance.COSINE) ) - print(f"✓ Created collection: {collection_name}") - - return client, embedding_model - except Exception as e: - if "already exists" in str(e): - print(f"✓ Collection {collection_name} already exists") - return client, embedding_model - raise e + if "already exists" not in str(e): + raise e + + return client, embedding_model def crawl_documentation(firecrawl_api_key: str, url: str, output_dir: Optional[str] = None): - print("\n--- Step 2: Crawling Documentation ---") - try: - firecrawl = FirecrawlApp(api_key=firecrawl_api_key) - print(f"✓ Initialized Firecrawl") - - if output_dir: - os.makedirs(output_dir, exist_ok=True) - print(f"✓ Created output directory: {output_dir}") - - print(f"Starting crawl of {url}...") - - pages = [] - - response = firecrawl.crawl_url( - url, - params={ - 'limit': 5, - 'scrapeOptions': { - 'formats': ['markdown', 'html'] - } - } - ) - - while True: - if response.get('status') == 'scraping': - print(f"Progress: {response.get('completed', 0)}/{response.get('total', 0)} pages") - print(f"Credits used: {response.get('creditsUsed', 0)}") - - for page in response.get('data', []): - content = page.get('markdown') or page.get('html', '') - metadata = page.get('metadata', {}) - source_url = metadata.get('sourceURL', '') - - if output_dir and content: - filename = f"{uuid.uuid4()}.md" - filepath = os.path.join(output_dir, filename) - with open(filepath, 'w', encoding='utf-8') as f: - f.write(content) - - pages.append({ - "content": content, - "url": source_url, - "metadata": { - "title": metadata.get('title', ''), - "description": metadata.get('description', ''), - "language": metadata.get('language', 'en'), - "crawl_date": datetime.now().isoformat() - } - }) - - print(f"✓ Processed page: {metadata.get('title', 'Untitled')}") - - next_url = response.get('next') - if not next_url: - break - - response = firecrawl.get(next_url) - time.sleep(1) - - print(f"✓ Crawled {len(pages)} pages") - return pages + firecrawl = FirecrawlApp(api_key=firecrawl_api_key) + pages = [] - except Exception as e: - print(f"Error crawling documentation: {str(e)}") - raise e + if output_dir: + os.makedirs(output_dir, exist_ok=True) + + response = firecrawl.crawl_url( + url, + params={ + 'limit': 5, + 'scrapeOptions': { + 'formats': ['markdown', 'html'] + } + } + ) + + while True: + for page in response.get('data', []): + content = page.get('markdown') or page.get('html', '') + metadata = page.get('metadata', {}) + source_url = metadata.get('sourceURL', '') + + if output_dir and content: + filename = f"{uuid.uuid4()}.md" + filepath = os.path.join(output_dir, filename) + with open(filepath, 'w', encoding='utf-8') as f: + f.write(content) + + pages.append({ + "content": content, + "url": source_url, + "metadata": { + "title": metadata.get('title', ''), + "description": metadata.get('description', ''), + "language": metadata.get('language', 'en'), + "crawl_date": datetime.now().isoformat() + } + }) + + next_url = response.get('next') + if not next_url: + break + + response = firecrawl.get(next_url) + time.sleep(1) + + return pages def store_embeddings(client: QdrantClient, embedding_model: TextEmbedding, pages: List[Dict], collection_name: str): - print("\n--- Step 3: Generating and Storing Embeddings ---") - try: - for page in pages: - embedding = list(embedding_model.embed([page["content"]]))[0] - - client.upsert( - collection_name=collection_name, - points=[ - models.PointStruct( - id=str(uuid.uuid4()), - vector=embedding.tolist(), - payload={ - "content": page["content"], - "url": page["url"], - **page["metadata"] - } - ) - ] - ) - print(f"✓ Stored embedding for: {page['metadata']['title'] or page['url']}") - - print(f"✓ Stored {len(pages)} embeddings in Qdrant") - - except Exception as e: - print(f"Error storing embeddings: {str(e)}") - raise e + for page in pages: + embedding = list(embedding_model.embed([page["content"]]))[0] + client.upsert( + collection_name=collection_name, + points=[ + models.PointStruct( + id=str(uuid.uuid4()), + vector=embedding.tolist(), + payload={ + "content": page["content"], + "url": page["url"], + **page["metadata"] + } + ) + ] + ) def setup_agents(openai_api_key: str): - print("\n--- Step 4: Setting up OpenAI Agents ---") - try: - # Set OpenAI API key in environment - os.environ["OPENAI_API_KEY"] = openai_api_key - print("✓ Set OpenAI API key in environment") - - processor_agent = Agent( - name="Documentation Processor", - instructions="""You are a helpful documentation assistant. Your task is to: - 1. Analyze the provided documentation content - 2. Answer the user's question clearly and concisely - 3. Include relevant examples when available - 4. Cite the source URLs when referencing specific content - 5. Keep responses natural and conversational - 6. Format your response in a way that's easy to speak out loud""", - model="gpt-4o" - ) - print("✓ Set up Documentation Processor Agent") - - tts_agent = Agent( - name="Text-to-Speech Agent", - instructions="""You are a text-to-speech agent. Your task is to: - 1. Convert the processed documentation response into natural speech - 2. Maintain proper pacing and emphasis - 3. Handle technical terms clearly - 4. Keep the tone professional but friendly - 5. Use appropriate pauses for better comprehension - 6. Ensure the speech is clear and well-articulated""", - model="gpt-4o-mini-tts" - ) - print("✓ Set up TTS Agent") - - return processor_agent, tts_agent + os.environ["OPENAI_API_KEY"] = openai_api_key - except Exception as e: - print(f"Error setting up agents: {str(e)}") - raise e + processor_agent = Agent( + name="Documentation Processor", + instructions="""You are a helpful documentation assistant. Your task is to: + 1. Analyze the provided documentation content + 2. Answer the user's question clearly and concisely + 3. Include relevant examples when available + 4. Cite the source URLs when referencing specific content + 5. Keep responses natural and conversational + 6. Format your response in a way that's easy to speak out loud""", + model="gpt-4o" + ) + + tts_agent = Agent( + name="Text-to-Speech Agent", + instructions="""You are a text-to-speech agent. Your task is to: + 1. Convert the processed documentation response into natural speech + 2. Maintain proper pacing and emphasis + 3. Handle technical terms clearly + 4. Keep the tone professional but friendly + 5. Use appropriate pauses for better comprehension + 6. Ensure the speech is clear and well-articulated""", + model="gpt-4o-mini-tts" + ) + + return processor_agent, tts_agent async def process_query( query: str, @@ -314,10 +252,7 @@ async def process_query( openai_api_key: str ): try: - # Generate query embedding query_embedding = list(embedding_model.embed([query]))[0] - - # Search in Qdrant search_response = client.query_points( collection_name=collection_name, query=query_embedding.tolist(), @@ -330,7 +265,6 @@ async def process_query( if not search_results: raise Exception("No relevant documents found in the vector database") - # Build context from search results context = "Based on the following documentation:\n\n" for result in search_results: payload = result.payload @@ -343,14 +277,12 @@ async def process_query( context += f"\nUser Question: {query}\n\n" context += "Please provide a clear, concise answer that can be easily spoken out loud." - # Process response with agents processor_result = await Runner.run(processor_agent, context) processor_response = processor_result.final_output tts_result = await Runner.run(tts_agent, processor_response) tts_response = tts_result.final_output - # Generate audio async_openai = AsyncOpenAI(api_key=openai_api_key) audio_response = await async_openai.audio.speech.create( model="gpt-4o-mini-tts", @@ -360,11 +292,9 @@ async def process_query( response_format="mp3" ) - # Save audio to a temporary file temp_dir = tempfile.gettempdir() audio_path = os.path.join(temp_dir, f"response_{uuid.uuid4()}.mp3") - # Write the audio content to the file with open(audio_path, "wb") as f: f.write(audio_response.content) @@ -382,7 +312,6 @@ async def process_query( } except Exception as e: - print(f"\nError processing query: {str(e)}") return { "status": "error", "error": str(e), @@ -390,7 +319,6 @@ async def process_query( } def run_streamlit(): - """Main Streamlit application.""" st.set_page_config( page_title="AI Voice Documentation Agent Team", page_icon="🎙️", @@ -400,7 +328,6 @@ def run_streamlit(): init_session_state() sidebar_config() - # Main content area st.title("🎙️ AI Voice Documentation Agent Team") st.markdown(""" Get OpenAI SDK voice-powered answers to your documentation questions! Simply: @@ -409,7 +336,6 @@ def run_streamlit(): 3. Ask your question below and get both text and voice responses """) - # Query input and processing query = st.text_input( "What would you like to know about the documentation?", placeholder="e.g., How do I authenticate API requests?", @@ -438,10 +364,8 @@ def run_streamlit(): if "audio_path" in result: st.markdown(f"### 🔊 Audio Response (Voice: {st.session_state.selected_voice})") - # Pass the file path directly to st.audio st.audio(result["audio_path"], format="audio/mp3", start_time=0) - # For download button, we still need to read the bytes with open(result["audio_path"], "rb") as audio_file: audio_bytes = audio_file.read() st.download_button(