Merge pull request #90 from Madhuvod/lead-generation-ai
Added new demo: AI lead generation Agent with Firecrawl
This commit is contained in:
Shubham Saboo
GitHub
commit
82eb4de6a9
3 changed files with 247 additions and 0 deletions
35
ai_agent_tutorials/ai_lead_generation_agent/README.md
Normal file
35
ai_agent_tutorials/ai_lead_generation_agent/README.md
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
## 🎯 AI Lead Generation Agent
|
||||
|
||||
The AI Lead Generation Agent is a firecrawl powered agent that automates the process of finding and qualifying potential leads from Quora. It leverages Firecrawl's search and LLM extraction capabilities to identify relevant user profiles, extract valuable information, and organize it into a structured format in Google Sheets. This agent helps sales and marketing teams efficiently build targeted lead lists while saving hours of manual research.
|
||||
|
||||
### Features
|
||||
- **Targeted Search**: Uses Firecrawl's search endpoint to find relevant Quora URLs based on your search criteria
|
||||
- **Intelligent Extraction**: Leverages Firecrawl's LLM extract functionality to pull user information from Quora profiles
|
||||
- **Automated Processing**: Formats extracted user information into a clean, structured format
|
||||
- **Google Sheets Integration**: Automatically creates and populates Google Sheets with lead information
|
||||
- **Customizable Criteria**: Allows you to define specific search parameters to find your ideal leads for your niche
|
||||
|
||||
### How to Get Started
|
||||
1. **Clone the repository**:
|
||||
```bash
|
||||
git clone https://github.com/Shubhamsaboo/awesome-llm-apps.git
|
||||
cd ai_agent_tutorials/ai_lead_generation_agent
|
||||
```
|
||||
3. **Install the required packages**:
|
||||
```bash
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
4. **Important thing to do in composio**:
|
||||
- in the terminal, run this command: `composio add googlesheets`
|
||||
- In your compposio dashboard, create a new google sheet intergation and make sure it is active in the active integrations/connections tab
|
||||
|
||||
5. **Set up your API keys**:
|
||||
- Get your Firecrawl API key from [Firecrawl's website](https://www.firecrawl.dev/app/api-keys)
|
||||
- Get your Composio API key from [Composio's website](https://composio.ai)
|
||||
- Get your OpenAI API key from [OpenAI's website](https://platform.openai.com/api-keys)
|
||||
|
||||
6. **Run the application**:
|
||||
```bash
|
||||
streamlit run ai_lead_generation_agent.py
|
||||
```
|
||||
|
||||
|
|
@ -0,0 +1,206 @@
|
|||
import streamlit as st
|
||||
import requests
|
||||
from phi.agent import Agent
|
||||
from phi.tools.firecrawl import FirecrawlTools
|
||||
from phi.model.openai import OpenAIChat
|
||||
from firecrawl import FirecrawlApp
|
||||
from pydantic import BaseModel, Field
|
||||
from typing import List
|
||||
from composio_phidata import Action, ComposioToolSet
|
||||
|
||||
# Define a schema for a single user interaction (question or answer)
|
||||
class QuoraUserInteractionSchema(BaseModel):
|
||||
username: str = Field(description="The username of the user who posted the question or answer")
|
||||
bio: str = Field(description="The bio or description of the user")
|
||||
post_type: str = Field(description="The type of post, either 'question' or 'answer'")
|
||||
timestamp: str = Field(description="When the question or answer was posted")
|
||||
upvotes: int = Field(default=0, description="Number of upvotes received")
|
||||
links: List[str] = Field(default_factory=list, description="Any links included in the post")
|
||||
|
||||
# Define a schema for the entire page, containing multiple interactions
|
||||
class QuoraPageSchema(BaseModel):
|
||||
interactions: List[QuoraUserInteractionSchema] = Field(description="List of all user interactions (questions and answers) on the page")
|
||||
|
||||
# Step 1: Search for relevant URLs using Firecrawl
|
||||
def search_for_urls(company_description, firecrawl_api_key, num_links):
|
||||
print("Step 1: Searching for relevant URLs using Firecrawl...")
|
||||
url = "https://api.firecrawl.dev/v1/search"
|
||||
headers = {
|
||||
"Authorization": f"Bearer {firecrawl_api_key}",
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
query1 = f"quora websites where people are looking for {company_description} services"
|
||||
payload = {
|
||||
"query": query1,
|
||||
"limit": num_links, # Use the num_links parameter here
|
||||
"lang": "en",
|
||||
"location": "United States",
|
||||
"timeout": 60000,
|
||||
}
|
||||
response = requests.post(url, json=payload, headers=headers)
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
if data.get("success"):
|
||||
results = data.get("data", [])
|
||||
print(f"Found {len(results)} relevant URLs.")
|
||||
return [result["url"] for result in results]
|
||||
else:
|
||||
print("Search request was not successful.")
|
||||
print(data.get("warning", "No warning provided."))
|
||||
else:
|
||||
print(f"Failed to retrieve data. Status code: {response.status_code}")
|
||||
return []
|
||||
|
||||
# Step 2: Extract user info from URLs using Firecrawl's LLM extract
|
||||
def extract_user_info_from_urls(urls, firecrawl_api_key):
|
||||
print("\nStep 2: Extracting user info from URLs using Firecrawl's LLM extract...")
|
||||
user_info_list = []
|
||||
firecrawl_app = FirecrawlApp(api_key=firecrawl_api_key)
|
||||
for website_url in urls:
|
||||
print(f"Extracting user info from: {website_url}")
|
||||
|
||||
# Use Firecrawl's LLM extract to get structured data
|
||||
data = firecrawl_app.scrape_url(website_url, {
|
||||
'formats': ['extract'],
|
||||
'extract': {
|
||||
'schema': QuoraPageSchema.model_json_schema(),
|
||||
}
|
||||
})
|
||||
|
||||
# Extract the interactions from the response
|
||||
extracted_data = data.get("extract", {})
|
||||
interactions = extracted_data.get("interactions", [])
|
||||
|
||||
# Store the results
|
||||
user_info_list.append({
|
||||
"website_url": website_url,
|
||||
"user_info": interactions
|
||||
})
|
||||
print(f"Extracted {len(interactions)} interactions from {website_url}.")
|
||||
return user_info_list
|
||||
|
||||
# Step 3: Format the extracted user info into a flattened JSON structure
|
||||
def format_user_info_to_flattened_json(user_info_list):
|
||||
print("\nStep 3: Formatting the extracted user info into a flattened JSON structure...")
|
||||
flattened_data = []
|
||||
|
||||
for info in user_info_list:
|
||||
website_url = info["website_url"]
|
||||
user_info = info["user_info"]
|
||||
|
||||
for interaction in user_info:
|
||||
flattened_interaction = {
|
||||
"Website URL": website_url,
|
||||
"Username": interaction.get("username", ""),
|
||||
"Bio": interaction.get("bio", ""),
|
||||
"Post Type": interaction.get("post_type", ""),
|
||||
"Timestamp": interaction.get("timestamp", ""),
|
||||
"Upvotes": interaction.get("upvotes", 0),
|
||||
"Links": ", ".join(interaction.get("links", [])), # Convert list of links to a single string
|
||||
}
|
||||
flattened_data.append(flattened_interaction)
|
||||
|
||||
print(f"Formatted {len(flattened_data)} interactions into flattened JSON.")
|
||||
return flattened_data
|
||||
|
||||
# Step 4: Create a new Phidata agent to interact with Google Sheets
|
||||
def create_google_sheets_agent(composio_api_key, openai_api_key):
|
||||
print("\nStep 4: Creating a new Phidata agent to interact with Google Sheets...")
|
||||
# Initialize Composio Toolset
|
||||
composio_toolset = ComposioToolSet(api_key=composio_api_key)
|
||||
|
||||
# Get the Google Sheets tool
|
||||
google_sheets_tool = composio_toolset.get_tools(actions=[Action.GOOGLESHEETS_SHEET_FROM_JSON])[0]
|
||||
|
||||
# Create the agent
|
||||
google_sheets_agent = Agent(
|
||||
model=OpenAIChat(id="gpt-4o-mini", api_key=openai_api_key),
|
||||
tools=[google_sheets_tool],
|
||||
show_tool_calls=True, # Enable verbose tool call output for debugging
|
||||
system_prompt="You are an expert at creating and updating Google Sheets. You will be given user information in JSON format, and you need to write it into a new Google Sheet.",
|
||||
markdown=True
|
||||
)
|
||||
print("Google Sheets agent created successfully.")
|
||||
return google_sheets_agent
|
||||
|
||||
# Step 5: Write formatted user info to Google Sheets
|
||||
def write_to_google_sheets(flattened_data, composio_api_key, openai_api_key):
|
||||
print("\nStep 5: Writing formatted user info to Google Sheets...")
|
||||
# Create the Google Sheets agent
|
||||
google_sheets_agent = create_google_sheets_agent(composio_api_key, openai_api_key)
|
||||
|
||||
# Create a new Google Sheet from the flattened JSON data
|
||||
print("Creating a new Google Sheet with the flattened JSON data...")
|
||||
create_sheet_response = google_sheets_agent.run(
|
||||
f"Create a new Google Sheet with the following data:\n"
|
||||
f"Title: Quora User Info\n"
|
||||
f"Sheet Name: Sheet1\n"
|
||||
f"Sheet JSON: {flattened_data}"
|
||||
)
|
||||
print("Create Sheet Response:", create_sheet_response.content)
|
||||
|
||||
# Extract the Google Sheets link from the response
|
||||
if "https://docs.google.com/spreadsheets/d/" in create_sheet_response.content:
|
||||
google_sheets_link = create_sheet_response.content.split("https://docs.google.com/spreadsheets/d/")[1].split(" ")[0]
|
||||
google_sheets_link = f"https://docs.google.com/spreadsheets/d/{google_sheets_link}"
|
||||
return google_sheets_link
|
||||
return None
|
||||
|
||||
# Streamlit UI
|
||||
def main():
|
||||
st.title("AI Lead Generation Agent")
|
||||
st.info("This firecrawl powered agent helps you generate leads from Quora by searching for relevant posts and extracting user information.")
|
||||
|
||||
# Sidebar for API keys and number of links
|
||||
with st.sidebar:
|
||||
st.header("API Keys")
|
||||
firecrawl_api_key = st.text_input("Firecrawl API Key", type="password")
|
||||
st.caption(" Get your Firecrawl API key from [Firecrawl's website](https://www.firecrawl.dev/app/api-keys)")
|
||||
openai_api_key = st.text_input("OpenAI API Key", type="password")
|
||||
st.caption(" Get your Composio API key from [Composio's website](https://composio.ai)")
|
||||
composio_api_key = st.text_input("Composio API Key", type="password")
|
||||
st.caption(" Get your OpenAI API key from [OpenAI's website](https://platform.openai.com/api-keys)")
|
||||
|
||||
# Add a numeric input for the number of links
|
||||
num_links = st.number_input("Number of links to search", min_value=1, max_value=10, value=3)
|
||||
|
||||
# Reset button
|
||||
if st.button("Reset"):
|
||||
st.session_state.clear()
|
||||
st.experimental_rerun()
|
||||
|
||||
# Main input for company description
|
||||
company_description = st.text_input("Enter your company description or the niche you want to find leads in:", placeholder="e.g. AI voice cloning, Video Generation AI tools")
|
||||
|
||||
if st.button("Generate Leads"):
|
||||
if not all([firecrawl_api_key, openai_api_key, composio_api_key, company_description]):
|
||||
st.error("Please fill in all the API keys and the company description.")
|
||||
else:
|
||||
with st.spinner("Searching for relevant URLs..."):
|
||||
urls = search_for_urls(company_description, firecrawl_api_key, num_links) # Pass num_links to search_for_urls
|
||||
|
||||
if urls:
|
||||
st.subheader("Quora Links Used:")
|
||||
for url in urls:
|
||||
st.write(url)
|
||||
|
||||
with st.spinner("Extracting user info from URLs..."):
|
||||
user_info_list = extract_user_info_from_urls(urls, firecrawl_api_key)
|
||||
|
||||
with st.spinner("Formatting user info..."):
|
||||
flattened_data = format_user_info_to_flattened_json(user_info_list)
|
||||
|
||||
with st.spinner("Writing to Google Sheets..."):
|
||||
google_sheets_link = write_to_google_sheets(flattened_data, composio_api_key, openai_api_key)
|
||||
|
||||
if google_sheets_link:
|
||||
st.success("Lead generation and data writing to Google Sheets completed successfully!")
|
||||
st.subheader("Google Sheets Link:")
|
||||
st.markdown(f"[View Google Sheet]({google_sheets_link})")
|
||||
else:
|
||||
st.error("Failed to retrieve the Google Sheets link.")
|
||||
else:
|
||||
st.warning("No relevant URLs found.")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
firecrawl-py==1.9.0
|
||||
phidata==2.7.3
|
||||
composio-phidata==0.6.15
|
||||
composio==0.1.1
|
||||
pydantic==2.10.5
|
||||
streamlit
|
||||
Loading…
Reference in a new issue