From 98c5222a6f0cab60fdc318b9916cc69d28b1614a Mon Sep 17 00:00:00 2001 From: Madhu Date: Sat, 11 Jan 2025 17:44:54 +0530 Subject: [PATCH 1/8] New Project: AI Competitor Intelligence Agent --- .../ai_competitors_analysis_team/README.md | 0 .../ai_competitors_analysis_team/exaai.py | 32 +++++++++++++++++++ .../ai_competitors_analysis_team/firecrawl.py | 0 .../ai_competitors_analysis_team/main.py | 0 .../requirements.txt | 1 + 5 files changed, 33 insertions(+) create mode 100644 ai_agent_tutorials/ai_competitors_analysis_team/README.md create mode 100644 ai_agent_tutorials/ai_competitors_analysis_team/exaai.py create mode 100644 ai_agent_tutorials/ai_competitors_analysis_team/firecrawl.py create mode 100644 ai_agent_tutorials/ai_competitors_analysis_team/main.py create mode 100644 ai_agent_tutorials/ai_competitors_analysis_team/requirements.txt diff --git a/ai_agent_tutorials/ai_competitors_analysis_team/README.md b/ai_agent_tutorials/ai_competitors_analysis_team/README.md new file mode 100644 index 0000000..e69de29 diff --git a/ai_agent_tutorials/ai_competitors_analysis_team/exaai.py b/ai_agent_tutorials/ai_competitors_analysis_team/exaai.py new file mode 100644 index 0000000..99e2cf3 --- /dev/null +++ b/ai_agent_tutorials/ai_competitors_analysis_team/exaai.py @@ -0,0 +1,32 @@ +from exa_py import Exa + +# Initialize Exa with your API key +exa = Exa(api_key="EXA_API_KEY") + +def get_competitor_urls(url=None, description=None): + if url: + # Use the URL-based API call + result = exa.find_similar( + url=url, + num_results=5, + exclude_source_domain=True, + category="company" + ) + elif description: + # Use the description-based API call + result = exa.search( + description, + type="neural", + category="company", + use_autoprompt=True, + num_results=5 + ) + else: + raise ValueError("Please provide either a URL or a description.") + + # Extracting and return only the competitor URLs + competitor_urls = [item.url for item in result.results] + return competitor_urls + # return result + +print(get_competitor_urls(description="Competitors of a company that works on AI Neuro Health Tech")) \ No newline at end of file diff --git a/ai_agent_tutorials/ai_competitors_analysis_team/firecrawl.py b/ai_agent_tutorials/ai_competitors_analysis_team/firecrawl.py new file mode 100644 index 0000000..e69de29 diff --git a/ai_agent_tutorials/ai_competitors_analysis_team/main.py b/ai_agent_tutorials/ai_competitors_analysis_team/main.py new file mode 100644 index 0000000..e69de29 diff --git a/ai_agent_tutorials/ai_competitors_analysis_team/requirements.txt b/ai_agent_tutorials/ai_competitors_analysis_team/requirements.txt new file mode 100644 index 0000000..367c38c --- /dev/null +++ b/ai_agent_tutorials/ai_competitors_analysis_team/requirements.txt @@ -0,0 +1 @@ +exa-py==1.0.6 \ No newline at end of file From 20c49d8f728d17e84903c7260ad5b62fb19a26a3 Mon Sep 17 00:00:00 2001 From: Madhu Date: Sat, 11 Jan 2025 21:57:13 +0530 Subject: [PATCH 2/8] firecrawl agent implementation --- .../__pycache__/firecrawl.cpython-312.pyc | Bin 0 -> 1618 bytes .../ai_competitors_analysis_team/exaai.py | 2 - .../ai_competitors_analysis_team/fire.py | 41 ++++++++++++++++++ .../ai_competitors_analysis_team/firecrawl.py | 0 .../ai_competitors_analysis_team/main.py | 37 ++++++++++++++++ .../requirements.txt | 3 +- 6 files changed, 80 insertions(+), 3 deletions(-) create mode 100644 ai_agent_tutorials/ai_competitors_analysis_team/__pycache__/firecrawl.cpython-312.pyc create mode 100644 ai_agent_tutorials/ai_competitors_analysis_team/fire.py delete mode 100644 ai_agent_tutorials/ai_competitors_analysis_team/firecrawl.py diff --git a/ai_agent_tutorials/ai_competitors_analysis_team/__pycache__/firecrawl.cpython-312.pyc b/ai_agent_tutorials/ai_competitors_analysis_team/__pycache__/firecrawl.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ccd244c2c6bda1cacd9ba1222f407f6c6d3d42b1 GIT binary patch literal 1618 zcmah}&1)M+6ra^ zV&Q;jdhnr_K*52KdrERBJ@#MdMQKu^EEo#yB{v1jk(M$vDkFKo3)g+{Ay87LM4K97Q(!#Pk*Kn)n8y{d7FS!>eahKLFxR`vnPL|2?5?&;Wi}?K28?Ovy zG#!uOh&dZ{GtLOG7}3~B49AAYd?vg3W+baiVbx~=13xK7K*#VyLcQ3K%6nWEt+3-L z+HtW5jI59O1_?VsTyjG{qLRswJ8kZ{2hb$|D*rm{z3<@IK{8>g}2_pd9r zCC9F`WQ$sj(DT9$3z`-S8X@?jmW#RBw*E?8%w>F$Xh0`zcb5HYg z{q)=(*hX>IaeT}I$LY?rq>RKx+wOEaH6E_B0J9ooVs{HGM?-eeTAeS&xv0fzN#0zI zQe08@WL9fO337#XJg#1nVXlSn1Hp`|sf^14mlq< Date: Sun, 12 Jan 2025 00:20:25 +0530 Subject: [PATCH 3/8] main code --- .../ai_competitors_analysis_team/main.py | 136 ++++++++++++++---- 1 file changed, 106 insertions(+), 30 deletions(-) diff --git a/ai_agent_tutorials/ai_competitors_analysis_team/main.py b/ai_agent_tutorials/ai_competitors_analysis_team/main.py index 78b9b57..5393f3f 100644 --- a/ai_agent_tutorials/ai_competitors_analysis_team/main.py +++ b/ai_agent_tutorials/ai_competitors_analysis_team/main.py @@ -1,37 +1,113 @@ -from firecrawl import FirecrawlApp -from pydantic import BaseModel, Field -from typing import Optional +from exa_py import Exa +from phi.agent import Agent +from phi.tools.firecrawl import FirecrawlTools +from phi.model.openai import OpenAIChat -# Define a simple schema for testing -class SimpleSchema(BaseModel): - title: Optional[str] = Field(description="Title of the webpage.") - description: Optional[str] = Field(description="Meta description of the webpage.") +exa = Exa(api_key="50a85") -# Initialize the FirecrawlApp with your API key -app = FirecrawlApp(api_key='fc-') # Replace with your API key +firecrawl_tools = FirecrawlTools( + api_key="f2", + scrape=False, + crawl=True, + limit=5 +) + +firecrawl_agent = Agent( + model=OpenAIChat(id="gpt-4o-mini", api_key="s"), + tools=[firecrawl_tools, ], + show_tool_calls=True, + markdown=True +) + +analysis_agent = Agent( + model=OpenAIChat(id="gpt-4o-mini", api_key="s"), + show_tool_calls=True, + markdown=True +) + +def get_competitor_urls(url=None, description=None): + + if url: + result = exa.find_similar( + url=url, + excludeDomains=[url], + num_results=3, + exclude_source_domain=True, + category="company" + ) + elif description: + result = exa.search( + description, + type="neural", + category="company", + use_autoprompt=True, + num_results=3 + ) + else: + raise ValueError("Please provide either a URL or a description.") + + competitor_urls = [item.url for item in result.results] + return competitor_urls def extract_competitor_info(competitor_url: str): - try: - # Use Firecrawl to scrape and extract data - data = app.scrape_url(competitor_url, { - 'formats': ['extract'], - 'extract': { - 'schema': SimpleSchema.model_json_schema(), - } - }) - return data.get("extract", {}) - except Exception as e: - return {"error": str(e)} + crawl_response = firecrawl_agent.run(f"Crawl and summarize {competitor_url}") + crawled_data = crawl_response.content + + structured_info = firecrawl_agent.run( + f"""Extract the following information from the crawled data: + - Product pricing and features: Extract exact pricing numbers from their pricing page. + - Technology stack information + - Marketing messaging/positioning + - Customer testimonials/case studies + - Latest news and developements + + Crawled Data: + {crawled_data} + """ + ) + + return { + "competitor": competitor_url, + "data": structured_info.content + } + except Exception as e: + return { + "competitor": competitor_url, + "error": str(e) + } + +def generate_analysis_report(competitor_data: list): + combined_data = "\n\n".join([str(data) for data in competitor_data]) + + report = analysis_agent.run( + f"""Analyze the following competitor data and generate a detailed report: + {combined_data} + + Tasks: + 1. Compare pricing and identify opportunities for competitive pricing. + 2. Analyze features and highlight unique or missing features. + 3. Evaluate marketing messaging and suggest positioning strategies. + 4. Summarize customer testimonials and case studies. + 5. Provide actionable insights for market positioning. + """ + ) + return report.response + +def main(): + competitor_urls = get_competitor_urls(url="https://jenni.ai") + print(f"Competitor URLs: {competitor_urls}") + + competitor_data = [] + for url in competitor_urls: + print(f"\nAnalyzing Competitor: {url}") + competitor_info = extract_competitor_info(url) + competitor_data.append(competitor_info) + + analysis_report = generate_analysis_report(competitor_data) + + print("\nCompetitor Analysis Report:") + print(analysis_report) -# Example usage if __name__ == "__main__": - # Competitor URL to analyze - competitor_url = "https://www.equal.in" # Replace with your competitor URL - - # Extract competitor information - competitor_info = extract_competitor_info(competitor_url) - - # Print the structured information - print(f"Competitor: {competitor_url}") - print(competitor_info) \ No newline at end of file + main() \ No newline at end of file From 0aa4db198c6961a86c9b5b24a6262d47a5fd5a6f Mon Sep 17 00:00:00 2001 From: Madhu Date: Sun, 12 Jan 2025 00:46:38 +0530 Subject: [PATCH 4/8] New additions --- .../__pycache__/firecrawl.cpython-312.pyc | Bin 1618 -> 0 bytes .../ai_competitors_analysis_team/exaai.py | 30 -------------- .../ai_competitors_analysis_team/main.py | 38 ++++++++++-------- .../requirements.txt | 5 ++- 4 files changed, 25 insertions(+), 48 deletions(-) delete mode 100644 ai_agent_tutorials/ai_competitors_analysis_team/__pycache__/firecrawl.cpython-312.pyc delete mode 100644 ai_agent_tutorials/ai_competitors_analysis_team/exaai.py diff --git a/ai_agent_tutorials/ai_competitors_analysis_team/__pycache__/firecrawl.cpython-312.pyc b/ai_agent_tutorials/ai_competitors_analysis_team/__pycache__/firecrawl.cpython-312.pyc deleted file mode 100644 index ccd244c2c6bda1cacd9ba1222f407f6c6d3d42b1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1618 zcmah}&1)M+6ra^ zV&Q;jdhnr_K*52KdrERBJ@#MdMQKu^EEo#yB{v1jk(M$vDkFKo3)g+{Ay87LM4K97Q(!#Pk*Kn)n8y{d7FS!>eahKLFxR`vnPL|2?5?&;Wi}?K28?Ovy zG#!uOh&dZ{GtLOG7}3~B49AAYd?vg3W+baiVbx~=13xK7K*#VyLcQ3K%6nWEt+3-L z+HtW5jI59O1_?VsTyjG{qLRswJ8kZ{2hb$|D*rm{z3<@IK{8>g}2_pd9r zCC9F`WQ$sj(DT9$3z`-S8X@?jmW#RBw*E?8%w>F$Xh0`zcb5HYg z{q)=(*hX>IaeT}I$LY?rq>RKx+wOEaH6E_B0J9ooVs{HGM?-eeTAeS&xv0fzN#0zI zQe08@WL9fO337#XJg#1nVXlSn1Hp`|sf^14mlq< Date: Sun, 12 Jan 2025 01:17:44 +0530 Subject: [PATCH 5/8] clean code - few features to ad --- .../ai_competitor_analyser.py | 154 ++++++++++++++++++ .../ai_competitors_analysis_team/fire.py | 41 ----- .../ai_competitors_analysis_team/main.py | 117 ------------- 3 files changed, 154 insertions(+), 158 deletions(-) create mode 100644 ai_agent_tutorials/ai_competitors_analysis_team/ai_competitor_analyser.py delete mode 100644 ai_agent_tutorials/ai_competitors_analysis_team/fire.py delete mode 100644 ai_agent_tutorials/ai_competitors_analysis_team/main.py diff --git a/ai_agent_tutorials/ai_competitors_analysis_team/ai_competitor_analyser.py b/ai_agent_tutorials/ai_competitors_analysis_team/ai_competitor_analyser.py new file mode 100644 index 0000000..0a43f05 --- /dev/null +++ b/ai_agent_tutorials/ai_competitors_analysis_team/ai_competitor_analyser.py @@ -0,0 +1,154 @@ +import streamlit as st +from exa_py import Exa +from phi.agent import Agent +from phi.tools.firecrawl import FirecrawlTools +from phi.model.openai import OpenAIChat +from phi.tools.duckduckgo import DuckDuckGo + +# Streamlit UI +st.set_page_config(page_title="AI Competitor Intelligence Agent", layout="wide") + +# Sidebar for API keys +st.sidebar.title("API Keys") +openai_api_key = st.sidebar.text_input("OpenAI API Key", type="password") +firecrawl_api_key = st.sidebar.text_input("Firecrawl API Key", type="password") +exa_api_key = st.sidebar.text_input("Exa API Key", type="password") + +# Store API keys in session state +if openai_api_key and firecrawl_api_key and exa_api_key: + st.session_state.openai_api_key = openai_api_key + st.session_state.firecrawl_api_key = firecrawl_api_key + st.session_state.exa_api_key = exa_api_key +else: + st.sidebar.warning("Please enter all API keys to proceed.") + +# Main UI +st.title("AI Competitor Intelligence Agent") +st.info( + """ + This app helps businesses analyze their competitors by extracting structured data from competitor websites and generating insights using AI. + - Provide a **URL** or a **description** of your company. + - The app will fetch competitor URLs, extract relevant information, and generate a detailed analysis report. + """ +) + +# Input fields for URL and description +url = st.text_input("Enter your company URL (optional):") +description = st.text_area("Enter a description of your company (if URL is not available):") + +# Initialize API keys and tools +if "openai_api_key" in st.session_state and "firecrawl_api_key" in st.session_state and "exa_api_key" in st.session_state: + exa = Exa(api_key=st.session_state.exa_api_key) + + firecrawl_tools = FirecrawlTools( + api_key=st.session_state.firecrawl_api_key, + scrape=False, + crawl=True, + limit=5 + ) + + firecrawl_agent = Agent( + model=OpenAIChat(id="gpt-4", api_key=st.session_state.openai_api_key), + tools=[firecrawl_tools, DuckDuckGo()], + show_tool_calls=True, + markdown=True + ) + + analysis_agent = Agent( + model=OpenAIChat(id="gpt-4", api_key=st.session_state.openai_api_key), + show_tool_calls=True, + markdown=True + ) + + def get_competitor_urls(url=None, description=None): + if url: + result = exa.find_similar( + url=url, + num_results=2, + exclude_source_domain=True, + category="company" + ) + elif description: + result = exa.search( + description, + type="neural", + category="company", + use_autoprompt=True, + num_results=2 + ) + else: + raise ValueError("Please provide either a URL or a description.") + + competitor_urls = [item.url for item in result.results] + return competitor_urls + + def extract_competitor_info(competitor_url: str): + try: + crawl_response = firecrawl_agent.run(f"Crawl and summarize {competitor_url}") + crawled_data = crawl_response.content + + structured_info = firecrawl_agent.run( + f"""Extract the following information from the crawled data: + - Product pricing and features: Extract exact pricing numbers from their pricing page. + - Technology stack information + - Marketing messaging/positioning + - Customer testimonials/case studies + - Latest news and developments (use DuckDuckGo to search for the latest news and developments) + + Crawled Data: + {crawled_data} + """ + ) + + return { + "competitor": competitor_url, + "data": structured_info.content + } + except Exception as e: + return { + "competitor": competitor_url, + "error": str(e) + } + + def generate_analysis_report(competitor_data: list): + combined_data = "\n\n".join([str(data) for data in competitor_data]) + + report = analysis_agent.run( + f"""Analyze the following competitor data and identify market opportunities to improve my own company: + {combined_data} + + Tasks: + 1. Identify market gaps and opportunities based on competitor offerings + 2. Analyze competitor weaknesses that we can capitalize on + 3. Recommend unique features or capabilities we should develop + 4. Suggest pricing and positioning strategies to gain competitive advantage + 5. Outline specific growth opportunities in underserved market segments + 6. Provide actionable recommendations for product development and go-to-market strategy + + Focus on finding opportunities where we can differentiate and do better than competitors. + Highlight any unmet customer needs or pain points we can address. + """ + ) + return report.content + + # Run analysis when the user clicks the button + if st.button("Analyze Competitors"): + if url or description: + with st.spinner("Fetching competitor URLs..."): + competitor_urls = get_competitor_urls(url=url, description=description) + st.write(f"Competitor URLs: {competitor_urls}") + + competitor_data = [] + for url in competitor_urls: + with st.spinner(f"Analyzing Competitor: {url}..."): + competitor_info = extract_competitor_info(url) + competitor_data.append(competitor_info) + + with st.spinner("Generating analysis report..."): + analysis_report = generate_analysis_report(competitor_data) + + st.success("Analysis complete!") + st.subheader("Competitor Analysis Report") + st.markdown(analysis_report) + else: + st.error("Please provide either a URL or a description.") \ No newline at end of file diff --git a/ai_agent_tutorials/ai_competitors_analysis_team/fire.py b/ai_agent_tutorials/ai_competitors_analysis_team/fire.py deleted file mode 100644 index 01c4432..0000000 --- a/ai_agent_tutorials/ai_competitors_analysis_team/fire.py +++ /dev/null @@ -1,41 +0,0 @@ -from phi.agent import Agent -from phi.tools.firecrawl import FirecrawlTools -from phi.model.openai import OpenAIChat - -firecrawl_tools = FirecrawlTools( - api_key="", - scrape=False, - crawl=True, - limit=5 -) - -agent = Agent( - model=OpenAIChat(id="gpt-4o-mini", api_key=""), - tools=[firecrawl_tools], - show_tool_calls=True, - markdown=True -) - -def extract_competitor_info(competitor_url: str): - crawl_response = agent.run(f"Crawl and summarize {competitor_url}") - crawled_data = crawl_response.content - - structured_info = agent.run( - f"""Extract the following information from the crawled data: - - Product pricing and features - exact pricing numbers from their pricing page - - Technology stack information - - Marketing messaging/positioning - - Customer testimonials/case studies - - Crawled Data: - {crawled_data} - """ - ) - - return structured_info.content - -if __name__ == "__main__": - competitor_url = "https://www.equal.in" - competitor_info = extract_competitor_info(competitor_url) - print(f"Competitor: {competitor_url}") - print(competitor_info) \ No newline at end of file diff --git a/ai_agent_tutorials/ai_competitors_analysis_team/main.py b/ai_agent_tutorials/ai_competitors_analysis_team/main.py deleted file mode 100644 index bf4d15f..0000000 --- a/ai_agent_tutorials/ai_competitors_analysis_team/main.py +++ /dev/null @@ -1,117 +0,0 @@ -from exa_py import Exa -from phi.agent import Agent -from phi.tools.firecrawl import FirecrawlTools -from phi.model.openai import OpenAIChat -from phi.tools.duckduckgo import DuckDuckGo - -exa = Exa(api_key="") - -firecrawl_tools = FirecrawlTools( - api_key="fc-", - scrape=False, - crawl=True, - limit=5 -) - -firecrawl_agent = Agent( - model=OpenAIChat(id="gpt-4o-mini", api_key="sk-proj-"), - tools=[firecrawl_tools, DuckDuckGo() ], - show_tool_calls=True, - markdown=True -) - -analysis_agent = Agent( - model=OpenAIChat(id="gpt-4o-mini", api_key="sk-proj-"), - show_tool_calls=True, - markdown=True -) - -def get_competitor_urls(url=None, description=None): - - if url: - result = exa.find_similar( - url=url, - num_results=2, - exclude_source_domain=True, - category="company" - ) - elif description: - result = exa.search( - description, - type="neural", - category="company", - use_autoprompt=True, - num_results=2 - ) - else: - raise ValueError("Please provide either a URL or a description.") - - competitor_urls = [item.url for item in result.results] - return competitor_urls - -def extract_competitor_info(competitor_url: str): - try: - crawl_response = firecrawl_agent.run(f"Crawl and summarize {competitor_url}") - crawled_data = crawl_response.content - - structured_info = firecrawl_agent.run( - f"""Extract the following information from the crawled data: - - Product pricing and features: Extract exact pricing numbers from their pricing page. - - Technology stack information - - Marketing messaging/positioning - - Customer testimonials/case studies - - Latest news and developements (use DuckDuckGo to search for the latest news and developements) - - Crawled Data: - {crawled_data} - """ - ) - - return { - "competitor": competitor_url, - "data": structured_info.content - } - except Exception as e: - return { - "competitor": competitor_url, - "error": str(e) - } - -def generate_analysis_report(competitor_data: list): - combined_data = "\n\n".join([str(data) for data in competitor_data]) - - report = analysis_agent.run( - f"""Analyze the following competitor data and identify market opportunities to improve my own company: - {combined_data} - - Tasks: - 1. Identify market gaps and opportunities based on competitor offerings - 2. Analyze competitor weaknesses that we can capitalize on - 3. Recommend unique features or capabilities we should develop - 4. Suggest pricing and positioning strategies to gain competitive advantage - 5. Outline specific growth opportunities in underserved market segments - 6. Provide actionable recommendations for product development and go-to-market strategy - - Focus on finding opportunities where we can differentiate and do better than competitors. - Highlight any unmet customer needs or pain points we can address. - """ - ) - return report.content - -def main(): - competitor_urls = get_competitor_urls(url="https://www.harvey.ai") - print(f"Competitor URLs: {competitor_urls}") - - competitor_data = [] - for url in competitor_urls: - print(f"\nAnalyzing Competitor: {url}") - competitor_info = extract_competitor_info(url) - competitor_data.append(competitor_info) - - analysis_report = generate_analysis_report(competitor_data) - - print("\nCompetitor Analysis Report:") - print(analysis_report) - -if __name__ == "__main__": - main() \ No newline at end of file From 9c6540efd15df3d74389e27ca761e38a8f52accb Mon Sep 17 00:00:00 2001 From: Madhu Date: Sun, 12 Jan 2025 02:49:42 +0530 Subject: [PATCH 6/8] clean code - few features to addd --- .../ai_competitors_analysis_team/ai_competitor_analyser.py | 4 ++-- .../ai_competitors_analysis_team/requirements.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ai_agent_tutorials/ai_competitors_analysis_team/ai_competitor_analyser.py b/ai_agent_tutorials/ai_competitors_analysis_team/ai_competitor_analyser.py index 0a43f05..af15e15 100644 --- a/ai_agent_tutorials/ai_competitors_analysis_team/ai_competitor_analyser.py +++ b/ai_agent_tutorials/ai_competitors_analysis_team/ai_competitor_analyser.py @@ -64,7 +64,7 @@ if "openai_api_key" in st.session_state and "firecrawl_api_key" in st.session_st if url: result = exa.find_similar( url=url, - num_results=2, + num_results=3, exclude_source_domain=True, category="company" ) @@ -74,7 +74,7 @@ if "openai_api_key" in st.session_state and "firecrawl_api_key" in st.session_st type="neural", category="company", use_autoprompt=True, - num_results=2 + num_results=3 ) else: raise ValueError("Please provide either a URL or a description.") diff --git a/ai_agent_tutorials/ai_competitors_analysis_team/requirements.txt b/ai_agent_tutorials/ai_competitors_analysis_team/requirements.txt index 42c07f1..83050a0 100644 --- a/ai_agent_tutorials/ai_competitors_analysis_team/requirements.txt +++ b/ai_agent_tutorials/ai_competitors_analysis_team/requirements.txt @@ -1,5 +1,5 @@ exa-py==1.0.6 firecrawl-py==1.9.0 duckduckgo-search==7.2.1 -phidata +phidata==2.7.3 streamlit \ No newline at end of file From 2243b6d54cc1e8ccda8381734ff1553532b0c544 Mon Sep 17 00:00:00 2001 From: Madhu Date: Sun, 12 Jan 2025 22:35:52 +0530 Subject: [PATCH 7/8] addition of comparison tables --- .../ai_competitor_analyser.py | 119 +++++++++++++++--- 1 file changed, 101 insertions(+), 18 deletions(-) diff --git a/ai_agent_tutorials/ai_competitors_analysis_team/ai_competitor_analyser.py b/ai_agent_tutorials/ai_competitors_analysis_team/ai_competitor_analyser.py index af15e15..c19cb7e 100644 --- a/ai_agent_tutorials/ai_competitors_analysis_team/ai_competitor_analyser.py +++ b/ai_agent_tutorials/ai_competitors_analysis_team/ai_competitor_analyser.py @@ -4,6 +4,7 @@ from phi.agent import Agent from phi.tools.firecrawl import FirecrawlTools from phi.model.openai import OpenAIChat from phi.tools.duckduckgo import DuckDuckGo +import pandas as pd # Streamlit UI st.set_page_config(page_title="AI Competitor Intelligence Agent", layout="wide") @@ -48,14 +49,21 @@ if "openai_api_key" in st.session_state and "firecrawl_api_key" in st.session_st ) firecrawl_agent = Agent( - model=OpenAIChat(id="gpt-4", api_key=st.session_state.openai_api_key), + model=OpenAIChat(id="gpt-4o-mini", api_key=st.session_state.openai_api_key), tools=[firecrawl_tools, DuckDuckGo()], show_tool_calls=True, markdown=True ) analysis_agent = Agent( - model=OpenAIChat(id="gpt-4", api_key=st.session_state.openai_api_key), + model=OpenAIChat(id="gpt-4o-mini", api_key=st.session_state.openai_api_key), + show_tool_calls=True, + markdown=True + ) + + # New agent for comparing competitor data + comparison_agent = Agent( + model=OpenAIChat(id="gpt-4o-mini", api_key=st.session_state.openai_api_key), show_tool_calls=True, markdown=True ) @@ -87,29 +95,99 @@ if "openai_api_key" in st.session_state and "firecrawl_api_key" in st.session_st crawl_response = firecrawl_agent.run(f"Crawl and summarize {competitor_url}") crawled_data = crawl_response.content - structured_info = firecrawl_agent.run( - f"""Extract the following information from the crawled data: - - Product pricing and features: Extract exact pricing numbers from their pricing page. - - Technology stack information - - Marketing messaging/positioning - - Customer testimonials/case studies - - Latest news and developments (use DuckDuckGo to search for the latest news and developments) - - Crawled Data: - {crawled_data} - """ - ) - return { "competitor": competitor_url, - "data": structured_info.content + "data": crawled_data } except Exception as e: + st.error(f"Error extracting info for {competitor_url}: {e}") return { "competitor": competitor_url, "error": str(e) } + def generate_comparison_report(competitor_data: list) -> None: + """ + Generate and display a comparison report of competitor data. + + Args: + competitor_data: List of dictionaries containing competitor information + """ + # Combine all competitor data into a single string + combined_data = "\n\n".join([str(data) for data in competitor_data]) + + # Updated system prompt for more structured output + system_prompt = """ + As an expert business analyst, analyze the competitor data and create a structured comparison table. + + Format the data in EXACTLY this markdown table structure: + | Company | Pricing | Key Features | Tech Stack | Marketing Focus | Customer Feedback | + |---------|---------|--------------|------------|-----------------|-------------------| + | [Company Name 1] | ... | ... | ... | ... | ... | + | [Company Name 2] | ... | ... | ... | ... | ... | + | [Company Name 3] | ... | ... | ... | ... | ... | + + Rules: + 1. Always include all columns + 2. Use the exact column names specified above + 3. Keep entries concise but informative + 4. Use pipe symbols (|) to separate columns + 5. Include the separator row (|---|) after headers + + Competitor Data: + {combined_data} + """ + + # Get comparison table from agent + comparison_response = comparison_agent.run( + system_prompt.format(combined_data=combined_data) + ) + + # Display the raw markdown table first + # st.subheader("Competitor Comparison (Markdown)") + # st.markdown(comparison_response.content) + + try: + # Split the markdown table into lines and clean them + table_lines = [ + line.strip() + for line in comparison_response.content.split('\n') + if line.strip() and '|' in line + ] + + # Extract headers (first row) + headers = [ + col.strip() + for col in table_lines[0].split('|') + if col.strip() + ] + + # Extract data rows (skip header and separator rows) + data_rows = [] + for line in table_lines[2:]: # Skip header and separator rows + row_data = [ + cell.strip() + for cell in line.split('|') + if cell.strip() + ] + if len(row_data) == len(headers): + data_rows.append(row_data) + + # Create DataFrame with explicit index + df = pd.DataFrame( + data_rows, + columns=headers, + index=range(len(data_rows)) + ) + + # Display the DataFrame + st.subheader("Competitor Comparison (Table)") + st.table(df) + + except Exception as e: + st.error(f"Error converting table to DataFrame: {str(e)}") + st.write("Raw table data for debugging:", table_lines) + def generate_analysis_report(competitor_data: list): combined_data = "\n\n".join([str(data) for data in competitor_data]) @@ -144,11 +222,16 @@ if "openai_api_key" in st.session_state and "firecrawl_api_key" in st.session_st competitor_info = extract_competitor_info(url) competitor_data.append(competitor_info) + # Generate and display comparison report + with st.spinner("Generating comparison table..."): + generate_comparison_report(competitor_data) + + # Generate and display final analysis report with st.spinner("Generating analysis report..."): analysis_report = generate_analysis_report(competitor_data) + st.subheader("Competitor Analysis Report") + st.markdown(analysis_report) st.success("Analysis complete!") - st.subheader("Competitor Analysis Report") - st.markdown(analysis_report) else: st.error("Please provide either a URL or a description.") \ No newline at end of file From 9cfa8b5888dec22eaa9677473ee54e2b3cb5e615 Mon Sep 17 00:00:00 2001 From: Madhu Date: Mon, 13 Jan 2025 20:18:46 +0530 Subject: [PATCH 8/8] added readme and requirements.txt --- .../ai_competitors_analysis_team/README.md | 77 +++++++++++++++++++ .../ai_competitor_analyser.py | 12 +-- .../requirements.txt | 4 +- 3 files changed, 85 insertions(+), 8 deletions(-) diff --git a/ai_agent_tutorials/ai_competitors_analysis_team/README.md b/ai_agent_tutorials/ai_competitors_analysis_team/README.md index e69de29..50f5382 100644 --- a/ai_agent_tutorials/ai_competitors_analysis_team/README.md +++ b/ai_agent_tutorials/ai_competitors_analysis_team/README.md @@ -0,0 +1,77 @@ +# AI Competitor Intelligence Agent 🔍 + +The AI Competitor Intelligence Agent is a powerful competitor analysis tool powered by Firecrawl and Phidata's AI Agent framework. This app helps businesses analyze their competitors by extracting structured data from competitor websites and generating actionable insights using AI. + +## Features + +- **Multi-Agent System** + - **Firecrawl Agent**: Specializes in crawling and summarizing competitor websites + - **Analysis Agent**: Generates detailed competitive analysis reports + - **Comparison Agent**: Creates structured comparisons between competitors + +- **Competitor Discovery**: + - Finds similar companies using URL matching with Exa AI + - Discovers competitors based on business descriptions + - Automatically extracts relevant competitor URLs + +- **Comprehensive Analysis**: + - Provides structured analysis reports with: + - Market gaps and opportunities + - Competitor weaknesses + - Recommended features + - Pricing strategies + - Growth opportunities + - Actionable recommendations + +- **Interactive Analysis**: Users can input either their company URL or description for analysis + +## Requirements + +The application requires the following Python libraries: + +- `phidata` +- `exa-py` +- `streamlit` +- `pandas` +- `firecrawl-py` + +You'll also need API keys for: +- OpenAI +- Firecrawl +- Exa + +## How to Run + +Follow these steps to set up and run the application: + +1. **Clone the Repository**: + ```bash + git clone https://github.com/Shubhamsaboo/awesome-llm-apps.git + cd ai_agent_tutorials/ai_competitors_analysis_team + ``` + +2. **Install the dependencies**: + ```bash + pip install -r requirements.txt + ``` + +3. **Set up your API keys**: + - Get an OpenAI API key from: https://platform.openai.com/api-keys + - Get a Firecrawl API key from: [Firecrawl website](https://www.firecrawl.dev/app/api-keys) + - Get an Exa API key from: [Exa website](https://dashboard.exa.ai/api-keys) + +4. **Run the Streamlit app**: + ```bash + streamlit run ai_competitor_analyser.py + ``` + +## Usage + +1. Enter your API keys in the sidebar +2. Input either: + - Your company's website URL + - A description of your company +3. Click "Analyze Competitors" to generate: + - Competitor comparison table + - Detailed analysis report + - Strategic recommendations diff --git a/ai_agent_tutorials/ai_competitors_analysis_team/ai_competitor_analyser.py b/ai_agent_tutorials/ai_competitors_analysis_team/ai_competitor_analyser.py index c19cb7e..626e622 100644 --- a/ai_agent_tutorials/ai_competitors_analysis_team/ai_competitor_analyser.py +++ b/ai_agent_tutorials/ai_competitors_analysis_team/ai_competitor_analyser.py @@ -34,7 +34,7 @@ st.info( ) # Input fields for URL and description -url = st.text_input("Enter your company URL (optional):") +url = st.text_input("Enter your company URL :") description = st.text_area("Enter a description of your company (if URL is not available):") # Initialize API keys and tools @@ -144,8 +144,8 @@ if "openai_api_key" in st.session_state and "firecrawl_api_key" in st.session_st ) # Display the raw markdown table first - # st.subheader("Competitor Comparison (Markdown)") - # st.markdown(comparison_response.content) + st.subheader("Competitor Comparison") + st.markdown(comparison_response.content) try: # Split the markdown table into lines and clean them @@ -180,9 +180,9 @@ if "openai_api_key" in st.session_state and "firecrawl_api_key" in st.session_st index=range(len(data_rows)) ) - # Display the DataFrame - st.subheader("Competitor Comparison (Table)") - st.table(df) + # # Display the DataFrame + # st.subheader("Competitor Comparison Table") + # st.table(df) except Exception as e: st.error(f"Error converting table to DataFrame: {str(e)}") diff --git a/ai_agent_tutorials/ai_competitors_analysis_team/requirements.txt b/ai_agent_tutorials/ai_competitors_analysis_team/requirements.txt index 83050a0..f34f800 100644 --- a/ai_agent_tutorials/ai_competitors_analysis_team/requirements.txt +++ b/ai_agent_tutorials/ai_competitors_analysis_team/requirements.txt @@ -1,5 +1,5 @@ -exa-py==1.0.6 +exa-py==1.7.1 firecrawl-py==1.9.0 duckduckgo-search==7.2.1 phidata==2.7.3 -streamlit \ No newline at end of file +streamlit==1.41.1 \ No newline at end of file