diff --git a/web_scrapping_ai_agent/local_ai_scrapper.py b/web_scrapping_ai_agent/local_ai_scrapper.py new file mode 100644 index 0000000..38ca154 --- /dev/null +++ b/web_scrapping_ai_agent/local_ai_scrapper.py @@ -0,0 +1,37 @@ +# Import the required libraries +import streamlit as st +from scrapegraphai.graphs import SmartScraperGraph + +# Set up the Streamlit app +st.title("Web Scrapping AI Agent 🕵️‍♂️") +st.caption("This app allows you to scrape a website using OpenAI API") + +# Set up the configuration for the SmartScraperGraph +graph_config = { + "llm": { + "model": "ollama/llama3", + "temperature": 0, + "format": "json", # Ollama needs the format to be specified explicitly + "base_url": "http://localhost:11434", # set Ollama URL + }, + "embeddings": { + "model": "ollama/nomic-embed-text", + "base_url": "http://localhost:11434", # set Ollama URL + }, + "verbose": True, +} +# Get the URL of the website to scrape +url = st.text_input("Enter the URL of the website you want to scrape") +# Get the user prompt +user_prompt = st.text_input("What you want the AI agent to scrae from the website?") + +# Create a SmartScraperGraph object +smart_scraper_graph = SmartScraperGraph( + prompt=user_prompt, + source=url, + config=graph_config +) +# Scrape the website +if st.button("Scrape"): + result = smart_scraper_graph.run() + st.write(result) \ No newline at end of file