274fb1c025
# Add Bright Data Web Scraping and Data Extraction Toolkit ## Overview This PR introduces a comprehensive Bright Data toolkit that provides web scraping, search, and structured data extraction capabilities through the Bright Data API. ## Features Added ### Core Tools 1. **`scrape_as_markdown`** - Scrapes any webpage and returns clean Markdown content 2. **`get_screenshot`** - Captures screenshots of webpages and saves them locally 3. **`search_engine`** - Advanced search functionality across Google, Bing, and Yandex with customizable parameters 4. **`web_data_feed`** - Extracts structured data from major platforms (LinkedIn, Amazon, Instagram, Facebook, X, YouTube, Zillow, Booking.com, etc.) ### Supporting Infrastructure - **`BrightDataClient`** - Error handling - URL encoding utilities and request optimization ## Technical Details ### Search Engine Capabilities - Multi-engine support (Google, Bing, Yandex) - Advanced parameters: language, country, search type (images, shopping, news) - Device targeting (mobile, iOS, Android, iPad) - Pagination and result count control - Location-based searches ### Structured Data Sources Supports 13+ data sources including: - **E-commerce**: Amazon products and reviews - **Professional**: LinkedIn profiles and companies, ZoomInfo - **Social Media**: Instagram, Facebook, X (Twitter) content - **Real Estate**: Zillow property listings - **Travel**: Booking.com hotel listings - **Video**: YouTube videos and metadata ## Testing & Validation - [x] Deployed and tested on personal account - [x] Tested via ngrok as well - [x] Verified all tool functions work as expected - [x] Validated against multiple data sources and search engines - [x] Confirmed error handling and edge cases ## Security & Best Practices - Requires proper API key and zone configuration via secrets ## Dependencies - `requests` - HTTP client - `arcade_tdk` - Arcade toolkit framework - Standard library modules: `json`, `time`, `typing`, `urllib.parse` ## Notes - All tools require `BRIGHTDATA_API_KEY` secret - Search and scraping tools also require `BRIGHTDATA_ZONE` secret - Follows Arcade AI toolkit patterns and conventions - Comprehensive docstrings with examples provided This toolkit significantly expands Arcade AI's web data capabilities, enabling users to scrape, search, and extract structured data from across the web through a single, unified interface. --------- Authored-by: meirk-brd
58 lines
1.3 KiB
TOML
58 lines
1.3 KiB
TOML
[build-system]
|
|
requires = [ "hatchling",]
|
|
build-backend = "hatchling.build"
|
|
|
|
[project]
|
|
name = "brightdata"
|
|
version = "0.1.0"
|
|
description = "Search, Crawl and Scrape any site, at scale, without getting blocked"
|
|
requires-python = ">=3.10"
|
|
dependencies = [
|
|
"arcade-tdk>=2.0.0,<3.0.0",
|
|
"requests>=2.32.5",
|
|
|
|
]
|
|
[[project.authors]]
|
|
name = "meirk-brd"
|
|
email = "meirk@brightdata.com"
|
|
|
|
|
|
[project.optional-dependencies]
|
|
dev = [
|
|
"arcade-ai[evals]>=2.1.4,<3.0.0",
|
|
"arcade-serve>=2.0.0,<3.0.0",
|
|
"pytest>=8.3.0,<8.4.0",
|
|
"pytest-cov>=4.0.0,<4.1.0",
|
|
"pytest-mock>=3.11.1,<3.12.0",
|
|
"pytest-asyncio>=0.24.0,<0.25.0",
|
|
"types-requests>=2.32.0",
|
|
"mypy>=1.5.1,<1.6.0",
|
|
"pre-commit>=3.4.0,<3.5.0",
|
|
"tox>=4.11.1,<4.12.0",
|
|
"ruff>=0.7.4,<0.8.0",
|
|
]
|
|
|
|
# Tell Arcade.dev that this package is a toolkit
|
|
[project.entry-points.arcade_toolkits]
|
|
toolkit_name = "brightdata"
|
|
|
|
[tool.mypy]
|
|
files = [ "brightdata/**/*.py",]
|
|
python_version = "3.10"
|
|
disallow_untyped_defs = "True"
|
|
disallow_any_unimported = "True"
|
|
no_implicit_optional = "True"
|
|
check_untyped_defs = "True"
|
|
warn_return_any = "True"
|
|
warn_unused_ignores = "True"
|
|
show_error_codes = "True"
|
|
ignore_missing_imports = "True"
|
|
|
|
[tool.pytest.ini_options]
|
|
testpaths = [ "tests",]
|
|
|
|
[tool.coverage.report]
|
|
skip_empty = true
|
|
|
|
[tool.hatch.build.targets.wheel]
|
|
packages = [ "brightdata",]
|