admin-valentin/openai-cookbook
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Catch the exception thrown by the With.Open and continue with the queue (#155)

Browse source 
Co-authored-by: Ashok Manghat <amanghat@rmplc.net>
This commit is contained in:
DevilsWorkShop 2023-09-12 04:25:57 +05:30 committed by GitHub
parent 18b7ee5f37
commit 39b62a6c09
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 19 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
apps/web-crawl-q-and-a/web-qa.py
View file

@ -20,6 +20,9 @@ from ast import literal_eval
# Regex pattern to match a URL # Regex pattern to match a URL
HTTP_URL_PATTERN = r'^http[s]{0,1}://.+$' HTTP_URL_PATTERN = r'^http[s]{0,1}://.+$'
# Define OpenAI api_key
# openai.api_key = '<Your API Key>'
# Define root domain to crawl # Define root domain to crawl
domain = "openai.com" domain = "openai.com"
full_url = "https://openai.com/" full_url = "https://openai.com/"
@ -137,6 +140,8 @@ def crawl(url):
url = queue.pop() url = queue.pop()
print(url) # for debugging and to see the progress print(url) # for debugging and to see the progress
# Try extracting the text from the link, if failed proceed with the next item in the queue
try:
# Save text from the url to a <url>.txt file # Save text from the url to a <url>.txt file
with open('text/'+local_domain+'/'+url[8:].replace("/", "_") + ".txt", "w", encoding="UTF-8") as f: with open('text/'+local_domain+'/'+url[8:].replace("/", "_") + ".txt", "w", encoding="UTF-8") as f:
@ -152,6 +157,8 @@ def crawl(url):
# Otherwise, write the text to the file in the text directory # Otherwise, write the text to the file in the text directory
f.write(text) f.write(text)
except Exception as e:
print("Unable to parse page " + url)
# Get the hyperlinks from the URL and add them to the queue # Get the hyperlinks from the URL and add them to the queue
for link in get_domain_hyperlinks(local_domain, url): for link in get_domain_hyperlinks(local_domain, url):