admin-valentin/openai-cookbook
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update web-qa.py

Browse source
This commit is contained in:
Logan Kilpatrick 2023-02-06 11:28:31 -06:00 committed by GitHub
parent a7b8506c6e
commit a2837aa5cd
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 2 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
solutions/web_crawl_Q&A/web-qa.py
View file

@ -135,7 +135,7 @@ def crawl(url):
print(url) # for debugging and to see the progress
# Save text from the url to a <url>.txt file
with open('text/'+local_domain+'/'+url[8:].replace("/", "_") + ".txt", "w", encoding="utf-8") as f:
with open('text/'+local_domain+'/'+url[8:].replace("/", "_") + ".txt", "w", encoding="UTF-8") as f:
# Get the text from the URL using BeautifulSoup
soup = BeautifulSoup(requests.get(url).text, "html.parser")
@ -181,7 +181,7 @@ texts=[]
for file in os.listdir("text/" + domain + "/"):
# Open the file and read the text
with open("text/" + domain + "/" + file, "r", encoding="utf-8") as f:
with open("text/" + domain + "/" + file, "r", encoding="UTF-8") as f:
text = f.read()
# Omit the first 11 lines and the last 4 lines, then replace -, _, and #update with spaces.