admin-valentin/open-notebook
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

improve cleanup function

Browse source
This commit is contained in:
LUIS NOVO 2024-11-14 15:19:21 -03:00
parent d1b01c4631
commit 01db97924e
1 changed files with 5 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
open_notebook/utils.py
View file

@ -84,12 +84,17 @@ def remove_non_printable(text) -> str:
# Replace any special Unicode whitespace characters with a regular space # Replace any special Unicode whitespace characters with a regular space
text = re.sub(r"[\u2000-\u200B\u202F\u205F\u3000]", " ", text) text = re.sub(r"[\u2000-\u200B\u202F\u205F\u3000]", " ", text)
# Replace unusual line terminators with a single newline
text = re.sub(r"[\u2028\u2029\r]", "\n", text)
# Remove control characters, except newlines and tabs # Remove control characters, except newlines and tabs
text = "".join( text = "".join(
char for char in text if unicodedata.category(char)[0] != "C" or char in "\n\t" char for char in text if unicodedata.category(char)[0] != "C" or char in "\n\t"
) )
# Replace non-breaking spaces with regular spaces # Replace non-breaking spaces with regular spaces
text = text.replace("\xa0", " ").strip() text = text.replace("\xa0", " ").strip()
# Keep letters (including accented ones), numbers, spaces, newlines, tabs, and basic punctuation # Keep letters (including accented ones), numbers, spaces, newlines, tabs, and basic punctuation
return re.sub(r"[^\w\s.,!?\-\n\t]", "", text, flags=re.UNICODE) return re.sub(r"[^\w\s.,!?\-\n\t]", "", text, flags=re.UNICODE)