diff --git a/Makefile b/Makefile index d10e1c7..0f0ce2f 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ PLATFORMS=linux/amd64,linux/arm64 #,linux/arm/v7,linux/386 database: - docker compose up -d + docker compose up surrealdb run: poetry run streamlit run app_home.py diff --git a/open_notebook/exceptions.py b/open_notebook/exceptions.py index 45a5ea4..501e67a 100644 --- a/open_notebook/exceptions.py +++ b/open_notebook/exceptions.py @@ -74,3 +74,9 @@ class InvalidDatabaseSchema(OpenNotebookError): """Raised when the database is not under the expected schema.""" pass + + +class NoTranscriptFound(OpenNotebookError): + """Raised when no transcript is found for a video.""" + + pass diff --git a/open_notebook/graphs/content_process.py b/open_notebook/graphs/content_process.py deleted file mode 100644 index 04a0d66..0000000 --- a/open_notebook/graphs/content_process.py +++ /dev/null @@ -1,560 +0,0 @@ -import json -import os -import re -import subprocess -import unicodedata -from math import ceil - -import fitz # type: ignore -import magic -import requests # type: ignore -from langgraph.graph import END, START, StateGraph -from loguru import logger -from pydub import AudioSegment -from typing_extensions import TypedDict -from youtube_transcript_api import YouTubeTranscriptApi # type: ignore -from youtube_transcript_api.formatters import TextFormatter # type: ignore - -from open_notebook.config import CONFIG -from open_notebook.exceptions import UnsupportedTypeException - - -class SourceState(TypedDict): - content: str - file_path: str - url: str - title: str - source_type: str - identified_type: str - identified_provider: str - - -def source_identification(state: SourceState): - """ - Identify the content source based on parameters - """ - if state.get("content"): - doc_type = "text" - elif state.get("file_path"): - doc_type = "file" - elif state.get("url"): - doc_type = "url" - else: - raise ValueError("No source provided.") - - return {"source_type": doc_type} - - -def url_provider(state: SourceState): - """ - Identify the provider - """ - return_dict = {} - url = state.get("url") - if url: - if "youtube.com" in url or "youtu.be" in url: - return_dict["identified_type"] = ( - "youtube" # playlists, channels in the future - ) - else: - return_dict["identified_type"] = "article" - # article providers in the future - return return_dict - - -def file_type(state: SourceState): - """ - Identify the file using python-magic - """ - return_dict = {} - file_path = state.get("file_path") - if file_path is not None: - return_dict["identified_type"] = magic.from_file(file_path, mime=True) - return return_dict - - -def clean_pdf_text(text): - """ - Clean text extracted from PDFs with enhanced space handling. - - Args: - text (str): The raw text extracted from a PDF - Returns: - str: Cleaned text with minimal necessary spacing - """ - if not text: - return text - - # Step 1: Normalize Unicode characters - text = unicodedata.normalize("NFKC", text) - - # Step 2: Replace common PDF artifacts - replacements = { - # Common ligatures - "fi": "fi", - "fl": "fl", - "ff": "ff", - "ffi": "ffi", - "ffl": "ffl", - # Quotation marks and apostrophes - """: "'", """: "'", - '"': '"', - "′": "'", - "‚": ",", - "„": '"', - # Dashes and hyphens - "‒": "-", - "–": "-", - "—": "-", - "―": "-", - # Other common replacements - "…": "...", - "•": "*", - "°": " degrees ", - "¹": "1", - "²": "2", - "³": "3", - "©": "(c)", - "®": "(R)", - "™": "(TM)", - } - for old, new in replacements.items(): - text = text.replace(old, new) - - # Step 3: Advanced space cleaning - # Remove control characters while preserving essential whitespace - text = "".join( - char for char in text if unicodedata.category(char)[0] != "C" or char in "\n\t " - ) - - # Step 4: Enhanced space cleaning - text = re.sub(r"[ \t]+", " ", text) # Consolidate horizontal whitespace - text = re.sub(r" +\n", "\n", text) # Remove spaces before newlines - text = re.sub(r"\n +", "\n", text) # Remove spaces after newlines - text = re.sub(r"\n\t+", "\n", text) # Remove tabs at start of lines - text = re.sub(r"\t+\n", "\n", text) # Remove tabs at end of lines - text = re.sub(r"\t+", " ", text) # Replace tabs with single space - - # Step 5: Remove empty lines while preserving paragraph structure - text = re.sub(r"\n{3,}", "\n\n", text) # Max two consecutive newlines - text = re.sub(r"^\s+", "", text) # Remove leading whitespace - text = re.sub(r"\s+$", "", text) # Remove trailing whitespace - - # Step 6: Clean up around punctuation - text = re.sub(r"\s+([.,;:!?)])", r"\1", text) # Remove spaces before punctuation - text = re.sub(r"(\()\s+", r"\1", text) # Remove spaces after opening parenthesis - text = re.sub( - r"\s+([.,])\s+", r"\1 ", text - ) # Ensure single space after periods and commas - - # Step 7: Remove zero-width and invisible characters - text = re.sub(r"[\u200b\u200c\u200d\ufeff\u200e\u200f]", "", text) - - # Step 8: Fix hyphenation and line breaks - text = re.sub( - r"(?<=\w)-\s*\n\s*(?=\w)", "", text - ) # Remove hyphenation at line breaks - - return text.strip() - - -def _extract_text_from_pdf(pdf_path): - doc = fitz.open(pdf_path) - text = "" - for page in doc: - text += page.get_text() - doc.close() - - normalized_text = clean_pdf_text(text) - return normalized_text - - -def extract_pdf(state: SourceState): - """ - Parse the text file and print its content. - """ - return_dict = {} - if ( - state.get("file_path") is not None - and state.get("identified_type") == "application/pdf" - ): - file_path = state.get("file_path") - try: - text = _extract_text_from_pdf(file_path) - return_dict["content"] = text - except FileNotFoundError: - raise FileNotFoundError(f"File not found at {file_path}") - except Exception as e: - raise Exception(f"An error occurred: {e}") - - return return_dict - - -def extract_url(state: SourceState): - """ - Get the content of a URL - """ - response = requests.get(f"https://r.jina.ai/{state.get('url')}") - text = response.text - if text.startswith("Title:") and "\n" in text: - title_end = text.index("\n") - title = text[6:title_end].strip() - logger.debug(f"Content has title - {title}") - logger.debug(text[:100]) - content = text[title_end + 1 :].strip() - return {"title": title, "content": content} - else: - logger.debug("Content does not have URL") - return {"content": text} - - -def _get_title(url): - """ - Get the content of a URL - """ - response = extract_url(dict(url=url)) - if "title" in response: - return response["title"] - - -def extract_txt(state: SourceState): - """ - Parse the text file and print its content. - """ - return_dict = {} - if ( - state.get("file_path") is not None - and state.get("identified_type") == "text/plain" - ): - file_path = state.get("file_path") - if file_path is not None: - try: - with open(file_path, "r", encoding="utf-8") as file: - content = file.read() - return_dict["content"] = content - except FileNotFoundError: - raise FileNotFoundError(f"File not found at {file_path}") - except Exception as e: - raise Exception(f"An error occurred: {e}") - - return return_dict - - -def _extract_youtube_id(url): - """ - Extract the YouTube video ID from a given URL using regular expressions. - - Args: - url (str): The YouTube URL from which to extract the video ID. - - Returns: - str: The extracted YouTube video ID or None if no valid ID is found. - """ - # Define a regular expression pattern to capture the YouTube video ID - youtube_regex = ( - r"(?:https?://)?" # Optional scheme - r"(?:www\.)?" # Optional www. - r"(?:" - r"youtu\.be/" # Shortened URL - r"|youtube\.com" # Main URL - r"(?:" # Group start - r"/embed/" # Embed URL - r"|/v/" # Older video URL - r"|/watch\?v=" # Standard watch URL - r"|/watch\?.+&v=" # Other watch URL - r")" # Group end - r")" # End main group - r"([\w-]{11})" # 11 characters (YouTube video ID) - ) - - # Search the URL for the pattern - match = re.search(youtube_regex, url) - - # Return the video ID if a match is found - return match.group(1) if match else None - - -def extract_youtube_transcript(state: SourceState): - """ - Parse the text file and print its content. - """ - - languages = CONFIG.get("youtube_transcripts", {}).get( - "preferred_languages", ["en", "es", "pt"] - ) - - video_id = _extract_youtube_id(state.get("url")) - transcript = YouTubeTranscriptApi.get_transcript(video_id, languages=languages) - formatter = TextFormatter() - title = _get_title(state.get("url")) - return {"content": formatter.format_transcript(transcript), "title": title} - - -def should_continue(data: SourceState): - if data.get("source_type") == "url": - return "parse_url" - else: - return "end" - - -def split_audio(input_file, segment_length_minutes=15, output_prefix=None): - """ - Split an audio file into segments of specified length. - - Args: - input_file (str): Path to the input audio file - segment_length_minutes (int): Length of each segment in minutes - output_dir (str): Directory to save the segments (defaults to input file's directory) - output_prefix (str): Prefix for output files (defaults to input filename) - - Returns: - list: List of paths to the created segment files - """ - # Convert input file to absolute path - input_file = os.path.abspath(input_file) - - output_dir = os.path.dirname(input_file) - os.makedirs(output_dir, exist_ok=True) - - # Set up output prefix - if output_prefix is None: - output_prefix = os.path.splitext(os.path.basename(input_file))[0] - - # Load the audio file - audio = AudioSegment.from_file(input_file) - - # Calculate segment length in milliseconds - segment_length_ms = segment_length_minutes * 60 * 1000 - - # Calculate number of segments - total_segments = ceil(len(audio) / segment_length_ms) - - # List to store output file paths - output_files = [] - - # Split the audio into segments - for i in range(total_segments): - # Calculate start and end times for this segment - start_time = i * segment_length_ms - end_time = min((i + 1) * segment_length_ms, len(audio)) - - # Extract segment - segment = audio[start_time:end_time] - - # Generate output filename - # Format: prefix_001.mp3 (padding with zeros ensures correct ordering) - output_filename = f"{output_prefix}_{str(i+1).zfill(3)}.mp3" - output_path = os.path.join(output_dir, output_filename) - - # Export segment - segment.export(output_path, format="mp3") - - output_files.append(output_path) - - # Optional progress indication - print(f"Exported segment {i+1}/{total_segments}: {output_filename}") - - return output_files - - -# todo: add a speechtotext model to the config -def extract_audio(data: SourceState): - input_audio_path = data.get("file_path") - from openai import OpenAI - - client = OpenAI() - - audio_files = split_audio(input_audio_path) - transcriptions = [] - for audio_file in audio_files: - audio_file = open(audio_file, "rb") - transcription = client.audio.transcriptions.create( - model="whisper-1", file=audio_file - ) - transcriptions.append(transcription.text) - return {"content": " ".join(transcriptions)} - - -def get_audio_streams(input_file): - """ - Analyze video file and return information about all audio streams - """ - try: - # Get stream information in JSON format - cmd = [ - "ffprobe", - "-v", - "quiet", - "-print_format", - "json", - "-show_streams", - "-select_streams", - "a", - input_file, - ] - - result = subprocess.run(cmd, capture_output=True, text=True) - if result.returncode != 0: - raise Exception(f"FFprobe failed: {result.stderr}") - - data = json.loads(result.stdout) - return data.get("streams", []) - - except Exception as e: - print(f"Error analyzing file: {str(e)}") - return [] - - -def select_best_audio_stream(streams): - """ - Select the best audio stream based on various quality metrics - """ - if not streams: - return None - - # Score each stream based on various factors - scored_streams = [] - for stream in streams: - score = 0 - - # Prefer higher bit rates - bit_rate = stream.get("bit_rate") - if bit_rate: - score += int(bit_rate) / 1000000 # Convert to Mbps - - # Prefer more channels (stereo over mono) - channels = stream.get("channels", 0) - score += channels * 10 - - # Prefer higher sample rates - sample_rate = stream.get("sample_rate", "0") - score += int(sample_rate) / 48000 - - scored_streams.append((score, stream)) - - # Return the stream with highest score - return max(scored_streams, key=lambda x: x[0])[1] - - -def extract_audio_from_video(input_file, output_file, stream_index): - """ - Extract the specified audio stream to MP3 format - """ - try: - cmd = [ - "ffmpeg", - "-i", - input_file, - "-map", - f"0:a:{stream_index}", # Select specific audio stream - "-codec:a", - "libmp3lame", # Use MP3 codec - "-q:a", - "2", # High quality setting - "-y", # Overwrite output file if exists - output_file, - ] - - result = subprocess.run(cmd, capture_output=True, text=True) - if result.returncode != 0: - raise Exception(f"FFmpeg failed: {result.stderr}") - - return True - - except Exception as e: - print(f"Error extracting audio: {str(e)}") - return False - - -def extract_best_audio_from_video(data: SourceState): - """ - Main function to extract the best audio stream from a video file - """ - input_file = data.get("file_path") - if not os.path.exists(input_file): - print(f"Input file not found: {input_file}") - return False - - base_name = os.path.splitext(input_file)[0] - output_file = f"{base_name}_audio.mp3" - - # Get all audio streams - streams = get_audio_streams(input_file) - if not streams: - print("No audio streams found in the file") - return False - - # Select best stream - best_stream = select_best_audio_stream(streams) - if not best_stream: - print("Could not determine best audio stream") - return False - - # Extract the selected stream - stream_index = streams.index(best_stream) - success = extract_audio_from_video(input_file, output_file, stream_index) - - if success: - print(f"Successfully extracted audio to: {output_file}") - print("Selected stream details:") - print(f"- Channels: {best_stream.get('channels', 'unknown')}") - print(f"- Sample rate: {best_stream.get('sample_rate', 'unknown')} Hz") - print(f"- Bit rate: {best_stream.get('bit_rate', 'unknown')} bits/s") - - return {"file_path": output_file, "identified_type": "audio/mp3"} - - -def file_type_edge(data: SourceState): - if data.get("identified_type") == "text/plain": - return "extract_txt" - elif data.get("identified_type") == "application/pdf": - return "extract_pdf" - elif data.get("identified_type").startswith("video"): - return "extract_best_audio_from_video" - elif data.get("identified_type").startswith("audio"): - return "extract_audio" - else: - raise UnsupportedTypeException( - f"Unsupported file type: {data.get('identified_type')}" - ) - - -workflow = StateGraph(SourceState) -workflow.add_node("source", source_identification) -workflow.add_node("url_provider", url_provider) -workflow.add_node("file_type", file_type) -workflow.add_node("extract_txt", extract_txt) -workflow.add_node("extract_pdf", extract_pdf) -workflow.add_node("extract_url", extract_url) -workflow.add_node("extract_best_audio_from_video", extract_best_audio_from_video) -workflow.add_node("extract_audio", extract_audio) -workflow.add_node("extract_youtube_transcript", extract_youtube_transcript) - -workflow.add_edge(START, "source") -workflow.add_conditional_edges( - "source", - lambda x: x.get("source_type"), - { - "url": "url_provider", - "file": "file_type", - "text": END, - }, -) -workflow.add_conditional_edges( - "file_type", - file_type_edge, -) -workflow.add_conditional_edges( - "url_provider", - lambda x: x.get("identified_type"), - {"article": "extract_url", "youtube": "extract_youtube_transcript"}, -) -workflow.add_edge("url_provider", END) -workflow.add_edge("file_type", END) -workflow.add_edge("extract_txt", END) -workflow.add_edge("extract_pdf", END) -workflow.add_edge("extract_url", END) -workflow.add_edge("extract_best_audio_from_video", "extract_audio") -workflow.add_edge("extract_audio", END) -workflow.add_edge("extract_youtube_transcript", END) -graph = workflow.compile() diff --git a/open_notebook/graphs/content_processing/__init__.py b/open_notebook/graphs/content_processing/__init__.py new file mode 100644 index 0000000..2c772dc --- /dev/null +++ b/open_notebook/graphs/content_processing/__init__.py @@ -0,0 +1,136 @@ +import os + +import magic +from langgraph.graph import END, START, StateGraph +from loguru import logger + +from open_notebook.exceptions import UnsupportedTypeException +from open_notebook.graphs.content_processing.audio import extract_audio +from open_notebook.graphs.content_processing.office import ( + SUPPORTED_OFFICE_TYPES, + extract_office_content, +) +from open_notebook.graphs.content_processing.pdf import ( + SUPPORTED_FITZ_TYPES, + extract_pdf, +) +from open_notebook.graphs.content_processing.state import SourceState +from open_notebook.graphs.content_processing.text import extract_txt +from open_notebook.graphs.content_processing.url import extract_url, url_provider +from open_notebook.graphs.content_processing.video import extract_best_audio_from_video +from open_notebook.graphs.content_processing.youtube import extract_youtube_transcript + + +def source_identification(state: SourceState): + """ + Identify the content source based on parameters + """ + if state.get("content"): + doc_type = "text" + elif state.get("file_path"): + doc_type = "file" + elif state.get("url"): + doc_type = "url" + else: + raise ValueError("No source provided.") + + return {"source_type": doc_type} + + +def file_type(state: SourceState): + """ + Identify the file using python-magic + """ + return_dict = {} + file_path = state.get("file_path") + if file_path is not None: + return_dict["identified_type"] = magic.from_file(file_path, mime=True) + return return_dict + + +# def _get_title(url): +# """ +# Get the content of a URL +# """ +# response = extract_url(dict(url=url)) +# if "title" in response: +# return response["title"] + + +def file_type_edge(data: SourceState): + assert data.get("identified_type"), "Type not identified" + identified_type = data["identified_type"] + + if identified_type == "text/plain": + return "extract_txt" + elif identified_type in SUPPORTED_FITZ_TYPES: + return "extract_pdf" + elif identified_type in SUPPORTED_OFFICE_TYPES: + return "extract_office_content" + elif identified_type.startswith("video"): + return "extract_best_audio_from_video" + elif identified_type.startswith("audio"): + return "extract_audio" + else: + raise UnsupportedTypeException( + f"Unsupported file type: {data.get('identified_type')}" + ) + + +def delete_file(data: SourceState): + if data.get("delete_source"): + logger.debug(f"Deleting file: {data.get('file_path')}") + file_path = data.get("file_path") + if file_path is not None: + try: + os.remove(file_path) + return {"file_path": None} + except FileNotFoundError: + logger.warning(f"File not found while trying to delete: {file_path}") + else: + logger.debug("Not deleting file") + + +workflow = StateGraph(SourceState) +workflow.add_node("source", source_identification) +workflow.add_node("url_provider", url_provider) +workflow.add_node("file_type", file_type) +workflow.add_node("extract_txt", extract_txt) +workflow.add_node("extract_pdf", extract_pdf) +workflow.add_node("extract_url", extract_url) +workflow.add_node("extract_office_content", extract_office_content) +workflow.add_node("extract_best_audio_from_video", extract_best_audio_from_video) +workflow.add_node("extract_audio", extract_audio) +workflow.add_node("extract_youtube_transcript", extract_youtube_transcript) +workflow.add_node("delete_file", delete_file) +workflow.add_edge(START, "source") +workflow.add_conditional_edges( + "source", + lambda x: x.get("source_type"), + { + "url": "url_provider", + "file": "file_type", + "text": END, + }, +) +workflow.add_conditional_edges( + "file_type", + file_type_edge, +) +workflow.add_conditional_edges( + "url_provider", + lambda x: x.get("identified_type"), + {"article": "extract_url", "youtube": "extract_youtube_transcript"}, +) +workflow.add_edge("url_provider", END) +workflow.add_edge("file_type", END) +workflow.add_edge("extract_url", END) +workflow.add_edge("extract_txt", END) +workflow.add_edge("extract_youtube_transcript", END) + +workflow.add_edge("extract_pdf", "delete_file") +workflow.add_edge("extract_office_content", "delete_file") +workflow.add_edge("extract_best_audio_from_video", "extract_audio") +workflow.add_edge("extract_audio", "delete_file") +workflow.add_edge("delete_file", END) +graph = workflow.compile() diff --git a/open_notebook/graphs/content_processing/audio.py b/open_notebook/graphs/content_processing/audio.py new file mode 100644 index 0000000..5afafb7 --- /dev/null +++ b/open_notebook/graphs/content_processing/audio.py @@ -0,0 +1,104 @@ +import os +from math import ceil + +from loguru import logger +from pydub import AudioSegment + +from open_notebook.graphs.content_processing.state import SourceState + +# todo: add a speechtotext model to the config +# future: parallelize the transcription process + + +def split_audio(input_file, segment_length_minutes=15, output_prefix=None): + """ + Split an audio file into segments of specified length. + + Args: + input_file (str): Path to the input audio file + segment_length_minutes (int): Length of each segment in minutes + output_dir (str): Directory to save the segments (defaults to input file's directory) + output_prefix (str): Prefix for output files (defaults to input filename) + + Returns: + list: List of paths to the created segment files + """ + # Convert input file to absolute path + input_file = os.path.abspath(input_file) + + output_dir = os.path.dirname(input_file) + os.makedirs(output_dir, exist_ok=True) + + # Set up output prefix + if output_prefix is None: + output_prefix = os.path.splitext(os.path.basename(input_file))[0] + + # Load the audio file + audio = AudioSegment.from_file(input_file) + + # Calculate segment length in milliseconds + segment_length_ms = segment_length_minutes * 60 * 1000 + + # Calculate number of segments + total_segments = ceil(len(audio) / segment_length_ms) + logger.debug(f"Splitting file: {input_file} into {total_segments} segments") + + # List to store output file paths + output_files = [] + + # Split the audio into segments + for i in range(total_segments): + # Calculate start and end times for this segment + start_time = i * segment_length_ms + end_time = min((i + 1) * segment_length_ms, len(audio)) + + # Extract segment + segment = audio[start_time:end_time] + + # Generate output filename + # Format: prefix_001.mp3 (padding with zeros ensures correct ordering) + output_filename = f"{output_prefix}_{str(i+1).zfill(3)}.mp3" + output_path = os.path.join(output_dir, output_filename) + + # Export segment + segment.export(output_path, format="mp3") + + output_files.append(output_path) + + # Optional progress indication + logger.debug(f"Exported segment {i+1}/{total_segments}: {output_filename}") + + return output_files + + +def extract_audio(data: SourceState): + input_audio_path = data.get("file_path") + from openai import OpenAI + + client = OpenAI() + audio_files = [] + + try: + audio_files = split_audio(input_audio_path) + transcriptions = [] + + for audio_file in audio_files: + with open(audio_file, "rb") as audio: + transcription = client.audio.transcriptions.create( + model="whisper-1", file=audio + ) + transcriptions.append(transcription.text) + + return {"content": " ".join(transcriptions)} + + except Exception as e: + logger.error(f"Error transcribing audio: {str(e)}") + logger.exception(e) + raise # Re-raise the exception after logging + + finally: + for file in audio_files: + try: + os.remove(file) + except OSError as e: + logger.error(f"Error removing temporary file {file}: {str(e)}") diff --git a/open_notebook/graphs/content_processing/office.py b/open_notebook/graphs/content_processing/office.py new file mode 100644 index 0000000..4736d8d --- /dev/null +++ b/open_notebook/graphs/content_processing/office.py @@ -0,0 +1,289 @@ +from docx import Document +from loguru import logger +from openpyxl import load_workbook +from pptx import Presentation + +from open_notebook.graphs.content_processing.state import SourceState + +SUPPORTED_OFFICE_TYPES = [ + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", +] + + +def extract_docx_content_detailed(file_path): + try: + doc = Document(file_path) + content = [] + + for paragraph in doc.paragraphs: + if not paragraph.text.strip(): + continue + + style = paragraph.style.name if paragraph.style else "Normal" + text = paragraph.text.strip() + + # Get paragraph formatting + p_format = paragraph.paragraph_format + indent = p_format.left_indent or 0 + + # Convert indent to spaces (1 level = 4 spaces) + indent_level = 0 + if hasattr(indent, "pt"): + indent_level = int(indent.pt / 72) # 72 points = 1 inch + indent_spaces = " " * (indent_level * 4) + + # Handle different types of formatting + if "Heading" in style: + level = style[-1] if style[-1].isdigit() else "1" + heading_marks = "#" * int(level) + content.append(f"\n{heading_marks} {text}\n") + + # Handle bullet points + elif ( + paragraph.style + and hasattr(paragraph.style, "name") + and paragraph.style.name.startswith("List") + ): + # Numbered list + if ( + hasattr(paragraph._p, "pPr") + and paragraph._p.pPr is not None + and hasattr(paragraph._p.pPr, "numPr") + and paragraph._p.pPr.numPr is not None + ): + # Try to get the actual number + try: + if ( + hasattr(paragraph._p.pPr.numPr, "numId") + and paragraph._p.pPr.numPr.numId is not None + and hasattr(paragraph._p.pPr.numPr.numId, "val") + ): + number = paragraph._p.pPr.numPr.numId.val + content.append(f"{indent_spaces}{number}. {text}") + else: + content.append(f"{indent_spaces}1. {text}") + except Exception: + content.append(f"{indent_spaces}1. {text}") + # Bullet list + else: + content.append(f"{indent_spaces}* {text}") + + else: + # Handle text formatting + formatted_text = [] + for run in paragraph.runs: + if run.bold: + formatted_text.append(f"**{run.text}**") + elif run.italic: + formatted_text.append(f"*{run.text}*") + else: + formatted_text.append(run.text) + + content.append(f"{indent_spaces}{''.join(formatted_text)}") + + return "\n\n".join(content) + + except Exception as e: + logger.error(f"Failed to extract DOCX content: {e}") + return None + + +# Example of usage with metadata +def get_docx_info(file_path): + try: + doc = Document(file_path) + + # Extract core properties if available + core_props = { + "author": doc.core_properties.author, + "created": doc.core_properties.created, + "modified": doc.core_properties.modified, + "title": doc.core_properties.title, + "subject": doc.core_properties.subject, + "keywords": doc.core_properties.keywords, + "category": doc.core_properties.category, + "comments": doc.core_properties.comments, + } + + # Get document content + content = extract_docx_content_detailed(file_path) + + # Get document statistics + stats = { + "paragraph_count": len(doc.paragraphs), + "word_count": sum( + len(p.text.split()) for p in doc.paragraphs if p.text.strip() + ), + "character_count": sum( + len(p.text) for p in doc.paragraphs if p.text.strip() + ), + } + + return {"metadata": core_props, "content": content, "statistics": stats} + + except Exception as e: + logger.error(f"Failed to get DOCX info: {e}") + return None + + +def extract_pptx_content(file_path): + try: + prs = Presentation(file_path) + content = [] + + for slide_number, slide in enumerate(prs.slides, 1): + content.append(f"\n# Slide {slide_number}\n") + + # Extract title + if slide.shapes.title: + content.append(f"## {slide.shapes.title.text}\n") + + # Extract text from all shapes + for shape in slide.shapes: + if hasattr(shape, "text") and shape.text.strip(): + if shape != slide.shapes.title: # Skip title as it's already added + content.append(shape.text.strip()) + + return "\n\n".join(content) + + except Exception as e: + logger.error(f"Failed to extract PPTX content: {e}") + return None + + +def extract_xlsx_content(file_path, max_rows=1000, max_cols=100): + try: + wb = load_workbook(file_path, data_only=True) + content = [] + + for sheet in wb.sheetnames: + ws = wb[sheet] + content.append(f"\n# Sheet: {sheet}\n") + + # Get the maximum row and column with data + max_row = min(ws.max_row, max_rows) + max_col = min(ws.max_column, max_cols) + + # Create markdown table header + headers = [] + for col in range(1, max_col + 1): + cell_value = ws.cell(row=1, column=col).value + headers.append(str(cell_value) if cell_value is not None else "") + + content.append("| " + " | ".join(headers) + " |") + content.append("| " + " | ".join(["---"] * len(headers)) + " |") + + # Add table content + for row in range(2, max_row + 1): + row_data = [] + for col in range(1, max_col + 1): + cell_value = ws.cell(row=row, column=col).value + row_data.append(str(cell_value) if cell_value is not None else "") + content.append("| " + " | ".join(row_data) + " |") + + return "\n".join(content) + + except Exception as e: + logger.error(f"Failed to extract XLSX content: {e}") + return None + + +def get_pptx_info(file_path): + try: + prs = Presentation(file_path) + + # Extract basic properties + props = { + "slide_count": len(prs.slides), + "title": "", # PowerPoint doesn't have built-in metadata like Word + } + + # Get document content + content = extract_pptx_content(file_path) + + # Get presentation statistics + stats = { + "slide_count": len(prs.slides), + "shape_count": sum(len(slide.shapes) for slide in prs.slides), + "text_frame_count": sum( + sum(1 for shape in slide.shapes if hasattr(shape, "text")) + for slide in prs.slides + ), + } + + return {"metadata": props, "content": content, "statistics": stats} + + except Exception as e: + logger.error(f"Failed to get PPTX info: {e}") + return None + + +def get_xlsx_info(file_path): + try: + wb = load_workbook(file_path, data_only=True) + + # Extract basic properties + props = { + "sheet_count": len(wb.sheetnames), + "sheets": wb.sheetnames, + "title": wb.properties.title, + "creator": wb.properties.creator, + "created": wb.properties.created, + "modified": wb.properties.modified, + } + + # Get document content + content = extract_xlsx_content(file_path) + + # Get workbook statistics + stats = { + "sheet_count": len(wb.sheetnames), + "total_rows": sum(sheet.max_row for sheet in wb.worksheets), + "total_columns": sum(sheet.max_column for sheet in wb.worksheets), + } + + return {"metadata": props, "content": content, "statistics": stats} + + except Exception as e: + logger.error(f"Failed to get XLSX info: {e}") + return None + + +def extract_office_content(state: SourceState): + """Universal function to extract content from Office files""" + assert state.get("file_path"), "No file path provided" + assert ( + state.get("identified_type") in SUPPORTED_OFFICE_TYPES + ), "Unsupported File Type" + + file_path = state["file_path"] + doc_type = state["identified_type"] + + if ( + doc_type + == "application/vnd.openxmlformats-officedocument.wordprocessingml.document" + ): + logger.debug("Extracting content from DOCX file") + content = extract_docx_content_detailed(file_path) + info = get_docx_info(file_path) + elif ( + doc_type + == "application/vnd.openxmlformats-officedocument.presentationml.presentation" + ): + logger.debug("Extracting content from PPTX file") + content = extract_pptx_content(file_path) + info = get_pptx_info(file_path) + elif ( + doc_type == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + ): + logger.debug("Extracting content from XLSX file") + content = extract_xlsx_content(file_path) + info = get_xlsx_info(file_path) + else: + raise Exception(f"Unsupported file format: {doc_type}") + + del info["content"] + + return {"content": content, "metadata": info} diff --git a/open_notebook/graphs/content_processing/pdf.py b/open_notebook/graphs/content_processing/pdf.py new file mode 100644 index 0000000..e842a67 --- /dev/null +++ b/open_notebook/graphs/content_processing/pdf.py @@ -0,0 +1,150 @@ +import re +import unicodedata + +import fitz # type: ignore +from loguru import logger + +from open_notebook.graphs.content_processing.state import SourceState + +# todo: find tables - https://pymupdf.readthedocs.io/en/latest/the-basics.html#extracting-tables-from-a-page +# todo: what else can we do to make the text more readable? +# todo: try to fix encoding for some PDF that is still breaking +# def _extract_text_from_pdf(pdf_path): +# doc = fitz.open(pdf_path) +# text = "" +# logger.debug(f"Found {len(doc)} pages in PDF") +# for page in doc: +# # Use encode/decode if you need to clean up any encoding issues +# text += page.get_text().encode('utf-8').decode('utf-8') +# doc.close() +# return text + +SUPPORTED_FITZ_TYPES = [ + "application/pdf", + "application/epub+zip", +] + + +def clean_pdf_text(text): + """ + Clean text extracted from PDFs with enhanced space handling. + Preserves special characters like (, ), %, = that are valid in code/math. + + Args: + text (str): The raw text extracted from a PDF + Returns: + str: Cleaned text with minimal necessary spacing + """ + if not text: + return text + + # Step 1: Normalize Unicode characters + text = unicodedata.normalize("NFKC", text) + + # Step 2: Replace common PDF artifacts + replacements = { + # Common ligatures + "fi": "fi", + "fl": "fl", + "ff": "ff", + "ffi": "ffi", + "ffl": "ffl", + # Quotation marks and apostrophes + """: "'", """: "'", + '"': '"', + "′": "'", + "‚": ",", + "„": '"', + # Dashes and hyphens + "‒": "-", + "–": "-", + "—": "-", + "―": "-", + # Other common replacements + "…": "...", + "•": "*", + "°": " degrees ", + "¹": "1", + "²": "2", + "³": "3", + "©": "(c)", + "®": "(R)", + "™": "(TM)", + } + for old, new in replacements.items(): + text = text.replace(old, new) + + # Step 3: Clean control characters while preserving essential whitespace and special chars + text = "".join( + char + for char in text + if unicodedata.category(char)[0] != "C" + or char in "\n\t " + or char in "()%=[]{}#$@!?.,;:+-*/^<>&|~" + ) + + # Step 4: Enhanced space cleaning + text = re.sub(r"[ \t]+", " ", text) # Consolidate horizontal whitespace + text = re.sub(r" +\n", "\n", text) # Remove spaces before newlines + text = re.sub(r"\n +", "\n", text) # Remove spaces after newlines + text = re.sub(r"\n\t+", "\n", text) # Remove tabs at start of lines + text = re.sub(r"\t+\n", "\n", text) # Remove tabs at end of lines + text = re.sub(r"\t+", " ", text) # Replace tabs with single space + + # Step 5: Remove empty lines while preserving paragraph structure + text = re.sub(r"\n{3,}", "\n\n", text) # Max two consecutive newlines + text = re.sub(r"^\s+", "", text) # Remove leading whitespace + text = re.sub(r"\s+$", "", text) # Remove trailing whitespace + + # Step 6: Clean up around punctuation + text = re.sub(r"\s+([.,;:!?)])", r"\1", text) # Remove spaces before punctuation + text = re.sub(r"(\()\s+", r"\1", text) # Remove spaces after opening parenthesis + text = re.sub( + r"\s+([.,])\s+", r"\1 ", text + ) # Ensure single space after periods and commas + + # Step 7: Remove zero-width and invisible characters + text = re.sub(r"[\u200b\u200c\u200d\ufeff\u200e\u200f]", "", text) + + # Step 8: Fix hyphenation and line breaks + text = re.sub( + r"(?<=\w)-\s*\n\s*(?=\w)", "", text + ) # Remove hyphenation at line breaks + + return text.strip() + + +def _extract_text_from_pdf(pdf_path): + doc = fitz.open(pdf_path) + try: + text = "" + logger.debug(f"Found {len(doc)} pages in PDF") + for page in doc: + text += page.get_text() + normalized_text = clean_pdf_text(text) + return normalized_text + finally: + doc.close() + + +def extract_pdf(state: SourceState): + """ + Parse the text file and print its content. + """ + return_dict = {} + assert state.get("file_path"), "No file path provided" + assert state.get("identified_type") in SUPPORTED_FITZ_TYPES, "Unsupported File Type" + if ( + state.get("file_path") is not None + and state.get("identified_type") in SUPPORTED_FITZ_TYPES + ): + file_path = state.get("file_path") + try: + text = _extract_text_from_pdf(file_path) + return_dict["content"] = text + except FileNotFoundError: + raise FileNotFoundError(f"File not found at {file_path}") + except Exception as e: + raise Exception(f"An error occurred: {e}") + + return return_dict diff --git a/open_notebook/graphs/content_processing/state.py b/open_notebook/graphs/content_processing/state.py new file mode 100644 index 0000000..37bffbf --- /dev/null +++ b/open_notebook/graphs/content_processing/state.py @@ -0,0 +1,13 @@ +from typing_extensions import TypedDict + + +class SourceState(TypedDict): + content: str + file_path: str + url: str + title: str + source_type: str + identified_type: str + identified_provider: str + metadata: dict + delete_source: bool = False diff --git a/open_notebook/graphs/content_processing/text.py b/open_notebook/graphs/content_processing/text.py new file mode 100644 index 0000000..e286e0f --- /dev/null +++ b/open_notebook/graphs/content_processing/text.py @@ -0,0 +1,28 @@ +from loguru import logger + +from open_notebook.graphs.content_processing.state import SourceState + + +def extract_txt(state: SourceState): + """ + Parse the text file and print its content. + """ + return_dict = {} + if ( + state.get("file_path") is not None + and state.get("identified_type") == "text/plain" + ): + logger.debug(f"Extracting text from {state.get('file_path')}") + file_path = state.get("file_path") + if file_path is not None: + try: + with open(file_path, "r", encoding="utf-8") as file: + content = file.read() + logger.debug(f"Extracted: {content[:100]}") + return_dict["content"] = content + except FileNotFoundError: + raise FileNotFoundError(f"File not found at {file_path}") + except Exception as e: + raise Exception(f"An error occurred: {e}") + + return return_dict diff --git a/open_notebook/graphs/content_processing/url.py b/open_notebook/graphs/content_processing/url.py new file mode 100644 index 0000000..05a00fd --- /dev/null +++ b/open_notebook/graphs/content_processing/url.py @@ -0,0 +1,190 @@ +import re +from urllib.parse import urlparse + +import requests # type: ignore +from bs4 import BeautifulSoup, Comment +from loguru import logger + +from open_notebook.graphs.content_processing.state import SourceState + +# future: better extraction methods +# https://github.com/buriy/python-readability +# also try readability: from readability import Document + + +def url_provider(state: SourceState): + """ + Identify the provider + """ + return_dict = {} + url = state.get("url") + if url: + if "youtube.com" in url or "youtu.be" in url: + return_dict["identified_type"] = ( + "youtube" # future: playlists, channels in the future + ) + else: + return_dict["identified_type"] = "article" + # future: article providers in the future + return return_dict + + +def extract_url_bs4(url: str): + """ + Get the title and content of a URL using bs4 + """ + try: + headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36" + } + + # If URL is actually HTML content + if url.startswith("") or url.startswith("") + else None, + "url": url if not url.startswith("") else None, + } + + except requests.exceptions.RequestException as e: + logger.error(f"Failed to fetch URL {url}: {e}") + return None + except Exception as e: + logger.error(f"Failed to process content: {e}") + return None + + +def extract_url_jina(url: str): + """ + Get the content of a URL using Jina + """ + response = requests.get(f"https://r.jina.ai/{url}") + text = response.text + if text.startswith("Title:") and "\n" in text: + title_end = text.index("\n") + title = text[6:title_end].strip() + content = text[title_end + 1 :].strip() + logger.debug( + f"Processed url: {url}, found title: {title}, content: {content[:100]}..." + ) + return {"title": title, "content": content} + else: + content = text + logger.debug( + f"Processed url: {url}, does not have Title prefix, returning full content: {content[:100]}..." + ) + return {"content": text} + + +def extract_url(state: SourceState): + assert state.get("url"), "No URL provided" + url = state["url"] + try: + result = extract_url_bs4(url) + if not result or not result.get("content"): + logger.debug( + f"BS4 extraction failed for url {url}, falling back to Jina extractor" + ) + result = extract_url_jina(url) + return result + except Exception as e: + logger.error(f"URL extraction failed for URL: {url}") + logger.exception(e) + return None diff --git a/open_notebook/graphs/content_processing/video.py b/open_notebook/graphs/content_processing/video.py new file mode 100644 index 0000000..acd23e4 --- /dev/null +++ b/open_notebook/graphs/content_processing/video.py @@ -0,0 +1,140 @@ +import json +import os +import subprocess + +from loguru import logger + +from open_notebook.graphs.content_processing.state import SourceState + + +def extract_audio_from_video(input_file, output_file, stream_index): + """ + Extract the specified audio stream to MP3 format + """ + try: + cmd = [ + "ffmpeg", + "-i", + input_file, + "-map", + f"0:a:{stream_index}", # Select specific audio stream + "-codec:a", + "libmp3lame", # Use MP3 codec + "-q:a", + "2", # High quality setting + "-y", # Overwrite output file if exists + output_file, + ] + + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + raise Exception(f"FFmpeg failed: {result.stderr}") + + return True + + except Exception as e: + print(f"Error extracting audio: {str(e)}") + return False + + +def get_audio_streams(input_file): + """ + Analyze video file and return information about all audio streams + """ + logger.debug(f"Analyzing video file {input_file} for audio streams") + try: + # Get stream information in JSON format + cmd = [ + "ffprobe", + "-v", + "quiet", + "-print_format", + "json", + "-show_streams", + "-select_streams", + "a", + input_file, + ] + + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + raise Exception(f"FFprobe failed: {result.stderr}") + + data = json.loads(result.stdout) + return data.get("streams", []) + + except Exception as e: + print(f"Error analyzing file: {str(e)}") + return [] + + +def select_best_audio_stream(streams): + """ + Select the best audio stream based on various quality metrics + """ + if not streams: + logger.debug("No audio streams found") + return None + else: + logger.debug(f"Found {len(streams)} audio streams") + + # Score each stream based on various factors + scored_streams = [] + for stream in streams: + score = 0 + + # Prefer higher bit rates + bit_rate = stream.get("bit_rate") + if bit_rate: + score += int(int(bit_rate) / 1000000) # Convert to Mbps and ensure int + + # Prefer more channels (stereo over mono) + channels = stream.get("channels", 0) + score += channels * 10 + + # Prefer higher sample rates + sample_rate = stream.get("sample_rate", "0") + score += int(int(sample_rate) / 48000) + + scored_streams.append((score, stream)) + + # Return the stream with highest score + return max(scored_streams, key=lambda x: x[0])[1] + + +def extract_best_audio_from_video(data: SourceState): + """ + Main function to extract the best audio stream from a video file + """ + input_file = data.get("file_path") + assert input_file is not None, "Input file path must be provided" + if not os.path.exists(input_file): + logger.critical(f"Input file not found: {input_file}") + return False + + base_name = os.path.splitext(input_file)[0] + output_file = f"{base_name}_audio.mp3" + + # Get all audio streams + streams = get_audio_streams(input_file) + if not streams: + logger.debug("No audio streams found in the file") + return False + + # Select best stream + best_stream = select_best_audio_stream(streams) + if not best_stream: + logger.error("Could not determine best audio stream") + return False + + # Extract the selected stream + stream_index = streams.index(best_stream) + success = extract_audio_from_video(input_file, output_file, stream_index) + + if success: + logger.debug(f"Successfully extracted audio to: {output_file}") + logger.debug(f"- Channels: {best_stream.get('channels', 'unknown')}") + logger.debug(f"- Sample rate: {best_stream.get('sample_rate', 'unknown')} Hz") + logger.debug(f"- Bit rate: {best_stream.get('bit_rate', 'unknown')} bits/s") + + return {"file_path": output_file, "identified_type": "audio/mp3"} diff --git a/open_notebook/graphs/content_processing/youtube.py b/open_notebook/graphs/content_processing/youtube.py new file mode 100644 index 0000000..8e73c51 --- /dev/null +++ b/open_notebook/graphs/content_processing/youtube.py @@ -0,0 +1,155 @@ +import re +import ssl + +import requests +from bs4 import BeautifulSoup +from loguru import logger +from youtube_transcript_api import YouTubeTranscriptApi # type: ignore +from youtube_transcript_api.formatters import TextFormatter # type: ignore + +from open_notebook.config import CONFIG +from open_notebook.exceptions import NoTranscriptFound +from open_notebook.graphs.content_processing.state import SourceState + +ssl._create_default_https_context = ssl._create_unverified_context + + +def get_video_title(video_id): + try: + url = f"https://www.youtube.com/watch?v={video_id}" + response = requests.get(url) + soup = BeautifulSoup(response.text, "html.parser") + + # YouTube stores title in a meta tag + title = soup.find("meta", property="og:title")["content"] + return title + + except Exception as e: + logger.error(f"Failed to get video title: {e}") + return None + + +def _extract_youtube_id(url): + """ + Extract the YouTube video ID from a given URL using regular expressions. + + Args: + url (str): The YouTube URL from which to extract the video ID. + + Returns: + str: The extracted YouTube video ID or None if no valid ID is found. + """ + # Define a regular expression pattern to capture the YouTube video ID + youtube_regex = ( + r"(?:https?://)?" # Optional scheme + r"(?:www\.)?" # Optional www. + r"(?:" + r"youtu\.be/" # Shortened URL + r"|youtube\.com" # Main URL + r"(?:" # Group start + r"/embed/" # Embed URL + r"|/v/" # Older video URL + r"|/watch\?v=" # Standard watch URL + r"|/watch\?.+&v=" # Other watch URL + r")" # Group end + r")" # End main group + r"([\w-]{11})" # 11 characters (YouTube video ID) + ) + + # Search the URL for the pattern + match = re.search(youtube_regex, url) + + # Return the video ID if a match is found + return match.group(1) if match else None + + +def get_best_transcript(video_id, preferred_langs=["en", "es", "pt"]): + try: + transcript_list = YouTubeTranscriptApi.list_transcripts(video_id) + + # First try: Manual transcripts in preferred languages + manual_transcripts = [] + try: + for transcript in transcript_list: + if not transcript.is_generated and not transcript.is_translatable: + manual_transcripts.append(transcript) + + if manual_transcripts: + # Sort based on preferred language order + for lang in preferred_langs: + for transcript in manual_transcripts: + if transcript.language_code == lang: + return transcript.fetch() + # If no preferred language found, return first manual transcript + return manual_transcripts[0].fetch() + except NoTranscriptFound: + pass + + # Second try: Auto-generated transcripts in preferred languages + generated_transcripts = [] + try: + for transcript in transcript_list: + if transcript.is_generated and not transcript.is_translatable: + generated_transcripts.append(transcript) + + if generated_transcripts: + # Sort based on preferred language order + for lang in preferred_langs: + for transcript in generated_transcripts: + if transcript.language_code == lang: + return transcript.fetch() + # If no preferred language found, return first generated transcript + return generated_transcripts[0].fetch() + except NoTranscriptFound: + pass + + # Last try: Translated transcripts in preferred languages + translated_transcripts = [] + try: + for transcript in transcript_list: + if transcript.is_translatable: + translated_transcripts.append(transcript) + + if translated_transcripts: + # Sort based on preferred language order + for lang in preferred_langs: + for transcript in translated_transcripts: + if transcript.language_code == lang: + return transcript.fetch() + # If no preferred language found, return translation to first preferred language + translation = translated_transcripts[0].translate(preferred_langs[0]) + return translation.fetch() + except NoTranscriptFound: + pass + + raise Exception("No suitable transcript found") + + except Exception as e: + logger.error(f"Failed to get transcript for video {video_id}: {e}") + return None + + +def extract_youtube_transcript(state: SourceState): + """ + Parse the text file and print its content. + """ + + languages = CONFIG.get("youtube_transcripts", {}).get( + "preferred_languages", ["en", "es", "pt"] + ) + + video_id = _extract_youtube_id(state.get("url")) + transcript = get_best_transcript(video_id, languages) + + logger.debug(f"Found transcript: {transcript}") + formatter = TextFormatter() + try: + title = get_video_title(video_id) + except Exception as e: + logger.critical(f"Failed to get video title for video_id: {video_id}") + logger.exception(e) + title = None + return { + "content": formatter.format_transcript(transcript), + "title": title, + } diff --git a/open_notebook/graphs/multipattern.py b/open_notebook/graphs/multipattern.py index 8e622ec..8f2b6d7 100644 --- a/open_notebook/graphs/multipattern.py +++ b/open_notebook/graphs/multipattern.py @@ -6,7 +6,6 @@ from langchain_core.runnables import ( RunnableConfig, ) from langgraph.graph import END, START, StateGraph -from loguru import logger from typing_extensions import Annotated, TypedDict from open_notebook.graphs.utils import run_pattern @@ -33,7 +32,6 @@ def call_model(state: dict, config: RunnableConfig) -> dict: } current_transformation = "patterns/custom" - logger.debug(f"Using input: {input_args}") transformation_result = run_pattern( pattern_name=current_transformation, model_name=model_name, diff --git a/open_notebook/graphs/utils.py b/open_notebook/graphs/utils.py index 05da335..67f6866 100644 --- a/open_notebook/graphs/utils.py +++ b/open_notebook/graphs/utils.py @@ -1,7 +1,6 @@ import os from langchain.output_parsers import OutputFixingParser -from loguru import logger from open_notebook.llm_router import get_langchain_model from open_notebook.prompter import Prompter @@ -32,7 +31,6 @@ def run_pattern( system_prompt = Prompter(prompt_template=pattern_name, parser=parser).render( data=state ) - logger.debug(f"System prompt: {system_prompt}") if len(messages) > 0: response = chain.invoke([system_prompt] + messages) diff --git a/open_notebook/utils.py b/open_notebook/utils.py index fe8ce42..ab43172 100644 --- a/open_notebook/utils.py +++ b/open_notebook/utils.py @@ -87,6 +87,7 @@ def remove_non_printable(text): text = "".join( char for char in text if unicodedata.category(char)[0] != "C" or char in "\n\t" ) + text = text.replace("\xa0", " ").strip() # Keep letters (including accented ones), numbers, spaces, newlines, tabs, and basic punctuation return re.sub(r"[^\w\s.,!?\-\n\t]", "", text, flags=re.UNICODE) diff --git a/poetry.lock b/poetry.lock index 0552ab6..2ac5bc1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -356,6 +356,20 @@ files = [ {file = "blinker-1.8.2.tar.gz", hash = "sha256:8f77b09d3bf7c795e969e9486f39c2c5e9c39d4ee07424be2bc594ece9642d83"}, ] +[[package]] +name = "bs4" +version = "0.0.2" +description = "Dummy package for Beautiful Soup (beautifulsoup4)" +optional = false +python-versions = "*" +files = [ + {file = "bs4-0.0.2-py2.py3-none-any.whl", hash = "sha256:abf8742c0805ef7f662dce4b51cca104cffe52b835238afc169142ab9b3fbccc"}, + {file = "bs4-0.0.2.tar.gz", hash = "sha256:a48685c58f50fe127722417bae83fe6badf500d54b55f7e39ffe43b798653925"}, +] + +[package.dependencies] +beautifulsoup4 = "*" + [[package]] name = "cachetools" version = "5.5.0" @@ -855,6 +869,17 @@ websockets = ">=11.0" [package.extras] pyaudio = ["pyaudio (>=0.2.14)"] +[[package]] +name = "et-xmlfile" +version = "2.0.0" +description = "An implementation of lxml.xmlfile for the standard library" +optional = false +python-versions = ">=3.8" +files = [ + {file = "et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa"}, + {file = "et_xmlfile-2.0.0.tar.gz", hash = "sha256:dab3f4764309081ce75662649be815c4c9081e88f0837825f90fd28317d4da54"}, +] + [[package]] name = "executing" version = "2.1.0" @@ -2554,6 +2579,160 @@ win32-setctime = {version = ">=1.0.0", markers = "sys_platform == \"win32\""} [package.extras] dev = ["Sphinx (==7.2.5)", "colorama (==0.4.5)", "colorama (==0.4.6)", "exceptiongroup (==1.1.3)", "freezegun (==1.1.0)", "freezegun (==1.2.2)", "mypy (==v0.910)", "mypy (==v0.971)", "mypy (==v1.4.1)", "mypy (==v1.5.1)", "pre-commit (==3.4.0)", "pytest (==6.1.2)", "pytest (==7.4.0)", "pytest-cov (==2.12.1)", "pytest-cov (==4.1.0)", "pytest-mypy-plugins (==1.9.3)", "pytest-mypy-plugins (==3.0.0)", "sphinx-autobuild (==2021.3.14)", "sphinx-rtd-theme (==1.3.0)", "tox (==3.27.1)", "tox (==4.11.0)"] +[[package]] +name = "lxml" +version = "5.3.0" +description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." +optional = false +python-versions = ">=3.6" +files = [ + {file = "lxml-5.3.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:dd36439be765e2dde7660212b5275641edbc813e7b24668831a5c8ac91180656"}, + {file = "lxml-5.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ae5fe5c4b525aa82b8076c1a59d642c17b6e8739ecf852522c6321852178119d"}, + {file = "lxml-5.3.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:501d0d7e26b4d261fca8132854d845e4988097611ba2531408ec91cf3fd9d20a"}, + {file = "lxml-5.3.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb66442c2546446944437df74379e9cf9e9db353e61301d1a0e26482f43f0dd8"}, + {file = "lxml-5.3.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9e41506fec7a7f9405b14aa2d5c8abbb4dbbd09d88f9496958b6d00cb4d45330"}, + {file = "lxml-5.3.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f7d4a670107d75dfe5ad080bed6c341d18c4442f9378c9f58e5851e86eb79965"}, + {file = "lxml-5.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:41ce1f1e2c7755abfc7e759dc34d7d05fd221723ff822947132dc934d122fe22"}, + {file = "lxml-5.3.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:44264ecae91b30e5633013fb66f6ddd05c006d3e0e884f75ce0b4755b3e3847b"}, + {file = "lxml-5.3.0-cp310-cp310-manylinux_2_28_ppc64le.whl", hash = "sha256:3c174dc350d3ec52deb77f2faf05c439331d6ed5e702fc247ccb4e6b62d884b7"}, + {file = "lxml-5.3.0-cp310-cp310-manylinux_2_28_s390x.whl", hash = "sha256:2dfab5fa6a28a0b60a20638dc48e6343c02ea9933e3279ccb132f555a62323d8"}, + {file = "lxml-5.3.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:b1c8c20847b9f34e98080da785bb2336ea982e7f913eed5809e5a3c872900f32"}, + {file = "lxml-5.3.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:2c86bf781b12ba417f64f3422cfc302523ac9cd1d8ae8c0f92a1c66e56ef2e86"}, + {file = "lxml-5.3.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:c162b216070f280fa7da844531169be0baf9ccb17263cf5a8bf876fcd3117fa5"}, + {file = "lxml-5.3.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:36aef61a1678cb778097b4a6eeae96a69875d51d1e8f4d4b491ab3cfb54b5a03"}, + {file = "lxml-5.3.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f65e5120863c2b266dbcc927b306c5b78e502c71edf3295dfcb9501ec96e5fc7"}, + {file = "lxml-5.3.0-cp310-cp310-win32.whl", hash = "sha256:ef0c1fe22171dd7c7c27147f2e9c3e86f8bdf473fed75f16b0c2e84a5030ce80"}, + {file = "lxml-5.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:052d99051e77a4f3e8482c65014cf6372e61b0a6f4fe9edb98503bb5364cfee3"}, + {file = "lxml-5.3.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:74bcb423462233bc5d6066e4e98b0264e7c1bed7541fff2f4e34fe6b21563c8b"}, + {file = "lxml-5.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a3d819eb6f9b8677f57f9664265d0a10dd6551d227afb4af2b9cd7bdc2ccbf18"}, + {file = "lxml-5.3.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5b8f5db71b28b8c404956ddf79575ea77aa8b1538e8b2ef9ec877945b3f46442"}, + {file = "lxml-5.3.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c3406b63232fc7e9b8783ab0b765d7c59e7c59ff96759d8ef9632fca27c7ee4"}, + {file = "lxml-5.3.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2ecdd78ab768f844c7a1d4a03595038c166b609f6395e25af9b0f3f26ae1230f"}, + {file = "lxml-5.3.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:168f2dfcfdedf611eb285efac1516c8454c8c99caf271dccda8943576b67552e"}, + {file = "lxml-5.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa617107a410245b8660028a7483b68e7914304a6d4882b5ff3d2d3eb5948d8c"}, + {file = "lxml-5.3.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:69959bd3167b993e6e710b99051265654133a98f20cec1d9b493b931942e9c16"}, + {file = "lxml-5.3.0-cp311-cp311-manylinux_2_28_ppc64le.whl", hash = "sha256:bd96517ef76c8654446fc3db9242d019a1bb5fe8b751ba414765d59f99210b79"}, + {file = "lxml-5.3.0-cp311-cp311-manylinux_2_28_s390x.whl", hash = "sha256:ab6dd83b970dc97c2d10bc71aa925b84788c7c05de30241b9e96f9b6d9ea3080"}, + {file = "lxml-5.3.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:eec1bb8cdbba2925bedc887bc0609a80e599c75b12d87ae42ac23fd199445654"}, + {file = "lxml-5.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6a7095eeec6f89111d03dabfe5883a1fd54da319c94e0fb104ee8f23616b572d"}, + {file = "lxml-5.3.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6f651ebd0b21ec65dfca93aa629610a0dbc13dbc13554f19b0113da2e61a4763"}, + {file = "lxml-5.3.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:f422a209d2455c56849442ae42f25dbaaba1c6c3f501d58761c619c7836642ec"}, + {file = "lxml-5.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:62f7fdb0d1ed2065451f086519865b4c90aa19aed51081979ecd05a21eb4d1be"}, + {file = "lxml-5.3.0-cp311-cp311-win32.whl", hash = "sha256:c6379f35350b655fd817cd0d6cbeef7f265f3ae5fedb1caae2eb442bbeae9ab9"}, + {file = "lxml-5.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:9c52100e2c2dbb0649b90467935c4b0de5528833c76a35ea1a2691ec9f1ee7a1"}, + {file = "lxml-5.3.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:e99f5507401436fdcc85036a2e7dc2e28d962550afe1cbfc07c40e454256a859"}, + {file = "lxml-5.3.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:384aacddf2e5813a36495233b64cb96b1949da72bef933918ba5c84e06af8f0e"}, + {file = "lxml-5.3.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:874a216bf6afaf97c263b56371434e47e2c652d215788396f60477540298218f"}, + {file = "lxml-5.3.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:65ab5685d56914b9a2a34d67dd5488b83213d680b0c5d10b47f81da5a16b0b0e"}, + {file = "lxml-5.3.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aac0bbd3e8dd2d9c45ceb82249e8bdd3ac99131a32b4d35c8af3cc9db1657179"}, + {file = "lxml-5.3.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b369d3db3c22ed14c75ccd5af429086f166a19627e84a8fdade3f8f31426e52a"}, + {file = "lxml-5.3.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c24037349665434f375645fa9d1f5304800cec574d0310f618490c871fd902b3"}, + {file = "lxml-5.3.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:62d172f358f33a26d6b41b28c170c63886742f5b6772a42b59b4f0fa10526cb1"}, + {file = "lxml-5.3.0-cp312-cp312-manylinux_2_28_ppc64le.whl", hash = "sha256:c1f794c02903c2824fccce5b20c339a1a14b114e83b306ff11b597c5f71a1c8d"}, + {file = "lxml-5.3.0-cp312-cp312-manylinux_2_28_s390x.whl", hash = "sha256:5d6a6972b93c426ace71e0be9a6f4b2cfae9b1baed2eed2006076a746692288c"}, + {file = "lxml-5.3.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:3879cc6ce938ff4eb4900d901ed63555c778731a96365e53fadb36437a131a99"}, + {file = "lxml-5.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:74068c601baff6ff021c70f0935b0c7bc528baa8ea210c202e03757c68c5a4ff"}, + {file = "lxml-5.3.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:ecd4ad8453ac17bc7ba3868371bffb46f628161ad0eefbd0a855d2c8c32dd81a"}, + {file = "lxml-5.3.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:7e2f58095acc211eb9d8b5771bf04df9ff37d6b87618d1cbf85f92399c98dae8"}, + {file = "lxml-5.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e63601ad5cd8f860aa99d109889b5ac34de571c7ee902d6812d5d9ddcc77fa7d"}, + {file = "lxml-5.3.0-cp312-cp312-win32.whl", hash = "sha256:17e8d968d04a37c50ad9c456a286b525d78c4a1c15dd53aa46c1d8e06bf6fa30"}, + {file = "lxml-5.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:c1a69e58a6bb2de65902051d57fde951febad631a20a64572677a1052690482f"}, + {file = "lxml-5.3.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8c72e9563347c7395910de6a3100a4840a75a6f60e05af5e58566868d5eb2d6a"}, + {file = "lxml-5.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e92ce66cd919d18d14b3856906a61d3f6b6a8500e0794142338da644260595cd"}, + {file = "lxml-5.3.0-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1d04f064bebdfef9240478f7a779e8c5dc32b8b7b0b2fc6a62e39b928d428e51"}, + {file = "lxml-5.3.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c2fb570d7823c2bbaf8b419ba6e5662137f8166e364a8b2b91051a1fb40ab8b"}, + {file = "lxml-5.3.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0c120f43553ec759f8de1fee2f4794452b0946773299d44c36bfe18e83caf002"}, + {file = "lxml-5.3.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:562e7494778a69086f0312ec9689f6b6ac1c6b65670ed7d0267e49f57ffa08c4"}, + {file = "lxml-5.3.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:423b121f7e6fa514ba0c7918e56955a1d4470ed35faa03e3d9f0e3baa4c7e492"}, + {file = "lxml-5.3.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:c00f323cc00576df6165cc9d21a4c21285fa6b9989c5c39830c3903dc4303ef3"}, + {file = "lxml-5.3.0-cp313-cp313-manylinux_2_28_ppc64le.whl", hash = "sha256:1fdc9fae8dd4c763e8a31e7630afef517eab9f5d5d31a278df087f307bf601f4"}, + {file = "lxml-5.3.0-cp313-cp313-manylinux_2_28_s390x.whl", hash = "sha256:658f2aa69d31e09699705949b5fc4719cbecbd4a97f9656a232e7d6c7be1a367"}, + {file = "lxml-5.3.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:1473427aff3d66a3fa2199004c3e601e6c4500ab86696edffdbc84954c72d832"}, + {file = "lxml-5.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a87de7dd873bf9a792bf1e58b1c3887b9264036629a5bf2d2e6579fe8e73edff"}, + {file = "lxml-5.3.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:0d7b36afa46c97875303a94e8f3ad932bf78bace9e18e603f2085b652422edcd"}, + {file = "lxml-5.3.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:cf120cce539453ae086eacc0130a324e7026113510efa83ab42ef3fcfccac7fb"}, + {file = "lxml-5.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:df5c7333167b9674aa8ae1d4008fa4bc17a313cc490b2cca27838bbdcc6bb15b"}, + {file = "lxml-5.3.0-cp313-cp313-win32.whl", hash = "sha256:c802e1c2ed9f0c06a65bc4ed0189d000ada8049312cfeab6ca635e39c9608957"}, + {file = "lxml-5.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:406246b96d552e0503e17a1006fd27edac678b3fcc9f1be71a2f94b4ff61528d"}, + {file = "lxml-5.3.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:8f0de2d390af441fe8b2c12626d103540b5d850d585b18fcada58d972b74a74e"}, + {file = "lxml-5.3.0-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1afe0a8c353746e610bd9031a630a95bcfb1a720684c3f2b36c4710a0a96528f"}, + {file = "lxml-5.3.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:56b9861a71575f5795bde89256e7467ece3d339c9b43141dbdd54544566b3b94"}, + {file = "lxml-5.3.0-cp36-cp36m-manylinux_2_28_x86_64.whl", hash = "sha256:9fb81d2824dff4f2e297a276297e9031f46d2682cafc484f49de182aa5e5df99"}, + {file = "lxml-5.3.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:2c226a06ecb8cdef28845ae976da407917542c5e6e75dcac7cc33eb04aaeb237"}, + {file = "lxml-5.3.0-cp36-cp36m-musllinux_1_2_x86_64.whl", hash = "sha256:7d3d1ca42870cdb6d0d29939630dbe48fa511c203724820fc0fd507b2fb46577"}, + {file = "lxml-5.3.0-cp36-cp36m-win32.whl", hash = "sha256:094cb601ba9f55296774c2d57ad68730daa0b13dc260e1f941b4d13678239e70"}, + {file = "lxml-5.3.0-cp36-cp36m-win_amd64.whl", hash = "sha256:eafa2c8658f4e560b098fe9fc54539f86528651f61849b22111a9b107d18910c"}, + {file = "lxml-5.3.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:cb83f8a875b3d9b458cada4f880fa498646874ba4011dc974e071a0a84a1b033"}, + {file = "lxml-5.3.0-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:25f1b69d41656b05885aa185f5fdf822cb01a586d1b32739633679699f220391"}, + {file = "lxml-5.3.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23e0553b8055600b3bf4a00b255ec5c92e1e4aebf8c2c09334f8368e8bd174d6"}, + {file = "lxml-5.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ada35dd21dc6c039259596b358caab6b13f4db4d4a7f8665764d616daf9cc1d"}, + {file = "lxml-5.3.0-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:81b4e48da4c69313192d8c8d4311e5d818b8be1afe68ee20f6385d0e96fc9512"}, + {file = "lxml-5.3.0-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:2bc9fd5ca4729af796f9f59cd8ff160fe06a474da40aca03fcc79655ddee1a8b"}, + {file = "lxml-5.3.0-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:07da23d7ee08577760f0a71d67a861019103e4812c87e2fab26b039054594cc5"}, + {file = "lxml-5.3.0-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:ea2e2f6f801696ad7de8aec061044d6c8c0dd4037608c7cab38a9a4d316bfb11"}, + {file = "lxml-5.3.0-cp37-cp37m-win32.whl", hash = "sha256:5c54afdcbb0182d06836cc3d1be921e540be3ebdf8b8a51ee3ef987537455f84"}, + {file = "lxml-5.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:f2901429da1e645ce548bf9171784c0f74f0718c3f6150ce166be39e4dd66c3e"}, + {file = "lxml-5.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c56a1d43b2f9ee4786e4658c7903f05da35b923fb53c11025712562d5cc02753"}, + {file = "lxml-5.3.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ee8c39582d2652dcd516d1b879451500f8db3fe3607ce45d7c5957ab2596040"}, + {file = "lxml-5.3.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0fdf3a3059611f7585a78ee10399a15566356116a4288380921a4b598d807a22"}, + {file = "lxml-5.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:146173654d79eb1fc97498b4280c1d3e1e5d58c398fa530905c9ea50ea849b22"}, + {file = "lxml-5.3.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:0a7056921edbdd7560746f4221dca89bb7a3fe457d3d74267995253f46343f15"}, + {file = "lxml-5.3.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:9e4b47ac0f5e749cfc618efdf4726269441014ae1d5583e047b452a32e221920"}, + {file = "lxml-5.3.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:f914c03e6a31deb632e2daa881fe198461f4d06e57ac3d0e05bbcab8eae01945"}, + {file = "lxml-5.3.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:213261f168c5e1d9b7535a67e68b1f59f92398dd17a56d934550837143f79c42"}, + {file = "lxml-5.3.0-cp38-cp38-win32.whl", hash = "sha256:218c1b2e17a710e363855594230f44060e2025b05c80d1f0661258142b2add2e"}, + {file = "lxml-5.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:315f9542011b2c4e1d280e4a20ddcca1761993dda3afc7a73b01235f8641e903"}, + {file = "lxml-5.3.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:1ffc23010330c2ab67fac02781df60998ca8fe759e8efde6f8b756a20599c5de"}, + {file = "lxml-5.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2b3778cb38212f52fac9fe913017deea2fdf4eb1a4f8e4cfc6b009a13a6d3fcc"}, + {file = "lxml-5.3.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b0c7a688944891086ba192e21c5229dea54382f4836a209ff8d0a660fac06be"}, + {file = "lxml-5.3.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:747a3d3e98e24597981ca0be0fd922aebd471fa99d0043a3842d00cdcad7ad6a"}, + {file = "lxml-5.3.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:86a6b24b19eaebc448dc56b87c4865527855145d851f9fc3891673ff97950540"}, + {file = "lxml-5.3.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b11a5d918a6216e521c715b02749240fb07ae5a1fefd4b7bf12f833bc8b4fe70"}, + {file = "lxml-5.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:68b87753c784d6acb8a25b05cb526c3406913c9d988d51f80adecc2b0775d6aa"}, + {file = "lxml-5.3.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:109fa6fede314cc50eed29e6e56c540075e63d922455346f11e4d7a036d2b8cf"}, + {file = "lxml-5.3.0-cp39-cp39-manylinux_2_28_ppc64le.whl", hash = "sha256:02ced472497b8362c8e902ade23e3300479f4f43e45f4105c85ef43b8db85229"}, + {file = "lxml-5.3.0-cp39-cp39-manylinux_2_28_s390x.whl", hash = "sha256:6b038cc86b285e4f9fea2ba5ee76e89f21ed1ea898e287dc277a25884f3a7dfe"}, + {file = "lxml-5.3.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:7437237c6a66b7ca341e868cda48be24b8701862757426852c9b3186de1da8a2"}, + {file = "lxml-5.3.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:7f41026c1d64043a36fda21d64c5026762d53a77043e73e94b71f0521939cc71"}, + {file = "lxml-5.3.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:482c2f67761868f0108b1743098640fbb2a28a8e15bf3f47ada9fa59d9fe08c3"}, + {file = "lxml-5.3.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:1483fd3358963cc5c1c9b122c80606a3a79ee0875bcac0204149fa09d6ff2727"}, + {file = "lxml-5.3.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:2dec2d1130a9cda5b904696cec33b2cfb451304ba9081eeda7f90f724097300a"}, + {file = "lxml-5.3.0-cp39-cp39-win32.whl", hash = "sha256:a0eabd0a81625049c5df745209dc7fcef6e2aea7793e5f003ba363610aa0a3ff"}, + {file = "lxml-5.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:89e043f1d9d341c52bf2af6d02e6adde62e0a46e6755d5eb60dc6e4f0b8aeca2"}, + {file = "lxml-5.3.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7b1cd427cb0d5f7393c31b7496419da594fe600e6fdc4b105a54f82405e6626c"}, + {file = "lxml-5.3.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:51806cfe0279e06ed8500ce19479d757db42a30fd509940b1701be9c86a5ff9a"}, + {file = "lxml-5.3.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ee70d08fd60c9565ba8190f41a46a54096afa0eeb8f76bd66f2c25d3b1b83005"}, + {file = "lxml-5.3.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:8dc2c0395bea8254d8daebc76dcf8eb3a95ec2a46fa6fae5eaccee366bfe02ce"}, + {file = "lxml-5.3.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:6ba0d3dcac281aad8a0e5b14c7ed6f9fa89c8612b47939fc94f80b16e2e9bc83"}, + {file = "lxml-5.3.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:6e91cf736959057f7aac7adfc83481e03615a8e8dd5758aa1d95ea69e8931dba"}, + {file = "lxml-5.3.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:94d6c3782907b5e40e21cadf94b13b0842ac421192f26b84c45f13f3c9d5dc27"}, + {file = "lxml-5.3.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c300306673aa0f3ed5ed9372b21867690a17dba38c68c44b287437c362ce486b"}, + {file = "lxml-5.3.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78d9b952e07aed35fe2e1a7ad26e929595412db48535921c5013edc8aa4a35ce"}, + {file = "lxml-5.3.0-pp37-pypy37_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:01220dca0d066d1349bd6a1726856a78f7929f3878f7e2ee83c296c69495309e"}, + {file = "lxml-5.3.0-pp37-pypy37_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:2d9b8d9177afaef80c53c0a9e30fa252ff3036fb1c6494d427c066a4ce6a282f"}, + {file = "lxml-5.3.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:20094fc3f21ea0a8669dc4c61ed7fa8263bd37d97d93b90f28fc613371e7a875"}, + {file = "lxml-5.3.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ace2c2326a319a0bb8a8b0e5b570c764962e95818de9f259ce814ee666603f19"}, + {file = "lxml-5.3.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:92e67a0be1639c251d21e35fe74df6bcc40cba445c2cda7c4a967656733249e2"}, + {file = "lxml-5.3.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd5350b55f9fecddc51385463a4f67a5da829bc741e38cf689f38ec9023f54ab"}, + {file = "lxml-5.3.0-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:4c1fefd7e3d00921c44dc9ca80a775af49698bbfd92ea84498e56acffd4c5469"}, + {file = "lxml-5.3.0-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:71a8dd38fbd2f2319136d4ae855a7078c69c9a38ae06e0c17c73fd70fc6caad8"}, + {file = "lxml-5.3.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:97acf1e1fd66ab53dacd2c35b319d7e548380c2e9e8c54525c6e76d21b1ae3b1"}, + {file = "lxml-5.3.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:68934b242c51eb02907c5b81d138cb977b2129a0a75a8f8b60b01cb8586c7b21"}, + {file = "lxml-5.3.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b710bc2b8292966b23a6a0121f7a6c51d45d2347edcc75f016ac123b8054d3f2"}, + {file = "lxml-5.3.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:18feb4b93302091b1541221196a2155aa296c363fd233814fa11e181adebc52f"}, + {file = "lxml-5.3.0-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:3eb44520c4724c2e1a57c0af33a379eee41792595023f367ba3952a2d96c2aab"}, + {file = "lxml-5.3.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:609251a0ca4770e5a8768ff902aa02bf636339c5a93f9349b48eb1f606f7f3e9"}, + {file = "lxml-5.3.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:516f491c834eb320d6c843156440fe7fc0d50b33e44387fcec5b02f0bc118a4c"}, + {file = "lxml-5.3.0.tar.gz", hash = "sha256:4e109ca30d1edec1ac60cdbe341905dc3b8f55b16855e03a54aaf59e51ec8c6f"}, +] + +[package.extras] +cssselect = ["cssselect (>=0.7)"] +html-clean = ["lxml-html-clean"] +html5 = ["html5lib"] +htmlsoup = ["BeautifulSoup4"] +source = ["Cython (>=3.0.11)"] + [[package]] name = "markdown-it-py" version = "3.0.0" @@ -3163,6 +3342,20 @@ typing-extensions = ">=4.11,<5" [package.extras] datalib = ["numpy (>=1)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)"] +[[package]] +name = "openpyxl" +version = "3.1.5" +description = "A Python library to read/write Excel 2010 xlsx/xlsm files" +optional = false +python-versions = ">=3.8" +files = [ + {file = "openpyxl-3.1.5-py2.py3-none-any.whl", hash = "sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2"}, + {file = "openpyxl-3.1.5.tar.gz", hash = "sha256:cf0e3cf56142039133628b5acffe8ef0c12bc902d2aadd3e0fe5878dc08d1050"}, +] + +[package.dependencies] +et-xmlfile = "*" + [[package]] name = "orjson" version = "3.10.10" @@ -4155,6 +4348,21 @@ files = [ [package.dependencies] six = ">=1.5" +[[package]] +name = "python-docx" +version = "1.1.2" +description = "Create, read, and update Microsoft Word .docx files." +optional = false +python-versions = ">=3.7" +files = [ + {file = "python_docx-1.1.2-py3-none-any.whl", hash = "sha256:08c20d6058916fb19853fcf080f7f42b6270d89eac9fa5f8c15f691c0017fabe"}, + {file = "python_docx-1.1.2.tar.gz", hash = "sha256:0cf1f22e95b9002addca7948e16f2cd7acdfd498047f1941ca5d293db7762efd"}, +] + +[package.dependencies] +lxml = ">=3.1.0" +typing-extensions = ">=4.9.0" + [[package]] name = "python-dotenv" version = "1.0.1" @@ -4194,6 +4402,23 @@ files = [ {file = "python_magic-0.4.27-py2.py3-none-any.whl", hash = "sha256:c212960ad306f700aa0d01e5d7a325d20548ff97eb9920dcd29513174f0294d3"}, ] +[[package]] +name = "python-pptx" +version = "1.0.2" +description = "Create, read, and update PowerPoint 2007+ (.pptx) files." +optional = false +python-versions = ">=3.8" +files = [ + {file = "python_pptx-1.0.2-py3-none-any.whl", hash = "sha256:160838e0b8565a8b1f67947675886e9fea18aa5e795db7ae531606d68e785cba"}, + {file = "python_pptx-1.0.2.tar.gz", hash = "sha256:479a8af0eaf0f0d76b6f00b0887732874ad2e3188230315290cd1f9dd9cc7095"}, +] + +[package.dependencies] +lxml = ">=3.1.0" +Pillow = ">=3.3.2" +typing-extensions = ">=4.9.0" +XlsxWriter = ">=0.5.7" + [[package]] name = "pytz" version = "2024.2" @@ -5670,6 +5895,20 @@ files = [ {file = "types_PyYAML-6.0.12.20240917-py3-none-any.whl", hash = "sha256:392b267f1c0fe6022952462bf5d6523f31e37f6cea49b14cee7ad634b6301570"}, ] +[[package]] +name = "types-requests" +version = "2.32.0.20241016" +description = "Typing stubs for requests" +optional = false +python-versions = ">=3.8" +files = [ + {file = "types-requests-2.32.0.20241016.tar.gz", hash = "sha256:0d9cad2f27515d0e3e3da7134a1b6f28fb97129d86b867f24d9c726452634d95"}, + {file = "types_requests-2.32.0.20241016-py3-none-any.whl", hash = "sha256:4195d62d6d3e043a4eaaf08ff8a62184584d2e8684e9d2aa178c7915a7da3747"}, +] + +[package.dependencies] +urllib3 = ">=2" + [[package]] name = "typing-extensions" version = "4.12.2" @@ -5942,6 +6181,17 @@ files = [ [package.extras] dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"] +[[package]] +name = "xlsxwriter" +version = "3.2.0" +description = "A Python module for creating Excel XLSX files." +optional = false +python-versions = ">=3.6" +files = [ + {file = "XlsxWriter-3.2.0-py3-none-any.whl", hash = "sha256:ecfd5405b3e0e228219bcaf24c2ca0915e012ca9464a14048021d21a995d490e"}, + {file = "XlsxWriter-3.2.0.tar.gz", hash = "sha256:9977d0c661a72866a61f9f7a809e25ebbb0fb7036baa3b9fe74afcfca6b3cb8c"}, +] + [[package]] name = "yarl" version = "1.16.0" @@ -6074,4 +6324,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "f6f2373a3c5f63afd6c2746ed87bb2e84dcea36fac6006b023cc7f6b2f7221c8" +content-hash = "4fa191c6df5a7a355eb0d61f9560ec70e4671ac49cd54fa3a166c1e25c325671" diff --git a/pyproject.toml b/pyproject.toml index 3646b73..a8bffdb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "open-notebook" -version = "0.0.5" +version = "0.0.6" description = "An open source implementation of a research assistant, inspired by Google Notebook LM" authors = ["Luis Novo "] license = "MIT" @@ -42,11 +42,16 @@ sdblpy = "^0.3.0" langchain-google-genai = "^2.0.1" podcastfy = "^0.2.8" tomli = "^2.0.2" +bs4 = "^0.0.2" +python-docx = "^1.1.2" +python-pptx = "^1.0.2" +openpyxl = "^3.1.5" [tool.poetry.group.dev.dependencies] ipykernel = "^6.29.5" ruff = "^0.5.5" mypy = "^1.11.1" +types-requests = "^2.32.0.20241016" [build-system] requires = ["poetry-core"] diff --git a/stream_app/source.py b/stream_app/source.py index 0962337..d8f316a 100644 --- a/stream_app/source.py +++ b/stream_app/source.py @@ -10,7 +10,7 @@ from loguru import logger from open_notebook.config import UPLOADS_FOLDER from open_notebook.domain import Asset, Source from open_notebook.exceptions import UnsupportedTypeException -from open_notebook.graphs.content_process import graph +from open_notebook.graphs.content_processing import graph from open_notebook.graphs.multipattern import graph as transform_graph from open_notebook.utils import surreal_clean @@ -112,25 +112,7 @@ def add_source(session_id): req["url"] = source_link elif source_type == "Upload": source_file = st.file_uploader("Upload") - if source_file is not None: - # Get the file name and extension - file_name = source_file.name - - file_extension = Path(file_name).suffix - - # Generate a unique file name - base_name = Path(file_name).stem - counter = 1 - new_path = os.path.join(UPLOADS_FOLDER, file_name) - while os.path.exists(new_path): - new_file_name = f"{base_name}_{counter}{file_extension}" - new_path = os.path.join(UPLOADS_FOLDER, new_file_name) - counter += 1 - - req["file_path"] = str(new_path) - # Save the file - with open(new_path, "wb") as f: - f.write(source_file.getbuffer()) + req["delete_source"] = st.checkbox("Delete source after processing", value=True) else: source_text = st.text_area("Text") @@ -140,6 +122,25 @@ def add_source(session_id): with st.status("Processing...", expanded=True): st.write("Processing document...") try: + if source_type == "Upload" and source_file is not None: + st.write("Uploading..") + file_name = source_file.name + file_extension = Path(file_name).suffix + base_name = Path(file_name).stem + + # Generate unique filename + new_path = os.path.join(UPLOADS_FOLDER, file_name) + counter = 0 + while os.path.exists(new_path): + counter += 1 + new_file_name = f"{base_name}_{counter}{file_extension}" + new_path = os.path.join(UPLOADS_FOLDER, new_file_name) + + req["file_path"] = str(new_path) + # Save the file + with open(new_path, "wb") as f: + f.write(source_file.getbuffer()) + result = graph.invoke(req) st.write("Saving..") source = Source( @@ -151,10 +152,11 @@ def add_source(session_id): source.add_to_notebook(st.session_state[session_id]["notebook"].id) st.write("Summarizing...") source.generate_toc_and_title() - except UnsupportedTypeException: + except UnsupportedTypeException as e: st.warning( "This type of content is not supported yet. If you think it should be, let us know on the project Issues's page" ) + st.error(e) st.link_button( "Go to Github Issues", url="https://www.github.com/lfnovo/open-notebook/issues",