feat: implement new content settings page and remove options from the source panel

open_notebook/domain/content_settings.py

@@ -0,0 +1,21 @@
+from typing import ClassVar, Literal, Optional
+
+from pydantic import Field
+
+from open_notebook.domain.base import RecordModel
+
+
+class ContentSettings(RecordModel):
+    record_id: ClassVar[str] = "open_notebook:content_settings"
+    default_content_processing_engine_doc: Optional[
+        Literal["auto", "docling", "simple"]
+    ] = Field("auto", description="Default Content Processing Engine for Documents")
+    default_content_processing_engine_url: Optional[
+        Literal["auto", "firecrawl", "jina", "simple"]
+    ] = Field("auto", description="Default Content Processing Engine for URLs")
+    default_embedding_option: Optional[Literal["ask", "always", "never"]] = Field(
+        "ask", description="Default Embedding Option for Vector Search"
+    )
+    auto_delete_files: Optional[Literal["yes", "no"]] = Field(
+        "yes", description="Auto Delete Uploaded Files"
+    )

open_notebook/graphs/source.py

@@ -1,24 +1,23 @@
 import operator
-from typing import List, Optional
+from typing import Any, Dict, List, Optional
 
-from langchain_core.runnables import (
-    RunnableConfig,
-)
+from content_core import extract_content
+from content_core.common import ProcessSourceState
+from langchain_core.runnables import RunnableConfig
 from langgraph.graph import END, START, StateGraph
 from langgraph.types import Send
 from loguru import logger
 from typing_extensions import Annotated, TypedDict
 
+from open_notebook.domain.content_settings import ContentSettings
 from open_notebook.domain.notebook import Asset, Source
 from open_notebook.domain.transformation import Transformation
-from open_notebook.graphs.content_processing import ContentState
-from open_notebook.graphs.content_processing import graph as content_graph
 from open_notebook.graphs.transformation import graph as transform_graph
 from open_notebook.utils import surreal_clean
 
 
 class SourceState(TypedDict):
-    content_state: ContentState
+    content_state: ProcessSourceState
     apply_transformations: List[Transformation]
     notebook_id: str
     source: Source
@@ -32,9 +31,18 @@ class TransformationState(TypedDict):
 
 
 async def content_process(state: SourceState) -> dict:
-    content_state = state["content_state"]
-    logger.info("Content processing started for new content")
-    processed_state = await content_graph.ainvoke(content_state)
+    content_settings = ContentSettings()
+    content_state: Dict[str, Any] = state["content_state"]
+
+    content_state["url_engine"] = (
+        content_settings.default_content_processing_engine_url or "auto"
+    )
+    content_state["document_engine"] = (
+        content_settings.default_content_processing_engine_doc or "auto"
+    )
+    content_state["output_format"] = "markdown"
+
+    processed_state = await extract_content(content_state)
     return {"content_state": processed_state}
 
 
@@ -42,11 +50,9 @@ def save_source(state: SourceState) -> dict:
     content_state = state["content_state"]
 
     source = Source(
-        asset=Asset(
-            url=content_state.get("url"), file_path=content_state.get("file_path")
-        ),
-        full_text=surreal_clean(content_state["content"]),
-        title=content_state.get("title"),
+        asset=Asset(url=content_state.url, file_path=content_state.file_path),
+        full_text=surreal_clean(content_state.content),
+        title=content_state.title,
     )
     source.save()
 

pages/10_⚙️_Settings.py

@@ -0,0 +1,122 @@
+import os
+
+import streamlit as st
+
+from open_notebook.domain.content_settings import ContentSettings
+from pages.stream_app.utils import setup_page
+
+setup_page("⚙️ Settings")
+
+st.header("⚙️ Settings")
+
+content_settings = ContentSettings()
+
+with st.container(border=True):
+    st.markdown("**Content Processing Engine for Documents**")
+
+    default_content_processing_engine_doc = st.selectbox(
+        "Default Content Processing Engine for Documents",
+        ["auto", "docling", "simple"],
+        index=(
+            ["auto", "docling", "simple"].index(
+                content_settings.default_content_processing_engine_doc
+            )
+            if content_settings.default_content_processing_engine_doc
+            else 0
+        ),
+    )
+    with st.expander("Help me choose"):
+        st.markdown(
+            "- Docling is a little slower but more accurate, specially if the documents contain tables and images.\n- Simple will extract any content from the document without formatiing it. It's ok for simple documents, but will lose quality in complex ones.\n- Auto (recommended) will try to process through docling and default to simple."
+        )
+
+
+with st.container(border=True):
+    st.markdown("**Content Processing Engine for URLs**")
+    firecrawl_enabled = os.getenv("FIRECRAWL_API_KEY") is not None
+    jina_enabled = os.getenv("JINA_API_KEY") is not None
+
+    default_content_processing_engine_url = st.selectbox(
+        "Default Content Processing Engine for URLs",
+        ["auto", "firecrawl", "jina", "simple"],
+        index=(
+            ["auto", "firecrawl", "jina", "simple"].index(
+                content_settings.default_content_processing_engine_url
+            )
+            if content_settings.default_content_processing_engine_url
+            else 0
+        ),
+    )
+    if not firecrawl_enabled and default_content_processing_engine_url in [
+        "firecrawl",
+        "auto",
+    ]:
+        st.warning(
+            "Firecrawl API Key missing. You need to add FIRECRAWL_API_KEY to use it. Get a key at [Firecrawl](https://firecrawl.dev/). If you don't add one, it will default to Jina."
+        )
+    if not jina_enabled and default_content_processing_engine_url in [
+        "jina",
+        "auto",
+    ]:
+        st.warning(
+            "Jina API Key missing. It will work for a few requests a day, but fallback to simple afterwards. Please add JINA_API_KEY to prevent that. Get a key at [Jina.ai](https://jina.ai/)."
+        )
+    with st.expander("Help me choose"):
+        st.markdown(
+            "- Firecrawl is a paid service (with a free tier), and very powerful.\n- Jina is a good option as well and also has a free tier.\n- Simple will use basic HTTP extraction and will miss content on javascript-based websites.\n- Auto (recommended) will try to use firecrawl (if API Key is present). Then, it will use Jina until reaches the limit (or will keep using Jina if you setup the API Key). It will fallback to simple, when none of the previous options is possible."
+        )
+
+with st.container(border=True):
+    st.markdown("**Content Embedding for Vector Search**")
+
+    default_embedding_option = st.selectbox(
+        "Default Embedding Option for Vector Search",
+        ["ask", "always", "never"],
+        index=(
+            ["ask", "always", "never"].index(content_settings.default_embedding_option)
+            if content_settings.default_embedding_option
+            else 0
+        ),
+    )
+
+    with st.expander("Help me choose"):
+        st.markdown(
+            "Embedding the content will make it easier to find by you and by your AI agents. If you are running a local embedding model (Ollama, for example), you shouldn't worry about cost and just embed everything. For online providers, you migtht want to be careful only if you process a lot of content (like 100s of documents at a day)."
+        )
+        st.markdown(
+            "\n\n- Choose **always** if you are running a local embedding model or if your content volume is not that big\n- Choose **ask** if you want to decide every time\n- Choose **never** if you don't care about vector search or do not have an embedding provider."
+        )
+        st.markdown(
+            "As a reference, OpenAI's text-embedding-3-small costs about 0.02 for 1 million tokens -- which is about 30 times the [Wikipedia page for Earth](https://en.wikipedia.org/wiki/Earth). With Gemini API, Text Embedding 004 is free with a rate limit of 1500 requests per minute."
+        )
+
+with st.container(border=True):
+    st.markdown("**Auto Delete Uploaded Files**")
+    auto_delete_files = st.selectbox(
+        "Auto Delete Uploaded Files",
+        ["yes", "no"],
+        index=(
+            ["yes", "no"].index(content_settings.auto_delete_files)
+            if content_settings.auto_delete_files
+            else 0
+        ),
+    )
+    with st.expander("Help me choose"):
+        st.markdown(
+            "Once your files are uploaded and processed, they are not required anymore. Most users should allow Open Notebook to delete uploaded files from the upload folder automatically. Choose **no**, ONLY if you are using Notebook as the primary storage location for those files (which you shouldn't be at all). This option will soon be deprecated in favor of always downloading the files."
+        )
+        st.markdown(
+            "\n\n- Choose **yes** if you are running a local embedding model or if your content volume is not that big\n- Choose **ask** if you want to decide every time\n- Choose **never** if you don't care about vector search or do not have an embedding provider."
+        )
+
+if st.button("Save", key="save_settings"):
+    content_settings.default_content_processing_engine_doc = (
+        default_content_processing_engine_doc
+    )
+    content_settings.default_content_processing_engine_url = (
+        default_content_processing_engine_url
+    )
+    content_settings.default_embedding_option = default_embedding_option
+    content_settings.auto_delete_files = auto_delete_files
+    content_settings.update()
+    st.toast("Settings saved successfully!")

pages/stream_app/source.py

@@ -2,19 +2,22 @@ import asyncio
 import os
 from pathlib import Path
 
+import nest_asyncio
 import streamlit as st
 from humanize import naturaltime
 from loguru import logger
 
 from open_notebook.config import UPLOADS_FOLDER
+from open_notebook.domain.content_settings import ContentSettings
 from open_notebook.domain.models import model_manager
 from open_notebook.domain.notebook import Source
 from open_notebook.domain.transformation import Transformation
 from open_notebook.exceptions import UnsupportedTypeException
 from open_notebook.graphs.source import source_graph
 from pages.components import source_panel
+from pages.stream_app.consts import source_context_icons
 
-from .consts import source_context_icons
+nest_asyncio.apply()
 
 
 @st.dialog("Source", width="large")
@@ -31,6 +34,7 @@ def add_source(notebook_id):
     source_link = None
     source_file = None
     source_text = None
+    content_settings = ContentSettings()
     source_type = st.radio("Type", ["Link", "Upload", "Text"])
     req = {}
     transformations = Transformation.get_all()
@@ -39,7 +43,7 @@ def add_source(notebook_id):
         req["url"] = source_link
     elif source_type == "Upload":
         source_file = st.file_uploader("Upload")
-        req["delete_source"] = st.checkbox("Delete source after processing", value=True)
+        req["delete_source"] = content_settings.auto_delete_files == "yes"
 
     else:
         source_text = st.text_area("Text")
@@ -53,10 +57,22 @@ def add_source(notebook_id):
         format_func=lambda t: t.name,
         default=default_transformations,
     )
-    run_embed = st.checkbox(
-        "Embed content for vector search",
-        help="Creates an embedded content for vector search. Costs a little money and takes a little bit more time. You can do this later if you prefer.",
-    )
+    if content_settings.default_embedding_option == "ask":
+        run_embed = st.checkbox(
+            "Embed content for vector search",
+            help="Creates an embedded content for vector search. Costs a little money and takes a little bit more time. You can do this later if you prefer.",
+        )
+        if not run_embed:
+            st.caption("You can always embed later by clicking on the source.")
+    elif content_settings.default_embedding_option == "always":
+        st.caption("Embedding content for vector search automatically")
+        run_embed = True
+    else:
+        st.caption(
+            "Not embedding content for vector search as per settings. You can always embed later by clicking on the source."
+        )
+        run_embed = False
+
     if st.button("Process", key="add_source"):
         logger.debug("Adding source")
         with st.status("Processing...", expanded=True):