organize data folders

Dockerfile

@@ -22,7 +22,6 @@ WORKDIR /app
 
 EXPOSE 8502
 
-RUN mkdir -p /app/sqlite-db
 RUN mkdir -p /app/data
 
 CMD ["poetry", "run", "streamlit", "run", "app_home.py"]

open_notebook/config.py

@@ -15,3 +15,20 @@ except Exception:
     logger.critical("Config file not found, using empty defaults")
     logger.debug(f"Looked in {config_path}")
     CONFIG = {}
+
+# ROOT DATA FOLDER
+# todo: make this configurable once podcastfy supports it
+DATA_FOLDER = "./data"
+
+# LANGGRAPH CHECKPOINT FILE
+sqlite_folder = f"{DATA_FOLDER}/sqlite-db"
+os.makedirs(sqlite_folder, exist_ok=True)
+LANGGRAPH_CHECKPOINT_FILE = f"{sqlite_folder}/checkpoints.sqlite"
+
+# UPLOADS FOLDER
+UPLOADS_FOLDER = f"{DATA_FOLDER}/uploads"
+os.makedirs(UPLOADS_FOLDER, exist_ok=True)
+
+# PODCASTS FOLDER
+PODCASTS_FOLDER = f"{DATA_FOLDER}/podcasts"
+os.makedirs(PODCASTS_FOLDER, exist_ok=True)

open_notebook/graphs/chat.py

@@ -1,4 +1,3 @@
-import os
 import sqlite3
 from typing import Annotated, Optional
 
@@ -10,6 +9,7 @@ from langgraph.graph import END, START, StateGraph
 from langgraph.graph.message import add_messages
 from typing_extensions import TypedDict
 
+from open_notebook.config import LANGGRAPH_CHECKPOINT_FILE
 from open_notebook.domain import Notebook
 from open_notebook.graphs.utils import run_pattern
 
@@ -33,7 +33,7 @@ def call_model_with_messages(state: ThreadState, config: RunnableConfig) -> dict
 
 
 conn = sqlite3.connect(
-    os.environ.get("CHECKPOINT_DATA_PATH", "sqlite-db/checkpoints.sqlite"),
+    LANGGRAPH_CHECKPOINT_FILE,
     check_same_thread=False,
 )
 memory = SqliteSaver(conn)

stream_app/source.py

@@ -1,3 +1,4 @@
+import os
 from pathlib import Path
 
 import streamlit as st
@@ -6,6 +7,7 @@ import yaml
 from humanize import naturaltime
 from loguru import logger
 
+from open_notebook.config import UPLOADS_FOLDER
 from open_notebook.domain import Asset, Source
 from open_notebook.graphs.content_process import graph
 from open_notebook.graphs.multipattern import graph as transform_graph
@@ -13,9 +15,6 @@ from open_notebook.utils import surreal_clean
 
 from .consts import context_icons
 
-uploads_dir = Path("./.uploads")
-uploads_dir.mkdir(parents=True, exist_ok=True)
-
 
 def run_transformations(input_text, transformations):
     output = transform_graph.invoke(
@@ -121,10 +120,10 @@ def add_source(session_id):
             # Generate a unique file name
             base_name = Path(file_name).stem
             counter = 1
-            new_path = uploads_dir / file_name
+            new_path = os.path.join(UPLOADS_FOLDER, file_name)
-            while new_path.exists():
+            while os.path.exists(new_path):
                 new_file_name = f"{base_name}_{counter}{file_extension}"
-                new_path = uploads_dir / new_file_name
+                new_path = os.path.join(UPLOADS_FOLDER, new_file_name)
                 counter += 1
 
             req["file_path"] = str(new_path)