151 lines
5.3 KiB
Python
151 lines
5.3 KiB
Python
import os
|
|
from pathlib import Path
|
|
|
|
import streamlit as st
|
|
from humanize import naturaltime
|
|
from loguru import logger
|
|
|
|
from open_notebook.config import UPLOADS_FOLDER
|
|
from open_notebook.domain.notebook import Asset, Source
|
|
from open_notebook.exceptions import UnsupportedTypeException
|
|
from open_notebook.graphs.content_processing import graph
|
|
from open_notebook.utils import surreal_clean
|
|
from pages.components import source_panel
|
|
from pages.stream_app.utils import run_patterns
|
|
|
|
from .consts import context_icons
|
|
|
|
|
|
# moved it here to replace it with the pipeline on 0.1.0
|
|
def generate_toc_and_title(source) -> "Source":
|
|
try:
|
|
patterns = ["patterns/default/toc"]
|
|
result = run_patterns(source.full_text, patterns=patterns)
|
|
source.add_insight("Table of Contents", surreal_clean(result))
|
|
if not source.title:
|
|
patterns = [
|
|
"Based on the Table of Contents below, please provide a Title for this content, with max 15 words"
|
|
]
|
|
output = run_patterns(result, patterns=patterns)
|
|
source.title = surreal_clean(output)
|
|
source.save()
|
|
return source
|
|
except Exception as e:
|
|
logger.error(f"Error summarizing source {source.id}: {str(e)}")
|
|
logger.exception(e)
|
|
raise
|
|
|
|
|
|
@st.dialog("Source", width="large")
|
|
def source_panel_dialog(source_id):
|
|
source_panel(source_id)
|
|
|
|
|
|
@st.dialog("Add a Source", width="large")
|
|
def add_source(notebook_id):
|
|
source_link = None
|
|
source_file = None
|
|
source_text = None
|
|
source_type = st.radio("Type", ["Link", "Upload", "Text"])
|
|
req = {}
|
|
if source_type == "Link":
|
|
source_link = st.text_input("Link")
|
|
req["url"] = source_link
|
|
elif source_type == "Upload":
|
|
source_file = st.file_uploader("Upload")
|
|
req["delete_source"] = st.checkbox("Delete source after processing", value=True)
|
|
|
|
else:
|
|
source_text = st.text_area("Text")
|
|
req["content"] = source_text
|
|
if st.button("Process", key="add_source"):
|
|
logger.debug("Adding source")
|
|
with st.status("Processing...", expanded=True):
|
|
st.write("Processing document...")
|
|
try:
|
|
if source_type == "Upload" and source_file is not None:
|
|
st.write("Uploading..")
|
|
file_name = source_file.name
|
|
file_extension = Path(file_name).suffix
|
|
base_name = Path(file_name).stem
|
|
|
|
# Generate unique filename
|
|
new_path = os.path.join(UPLOADS_FOLDER, file_name)
|
|
counter = 0
|
|
while os.path.exists(new_path):
|
|
counter += 1
|
|
new_file_name = f"{base_name}_{counter}{file_extension}"
|
|
new_path = os.path.join(UPLOADS_FOLDER, new_file_name)
|
|
|
|
req["file_path"] = str(new_path)
|
|
# Save the file
|
|
with open(new_path, "wb") as f:
|
|
f.write(source_file.getbuffer())
|
|
|
|
result = graph.invoke(req)
|
|
st.write("Saving..")
|
|
source = Source(
|
|
asset=Asset(url=req.get("url"), file_path=req.get("file_path")),
|
|
full_text=surreal_clean(result["content"]),
|
|
title=result.get("title"),
|
|
)
|
|
source.save()
|
|
source.add_to_notebook(notebook_id)
|
|
st.write("Summarizing...")
|
|
generate_toc_and_title(source)
|
|
except UnsupportedTypeException as e:
|
|
st.warning(
|
|
"This type of content is not supported yet. If you think it should be, let us know on the project Issues's page"
|
|
)
|
|
st.error(e)
|
|
st.link_button(
|
|
"Go to Github Issues",
|
|
url="https://www.github.com/lfnovo/open-notebook/issues",
|
|
)
|
|
st.stop()
|
|
|
|
except Exception as e:
|
|
st.exception(e)
|
|
return
|
|
|
|
st.rerun()
|
|
|
|
|
|
def source_card(source, notebook_id):
|
|
# todo: more descriptive icons
|
|
icon = "🔗"
|
|
|
|
with st.container(border=True):
|
|
title = (source.title if source.title else "No Title").strip()
|
|
st.markdown((f"{icon}**{title}**"))
|
|
context_state = st.selectbox(
|
|
"Context",
|
|
label_visibility="collapsed",
|
|
options=context_icons,
|
|
index=0,
|
|
key=f"source_{source.id}",
|
|
)
|
|
st.caption(
|
|
f"Updated: {naturaltime(source.updated)}, **{len(source.insights)}** insights"
|
|
)
|
|
if st.button("Expand", icon="📝", key=source.id):
|
|
source_panel_dialog(source.id)
|
|
|
|
st.session_state[notebook_id]["context_config"][source.id] = context_state
|
|
|
|
|
|
def source_list_item(source_id, score=None):
|
|
source: Source = Source.get(source_id)
|
|
if not source:
|
|
st.error("Source not found")
|
|
return
|
|
icon = "🔗"
|
|
|
|
with st.expander(
|
|
f"{icon} [{score:.2f}] **{source.title}** {naturaltime(source.updated)}"
|
|
):
|
|
for insight in source.insights:
|
|
st.markdown(f"**{insight.insight_type}**")
|
|
st.write(insight.content)
|
|
if st.button("Edit source", icon="📝", key=f"x_edit_source_{source.id}"):
|
|
source_panel_dialog(source_id=source.id)
|