add transformation to domain

pages/7_⚙️_Settings.py

@@ -3,6 +3,7 @@ import os
 import streamlit as st
 
 from open_notebook.domain.models import DefaultModels, Model, model_manager
+from open_notebook.domain.transformation import DefaultTransformations, Transformation
 from open_notebook.models import MODEL_CLASS_MAP
 from pages.stream_app.utils import setup_page
 
@@ -11,7 +12,9 @@ setup_page("⚙️ Settings")
 
 st.title("⚙️ Settings")
 
-model_tab, model_defaults_tab = st.tabs(["Models", "Model Defaults"])
+model_tab, model_defaults_tab, transformations_tab = st.tabs(
+    ["Models", "Model Defaults", "Transformations"]
+)
 
 provider_status = {}
 
@@ -231,3 +234,28 @@ with model_defaults_tab:
             defs[k] = v.id
     DefaultModels().update(defs)
     model_manager.refresh_defaults()
+
+with transformations_tab:
+    transformations = Transformation.get_all()
+    default_transformations = DefaultTransformations()
+    st.markdown("Please, select which transformations to apply by default on sources")
+    selected_transformations = {}
+    for transformation in transformations["source_insights"]:
+        with st.container(border=True):
+            selected_transformations[transformation["name"]] = st.checkbox(
+                f"**{transformation['name']}**",
+                value=(
+                    transformation["name"] in default_transformations.source_insights
+                ),
+            )
+            st.write(transformation["description"])
+            p = ["- " + pattern for pattern in transformation["patterns"]]
+            st.markdown("\n".join(p))
+    if st.button("Save Defaults", key="save_transformations"):
+        default_transformations.source_insights = [
+            transformation
+            for transformation, selected in selected_transformations.items()
+            if selected
+        ]
+        default_transformations.update(default_transformations.model_dump())
+        st.toast("Default Transformations saved successfully")

pages/components/source_panel.py

@@ -1,9 +1,9 @@
 import streamlit as st
 import streamlit_scrollable_textbox as stx  # type: ignore
-import yaml
 from humanize import naturaltime
 
 from open_notebook.domain.notebook import Source
+from open_notebook.domain.transformation import Transformation
 from open_notebook.utils import surreal_clean
 from pages.stream_app.utils import run_patterns
 
@@ -43,19 +43,16 @@ def source_panel(source_id: str, modal=False):
                         st.rerun(scope="fragment" if modal else "app")
 
         with c2:
-            with open("transformations.yaml", "r") as file:
+            transformations = Transformation.get_all()
-                transformations = yaml.safe_load(file)
+            for transformation in transformations["source_insights"]:
-                for transformation in transformations["source_insights"]:
+                if st.button(
-                    if st.button(
+                    transformation["name"], help=transformation["description"]
-                        transformation["name"], help=transformation["description"]
+                ):
-                    ):
+                    result = run_patterns(source.full_text, transformation["patterns"])
-                        result = run_patterns(
+                    source.add_insight(
-                            source.full_text, transformation["patterns"]
+                        transformation["insight_type"], surreal_clean(result)
-                        )
+                    )
-                        source.add_insight(
+                    st.rerun(scope="fragment" if modal else "app")
-                            transformation["insight_type"], surreal_clean(result)
-                        )
-                        st.rerun(scope="fragment" if modal else "app")
 
             if st.button(
                 "Embed vectors",