diff --git a/api/episode_profiles_service.py b/api/episode_profiles_service.py index 420690e..22deb31 100644 --- a/api/episode_profiles_service.py +++ b/api/episode_profiles_service.py @@ -7,7 +7,7 @@ from typing import List from loguru import logger from api.client import api_client -from open_notebook.domain.podcast import EpisodeProfile +from open_notebook.podcasts.models import EpisodeProfile class EpisodeProfilesService: diff --git a/api/main.py b/api/main.py index f2445de..d4746b8 100644 --- a/api/main.py +++ b/api/main.py @@ -1,5 +1,6 @@ # Load environment variables from dotenv import load_dotenv + load_dotenv() from contextlib import asynccontextmanager diff --git a/api/models_service.py b/api/models_service.py index 8196c61..956c8fd 100644 --- a/api/models_service.py +++ b/api/models_service.py @@ -7,7 +7,7 @@ from typing import List, Optional from loguru import logger from api.client import api_client -from open_notebook.domain.models import DefaultModels, Model +from open_notebook.ai.models import DefaultModels, Model class ModelsService: diff --git a/api/podcast_service.py b/api/podcast_service.py index 8bee41e..e7c6dc2 100644 --- a/api/podcast_service.py +++ b/api/podcast_service.py @@ -6,7 +6,7 @@ from pydantic import BaseModel from surreal_commands import get_command_status, submit_command from open_notebook.domain.notebook import Notebook -from open_notebook.domain.podcast import EpisodeProfile, PodcastEpisode, SpeakerProfile +from open_notebook.podcasts.models import EpisodeProfile, PodcastEpisode, SpeakerProfile class PodcastGenerationRequest(BaseModel): diff --git a/api/routers/embedding.py b/api/routers/embedding.py index 01613ae..40b70d9 100644 --- a/api/routers/embedding.py +++ b/api/routers/embedding.py @@ -3,7 +3,7 @@ from loguru import logger from api.command_service import CommandService from api.models import EmbedRequest, EmbedResponse -from open_notebook.domain.models import model_manager +from open_notebook.ai.models import model_manager from open_notebook.domain.notebook import Note, Source router = APIRouter() diff --git a/api/routers/episode_profiles.py b/api/routers/episode_profiles.py index 076723a..e35aa4e 100644 --- a/api/routers/episode_profiles.py +++ b/api/routers/episode_profiles.py @@ -4,7 +4,7 @@ from fastapi import APIRouter, HTTPException from loguru import logger from pydantic import BaseModel, Field -from open_notebook.domain.podcast import EpisodeProfile +from open_notebook.podcasts.models import EpisodeProfile router = APIRouter() diff --git a/api/routers/models.py b/api/routers/models.py index 261a4ba..7b2b31c 100644 --- a/api/routers/models.py +++ b/api/routers/models.py @@ -11,7 +11,7 @@ from api.models import ( ModelResponse, ProviderAvailabilityResponse, ) -from open_notebook.domain.models import DefaultModels, Model +from open_notebook.ai.models import DefaultModels, Model from open_notebook.exceptions import InvalidInputError router = APIRouter() diff --git a/api/routers/search.py b/api/routers/search.py index e6059ab..1c817be 100644 --- a/api/routers/search.py +++ b/api/routers/search.py @@ -6,7 +6,7 @@ from fastapi.responses import StreamingResponse from loguru import logger from api.models import AskRequest, AskResponse, SearchRequest, SearchResponse -from open_notebook.domain.models import Model, model_manager +from open_notebook.ai.models import Model, model_manager from open_notebook.domain.notebook import text_search, vector_search from open_notebook.exceptions import DatabaseOperationError, InvalidInputError from open_notebook.graphs.ask import graph as ask_graph diff --git a/api/routers/speaker_profiles.py b/api/routers/speaker_profiles.py index 3e3366d..e8611fd 100644 --- a/api/routers/speaker_profiles.py +++ b/api/routers/speaker_profiles.py @@ -4,7 +4,7 @@ from fastapi import APIRouter, HTTPException from loguru import logger from pydantic import BaseModel, Field -from open_notebook.domain.podcast import SpeakerProfile +from open_notebook.podcasts.models import SpeakerProfile router = APIRouter() diff --git a/api/routers/transformations.py b/api/routers/transformations.py index 7242a30..6e3d455 100644 --- a/api/routers/transformations.py +++ b/api/routers/transformations.py @@ -12,7 +12,7 @@ from api.models import ( TransformationResponse, TransformationUpdate, ) -from open_notebook.domain.models import Model +from open_notebook.ai.models import Model from open_notebook.domain.transformation import DefaultPrompts, Transformation from open_notebook.exceptions import InvalidInputError from open_notebook.graphs.transformation import graph as transformation_graph diff --git a/commands/embedding_commands.py b/commands/embedding_commands.py index 6e0445f..ec0fb78 100644 --- a/commands/embedding_commands.py +++ b/commands/embedding_commands.py @@ -5,8 +5,8 @@ from loguru import logger from pydantic import BaseModel from surreal_commands import CommandInput, CommandOutput, command, submit_command +from open_notebook.ai.models import model_manager from open_notebook.database.repository import ensure_record_id, repo_query -from open_notebook.domain.models import model_manager from open_notebook.domain.notebook import Note, Source, SourceInsight from open_notebook.utils.text_utils import split_text diff --git a/commands/podcast_commands.py b/commands/podcast_commands.py index 1382abb..2021f61 100644 --- a/commands/podcast_commands.py +++ b/commands/podcast_commands.py @@ -8,7 +8,7 @@ from surreal_commands import CommandInput, CommandOutput, command from open_notebook.config import DATA_FOLDER from open_notebook.database.repository import ensure_record_id, repo_query -from open_notebook.domain.podcast import EpisodeProfile, PodcastEpisode, SpeakerProfile +from open_notebook.podcasts.models import EpisodeProfile, PodcastEpisode, SpeakerProfile try: from podcast_creator import configure, create_podcast diff --git a/open_notebook/ai/__init__.py b/open_notebook/ai/__init__.py new file mode 100644 index 0000000..617d637 --- /dev/null +++ b/open_notebook/ai/__init__.py @@ -0,0 +1,2 @@ +# AI infrastructure module +# Contains model configuration, provisioning, and management diff --git a/open_notebook/domain/models.py b/open_notebook/ai/models.py similarity index 100% rename from open_notebook/domain/models.py rename to open_notebook/ai/models.py diff --git a/open_notebook/graphs/utils.py b/open_notebook/ai/provision.py similarity index 95% rename from open_notebook/graphs/utils.py rename to open_notebook/ai/provision.py index 05264e2..3548b82 100644 --- a/open_notebook/graphs/utils.py +++ b/open_notebook/ai/provision.py @@ -2,7 +2,7 @@ from esperanto import LanguageModel from langchain_core.language_models.chat_models import BaseChatModel from loguru import logger -from open_notebook.domain.models import model_manager +from open_notebook.ai.models import model_manager from open_notebook.utils import token_count diff --git a/open_notebook/database/async_migrate.py b/open_notebook/database/async_migrate.py index f305758..da333c0 100644 --- a/open_notebook/database/async_migrate.py +++ b/open_notebook/database/async_migrate.py @@ -96,26 +96,26 @@ class AsyncMigrationManager: def __init__(self): """Initialize migration manager.""" self.up_migrations = [ - AsyncMigration.from_file("migrations/1.surrealql"), - AsyncMigration.from_file("migrations/2.surrealql"), - AsyncMigration.from_file("migrations/3.surrealql"), - AsyncMigration.from_file("migrations/4.surrealql"), - AsyncMigration.from_file("migrations/5.surrealql"), - AsyncMigration.from_file("migrations/6.surrealql"), - AsyncMigration.from_file("migrations/7.surrealql"), - AsyncMigration.from_file("migrations/8.surrealql"), - AsyncMigration.from_file("migrations/9.surrealql"), + AsyncMigration.from_file("open_notebook/database/migrations/1.surrealql"), + AsyncMigration.from_file("open_notebook/database/migrations/2.surrealql"), + AsyncMigration.from_file("open_notebook/database/migrations/3.surrealql"), + AsyncMigration.from_file("open_notebook/database/migrations/4.surrealql"), + AsyncMigration.from_file("open_notebook/database/migrations/5.surrealql"), + AsyncMigration.from_file("open_notebook/database/migrations/6.surrealql"), + AsyncMigration.from_file("open_notebook/database/migrations/7.surrealql"), + AsyncMigration.from_file("open_notebook/database/migrations/8.surrealql"), + AsyncMigration.from_file("open_notebook/database/migrations/9.surrealql"), ] self.down_migrations = [ - AsyncMigration.from_file("migrations/1_down.surrealql"), - AsyncMigration.from_file("migrations/2_down.surrealql"), - AsyncMigration.from_file("migrations/3_down.surrealql"), - AsyncMigration.from_file("migrations/4_down.surrealql"), - AsyncMigration.from_file("migrations/5_down.surrealql"), - AsyncMigration.from_file("migrations/6_down.surrealql"), - AsyncMigration.from_file("migrations/7_down.surrealql"), - AsyncMigration.from_file("migrations/8_down.surrealql"), - AsyncMigration.from_file("migrations/9_down.surrealql"), + AsyncMigration.from_file("open_notebook/database/migrations/1_down.surrealql"), + AsyncMigration.from_file("open_notebook/database/migrations/2_down.surrealql"), + AsyncMigration.from_file("open_notebook/database/migrations/3_down.surrealql"), + AsyncMigration.from_file("open_notebook/database/migrations/4_down.surrealql"), + AsyncMigration.from_file("open_notebook/database/migrations/5_down.surrealql"), + AsyncMigration.from_file("open_notebook/database/migrations/6_down.surrealql"), + AsyncMigration.from_file("open_notebook/database/migrations/7_down.surrealql"), + AsyncMigration.from_file("open_notebook/database/migrations/8_down.surrealql"), + AsyncMigration.from_file("open_notebook/database/migrations/9_down.surrealql"), ] self.runner = AsyncMigrationRunner( up_migrations=self.up_migrations, diff --git a/migrations/1.surrealql b/open_notebook/database/migrations/1.surrealql similarity index 100% rename from migrations/1.surrealql rename to open_notebook/database/migrations/1.surrealql diff --git a/migrations/1_down.surrealql b/open_notebook/database/migrations/1_down.surrealql similarity index 100% rename from migrations/1_down.surrealql rename to open_notebook/database/migrations/1_down.surrealql diff --git a/migrations/2.surrealql b/open_notebook/database/migrations/2.surrealql similarity index 100% rename from migrations/2.surrealql rename to open_notebook/database/migrations/2.surrealql diff --git a/migrations/2_down.surrealql b/open_notebook/database/migrations/2_down.surrealql similarity index 100% rename from migrations/2_down.surrealql rename to open_notebook/database/migrations/2_down.surrealql diff --git a/migrations/3.surrealql b/open_notebook/database/migrations/3.surrealql similarity index 100% rename from migrations/3.surrealql rename to open_notebook/database/migrations/3.surrealql diff --git a/migrations/3_down.surrealql b/open_notebook/database/migrations/3_down.surrealql similarity index 100% rename from migrations/3_down.surrealql rename to open_notebook/database/migrations/3_down.surrealql diff --git a/migrations/4.surrealql b/open_notebook/database/migrations/4.surrealql similarity index 100% rename from migrations/4.surrealql rename to open_notebook/database/migrations/4.surrealql diff --git a/migrations/4_down.surrealql b/open_notebook/database/migrations/4_down.surrealql similarity index 100% rename from migrations/4_down.surrealql rename to open_notebook/database/migrations/4_down.surrealql diff --git a/migrations/5.surrealql b/open_notebook/database/migrations/5.surrealql similarity index 100% rename from migrations/5.surrealql rename to open_notebook/database/migrations/5.surrealql diff --git a/migrations/5_down.surrealql b/open_notebook/database/migrations/5_down.surrealql similarity index 100% rename from migrations/5_down.surrealql rename to open_notebook/database/migrations/5_down.surrealql diff --git a/migrations/6.surrealql b/open_notebook/database/migrations/6.surrealql similarity index 100% rename from migrations/6.surrealql rename to open_notebook/database/migrations/6.surrealql diff --git a/migrations/6_down.surrealql b/open_notebook/database/migrations/6_down.surrealql similarity index 100% rename from migrations/6_down.surrealql rename to open_notebook/database/migrations/6_down.surrealql diff --git a/migrations/7.surrealql b/open_notebook/database/migrations/7.surrealql similarity index 100% rename from migrations/7.surrealql rename to open_notebook/database/migrations/7.surrealql diff --git a/migrations/7_down.surrealql b/open_notebook/database/migrations/7_down.surrealql similarity index 100% rename from migrations/7_down.surrealql rename to open_notebook/database/migrations/7_down.surrealql diff --git a/migrations/8.surrealql b/open_notebook/database/migrations/8.surrealql similarity index 100% rename from migrations/8.surrealql rename to open_notebook/database/migrations/8.surrealql diff --git a/migrations/8_down.surrealql b/open_notebook/database/migrations/8_down.surrealql similarity index 100% rename from migrations/8_down.surrealql rename to open_notebook/database/migrations/8_down.surrealql diff --git a/migrations/9.surrealql b/open_notebook/database/migrations/9.surrealql similarity index 100% rename from migrations/9.surrealql rename to open_notebook/database/migrations/9.surrealql diff --git a/migrations/9_down.surrealql b/open_notebook/database/migrations/9_down.surrealql similarity index 100% rename from migrations/9_down.surrealql rename to open_notebook/database/migrations/9_down.surrealql diff --git a/open_notebook/domain/base.py b/open_notebook/domain/base.py index f9992e2..cf9dfdb 100644 --- a/open_notebook/domain/base.py +++ b/open_notebook/domain/base.py @@ -111,7 +111,7 @@ class ObjectModel(BaseModel): return None async def save(self) -> None: - from open_notebook.domain.models import model_manager + from open_notebook.ai.models import model_manager try: self.model_validate(self.model_dump(), strict=True) diff --git a/open_notebook/domain/notebook.py b/open_notebook/domain/notebook.py index 2f589a6..fd03d5b 100644 --- a/open_notebook/domain/notebook.py +++ b/open_notebook/domain/notebook.py @@ -6,9 +6,9 @@ from pydantic import BaseModel, Field, field_validator from surreal_commands import submit_command from surrealdb import RecordID +from open_notebook.ai.models import model_manager from open_notebook.database.repository import ensure_record_id, repo_query from open_notebook.domain.base import ObjectModel -from open_notebook.domain.models import model_manager from open_notebook.exceptions import DatabaseOperationError, InvalidInputError from open_notebook.utils import split_text diff --git a/open_notebook/graphs/ask.py b/open_notebook/graphs/ask.py index 8a0846b..bb11d12 100644 --- a/open_notebook/graphs/ask.py +++ b/open_notebook/graphs/ask.py @@ -9,8 +9,8 @@ from langgraph.types import Send from pydantic import BaseModel, Field from typing_extensions import TypedDict +from open_notebook.ai.provision import provision_langchain_model from open_notebook.domain.notebook import vector_search -from open_notebook.graphs.utils import provision_langchain_model from open_notebook.utils import clean_thinking_content diff --git a/open_notebook/graphs/chat.py b/open_notebook/graphs/chat.py index 442396a..408e019 100644 --- a/open_notebook/graphs/chat.py +++ b/open_notebook/graphs/chat.py @@ -5,16 +5,15 @@ from typing import Annotated, Optional from ai_prompter import Prompter from langchain_core.messages import AIMessage, SystemMessage from langchain_core.runnables import RunnableConfig - -from open_notebook.utils import clean_thinking_content from langgraph.checkpoint.sqlite import SqliteSaver from langgraph.graph import END, START, StateGraph from langgraph.graph.message import add_messages from typing_extensions import TypedDict +from open_notebook.ai.provision import provision_langchain_model from open_notebook.config import LANGGRAPH_CHECKPOINT_FILE from open_notebook.domain.notebook import Notebook -from open_notebook.graphs.utils import provision_langchain_model +from open_notebook.utils import clean_thinking_content class ThreadState(TypedDict): @@ -26,7 +25,7 @@ class ThreadState(TypedDict): def call_model_with_messages(state: ThreadState, config: RunnableConfig) -> dict: - system_prompt = Prompter(prompt_template="chat").render(data=state) # type: ignore[arg-type] + system_prompt = Prompter(prompt_template="chat/system").render(data=state) # type: ignore[arg-type] payload = [SystemMessage(content=system_prompt)] + state.get("messages", []) model_id = config.get("configurable", {}).get("model_id") or state.get( "model_override" diff --git a/open_notebook/graphs/prompt.py b/open_notebook/graphs/prompt.py index 176fdbf..b454904 100644 --- a/open_notebook/graphs/prompt.py +++ b/open_notebook/graphs/prompt.py @@ -6,7 +6,7 @@ from langchain_core.runnables import RunnableConfig from langgraph.graph import END, START, StateGraph from typing_extensions import TypedDict -from open_notebook.graphs.utils import provision_langchain_model +from open_notebook.ai.provision import provision_langchain_model class PatternChainState(TypedDict): diff --git a/open_notebook/graphs/source.py b/open_notebook/graphs/source.py index da0b3d6..68e16f5 100644 --- a/open_notebook/graphs/source.py +++ b/open_notebook/graphs/source.py @@ -9,8 +9,8 @@ from langgraph.types import Send from loguru import logger from typing_extensions import Annotated, TypedDict +from open_notebook.ai.models import Model, ModelManager from open_notebook.domain.content_settings import ContentSettings -from open_notebook.domain.models import Model, ModelManager from open_notebook.domain.notebook import Asset, Source from open_notebook.domain.transformation import Transformation from open_notebook.graphs.transformation import graph as transform_graph diff --git a/open_notebook/graphs/source_chat.py b/open_notebook/graphs/source_chat.py index 99217a8..dc353bf 100644 --- a/open_notebook/graphs/source_chat.py +++ b/open_notebook/graphs/source_chat.py @@ -5,16 +5,15 @@ from typing import Annotated, Dict, List, Optional from ai_prompter import Prompter from langchain_core.messages import AIMessage, SystemMessage from langchain_core.runnables import RunnableConfig - -from open_notebook.utils import clean_thinking_content from langgraph.checkpoint.sqlite import SqliteSaver from langgraph.graph import END, START, StateGraph from langgraph.graph.message import add_messages from typing_extensions import TypedDict +from open_notebook.ai.provision import provision_langchain_model from open_notebook.config import LANGGRAPH_CHECKPOINT_FILE from open_notebook.domain.notebook import Source, SourceInsight -from open_notebook.graphs.utils import provision_langchain_model +from open_notebook.utils import clean_thinking_content from open_notebook.utils.context_builder import ContextBuilder @@ -111,7 +110,7 @@ def call_model_with_source_context( } # Apply the source_chat prompt template - system_prompt = Prompter(prompt_template="source_chat").render(data=prompt_data) + system_prompt = Prompter(prompt_template="source_chat/system").render(data=prompt_data) payload = [SystemMessage(content=system_prompt)] + state.get("messages", []) # Handle async model provisioning from sync context diff --git a/open_notebook/graphs/transformation.py b/open_notebook/graphs/transformation.py index 8c86237..cb4906d 100644 --- a/open_notebook/graphs/transformation.py +++ b/open_notebook/graphs/transformation.py @@ -4,9 +4,9 @@ from langchain_core.runnables import RunnableConfig from langgraph.graph import END, START, StateGraph from typing_extensions import TypedDict +from open_notebook.ai.provision import provision_langchain_model from open_notebook.domain.notebook import Source from open_notebook.domain.transformation import DefaultPrompts, Transformation -from open_notebook.graphs.utils import provision_langchain_model from open_notebook.utils import clean_thinking_content diff --git a/open_notebook/podcasts/__init__.py b/open_notebook/podcasts/__init__.py new file mode 100644 index 0000000..526ebe2 --- /dev/null +++ b/open_notebook/podcasts/__init__.py @@ -0,0 +1,2 @@ +# Podcasts module +# Contains podcast episode models, profiles, and generation logic diff --git a/open_notebook/domain/podcast.py b/open_notebook/podcasts/models.py similarity index 100% rename from open_notebook/domain/podcast.py rename to open_notebook/podcasts/models.py diff --git a/prompts/chat.jinja b/prompts/chat/system.jinja similarity index 100% rename from prompts/chat.jinja rename to prompts/chat/system.jinja diff --git a/prompts/source_chat.jinja b/prompts/source_chat/system.jinja similarity index 100% rename from prompts/source_chat.jinja rename to prompts/source_chat/system.jinja diff --git a/tests/test_domain.py b/tests/test_domain.py index 0c10ab4..3f57ae2 100644 --- a/tests/test_domain.py +++ b/tests/test_domain.py @@ -8,13 +8,13 @@ that can be tested without database mocking. import pytest from pydantic import ValidationError +from open_notebook.ai.models import ModelManager from open_notebook.domain.base import RecordModel from open_notebook.domain.content_settings import ContentSettings -from open_notebook.domain.models import ModelManager from open_notebook.domain.notebook import Note, Notebook, Source -from open_notebook.domain.podcast import EpisodeProfile, SpeakerProfile from open_notebook.domain.transformation import Transformation from open_notebook.exceptions import InvalidInputError +from open_notebook.podcasts.models import EpisodeProfile, SpeakerProfile # ============================================================================ # TEST SUITE 1: RecordModel Singleton Pattern diff --git a/tests/test_models_api.py b/tests/test_models_api.py index 08c4938..50fd229 100644 --- a/tests/test_models_api.py +++ b/tests/test_models_api.py @@ -60,7 +60,7 @@ class TestModelCreation: @patch("open_notebook.database.repository.repo_query") async def test_create_same_model_name_different_provider(self, mock_repo_query, client): """Test that creating a model with same name but different provider is allowed.""" - from open_notebook.domain.models import Model + from open_notebook.ai.models import Model # Mock repo_query to return empty (no duplicate found for different provider) mock_repo_query.return_value = []