From b7e656a31928f0543dda2feff13b333679e537fd Mon Sep 17 00:00:00 2001
From: Luis Novo
Date: Sat, 18 Oct 2025 12:46:22 -0300
Subject: [PATCH] Version 1 (#160)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
New front-end
Launch Chat API
Manage Sources
Enable re-embedding of all contents
Sources can be added without a notebook now
Improved settings
Enable model selector on all chats
Background processing for better experience
Dark mode
Improved Notes
Improved Docs:
- Remove all Streamlit references from documentation
- Update deployment guides with React frontend setup
- Fix Docker environment variables format (SURREAL_URL, SURREAL_PASSWORD)
- Update docker image tag from :latest to :v1-latest
- Change navigation references (Settings → Models to just Models)
- Update development setup to include frontend npm commands
- Add MIGRATION.md guide for users upgrading from Streamlit
- Update quick-start guide with correct environment variables
- Add port 5055 documentation for API access
- Update project structure to reflect frontend/ directory
- Remove outdated source-chat documentation files
---
.claude/CLAUDE.md | 19 +
.claude/commands/quality.md | 4 +
.../api_troubleshoot/migration_plan.md | 319 -
.../migrate_surrealdb/architecture.md | 358 -
.claude/sessions/migrate_surrealdb/context.md | 110 -
.claude/sessions/migrate_surrealdb/plan.md | 898 --
.../migrate_surrealdb/requirements.txt | 15 -
.claude/sessions/oss-136/architecture.md | 454 -
.claude/sessions/oss-136/context.md | 133 -
.claude/sessions/oss-136/plan.md | 1795 ---
.claude/sessions/oss-136/test.md | 5 -
.claude/sessions/podcast_page/architecture.md | 321 -
.claude/sessions/podcast_page/context.md | 74 -
.claude/sessions/podcast_page/plan.md | 398 -
.../sessions/podcast_page/requirements.txt | 56 -
.dockerignore | 5 +
.env.example | 6 +
.github/workflows/build-and-release.yml | 10 +-
.github/workflows/build-dev.yml | 4 +-
.gitignore | 15 +-
CONFIGURATION.md | 101 +
Dockerfile | 26 +-
Dockerfile.single | 24 +-
MIGRATION.md | 394 +
Makefile | 47 +-
README.md | 43 +-
api/auth.py | 8 +-
api/chat_service.py | 172 +
api/client.py | 163 +-
api/command_service.py | 6 +-
api/context_service.py | 8 +-
api/embedding_service.py | 8 +-
api/episode_profiles_service.py | 6 +-
api/insights_service.py | 14 +-
api/main.py | 68 +-
api/models.py | 206 +-
api/models_service.py | 21 +-
api/notebook_service.py | 13 +-
api/notes_service.py | 15 +-
api/podcast_api_service.py | 12 +-
api/podcast_service.py | 4 +-
api/routers/auth.py | 24 +
api/routers/chat.py | 493 +
api/routers/commands.py | 10 +-
api/routers/config.py | 176 +
api/routers/context.py | 8 +-
api/routers/embedding.py | 75 +-
api/routers/embedding_rebuild.py | 190 +
api/routers/episode_profiles.py | 4 +-
api/routers/insights.py | 11 +-
api/routers/models.py | 117 +-
api/routers/notebooks.py | 81 +-
api/routers/notes.py | 38 +-
api/routers/podcasts.py | 58 +-
api/routers/search.py | 21 +-
api/routers/settings.py | 39 +-
api/routers/source_chat.py | 446 +
api/routers/sources.py | 896 +-
api/routers/speaker_profiles.py | 6 +-
api/routers/transformations.py | 51 +-
api/search_service.py | 16 +-
api/settings_service.py | 17 +-
api/sources_service.py | 164 +-
api/transformations_service.py | 20 +-
app_home.py | 4 -
batch_fix_services.py | 77 +
commands/__init__.py | 5 +
commands/embedding_commands.py | 392 +
commands/example_commands.py | 26 +-
commands/podcast_commands.py | 19 -
commands/source_commands.py | 137 +
doc_outline.md | 318 -
docker-compose.dev.yml | 25 +
docker-compose.single.yml | 10 +-
docker-compose.yml | 4 +-
docs/assets/asset_list.png | Bin 64157 -> 241289 bytes
docs/deployment/development.md | 69 +-
docs/deployment/docker.md | 41 +-
docs/deployment/security.md | 16 +-
docs/deployment/single-container.md | 20 +-
docs/development/api-reference.md | 500 +-
docs/development/architecture.md | 2 +-
docs/development/contributing.md | 8 +-
docs/development/index.md | 6 +-
docs/features/ai-models.md | 40 +-
docs/features/ollama.md | 8 +-
docs/getting-started/installation.md | 26 +-
docs/getting-started/quick-start.md | 19 +-
docs/migration/streamlit-to-nextjs.md | 288 +
docs/troubleshooting/common-issues.md | 22 +-
docs/troubleshooting/debugging.md | 8 +-
docs/troubleshooting/faq.md | 8 +-
docs/troubleshooting/index.md | 2 +-
docs/user-guide/chat.md | 2 +-
docs/user-guide/interface-overview.md | 2 +-
docs/user-guide/sources.md | 2 +-
frontend/.claude/logs/post_tool_use.json | 5506 ++++++++++
frontend/.claude/logs/pre_tool_use.json | 2291 ++++
frontend/.gitignore | 41 +
frontend/components.json | 21 +
frontend/eslint.config.mjs | 16 +
frontend/next.config.ts | 8 +
frontend/package-lock.json | 9579 +++++++++++++++++
frontend/package.json | 62 +
frontend/postcss.config.mjs | 5 +
frontend/public/file.svg | 1 +
frontend/public/globe.svg | 1 +
frontend/public/logo.svg | 60 +
frontend/public/next.svg | 1 +
frontend/public/vercel.svg | 1 +
frontend/public/window.svg | 1 +
frontend/src/app/(auth)/login/page.tsx | 10 +
.../advanced/components/RebuildEmbeddings.tsx | 362 +
.../advanced/components/SystemInfo.tsx | 117 +
.../src/app/(dashboard)/advanced/page.tsx | 27 +
frontend/src/app/(dashboard)/layout.tsx | 58 +
.../models/components/AddModelForm.tsx | 129 +
.../components/DefaultModelsSection.tsx | 266 +
.../components/EmbeddingModelChangeDialog.tsx | 119 +
.../models/components/ModelTypeSection.tsx | 211 +
.../models/components/ProviderStatus.tsx | 125 +
frontend/src/app/(dashboard)/models/page.tsx | 100 +
.../app/(dashboard)/notebooks/[id]/page.tsx | 144 +
.../notebooks/components/ChatColumn.tsx | 115 +
.../notebooks/components/NoteEditorDialog.tsx | 170 +
.../notebooks/components/NotebookCard.tsx | 129 +
.../notebooks/components/NotebookHeader.tsx | 128 +
.../notebooks/components/NotebookList.tsx | 77 +
.../notebooks/components/NotesColumn.tsx | 133 +
.../notebooks/components/SourcesColumn.tsx | 171 +
.../src/app/(dashboard)/notebooks/page.tsx | 100 +
frontend/src/app/(dashboard)/page.tsx | 5 +
.../src/app/(dashboard)/podcasts/page.tsx | 56 +
frontend/src/app/(dashboard)/search/page.tsx | 425 +
.../settings/components/SettingsForm.tsx | 278 +
.../src/app/(dashboard)/settings/page.tsx | 29 +
.../src/app/(dashboard)/sources/[id]/page.tsx | 78 +
frontend/src/app/(dashboard)/sources/page.tsx | 425 +
.../components/DefaultPromptEditor.tsx | 72 +
.../components/TransformationCard.tsx | 117 +
.../components/TransformationEditorDialog.tsx | 241 +
.../components/TransformationPlayground.tsx | 133 +
.../components/TransformationsList.tsx | 87 +
.../app/(dashboard)/transformations/page.tsx | 78 +
frontend/src/app/favicon.ico | Bin 0 -> 25931 bytes
frontend/src/app/globals.css | 177 +
frontend/src/app/layout.tsx | 42 +
frontend/src/app/page.tsx | 5 +
frontend/src/components/auth/LoginForm.tsx | 185 +
frontend/src/components/common/AddButton.tsx | 102 +
.../src/components/common/ConfirmDialog.tsx | 63 +
.../src/components/common/ConnectionGuard.tsx | 94 +
.../components/common/ContextIndicator.tsx | 116 +
.../src/components/common/ContextToggle.tsx | 86 +
frontend/src/components/common/EmptyState.tsx | 19 +
.../src/components/common/ErrorBoundary.tsx | 101 +
frontend/src/components/common/InlineEdit.tsx | 139 +
.../src/components/common/LoadingSpinner.tsx | 19 +
.../src/components/common/ModelSelector.tsx | 59 +
.../src/components/common/ThemeToggle.tsx | 61 +
.../errors/ConnectionErrorOverlay.tsx | 173 +
frontend/src/components/layout/AppShell.tsx | 20 +
frontend/src/components/layout/AppSidebar.tsx | 349 +
.../notebooks/CreateNotebookDialog.tsx | 110 +
.../src/components/podcasts/EpisodeCard.tsx | 397 +
.../podcasts/EpisodeProfilesPanel.tsx | 257 +
.../src/components/podcasts/EpisodesTab.tsx | 172 +
.../podcasts/GeneratePodcastDialog.tsx | 832 ++
.../podcasts/SpeakerProfilesPanel.tsx | 236 +
.../src/components/podcasts/TemplatesTab.tsx | 151 +
.../forms/EpisodeProfileFormDialog.tsx | 450 +
.../forms/SpeakerProfileFormDialog.tsx | 393 +
.../components/providers/ModalProvider.tsx | 53 +
.../components/providers/QueryProvider.tsx | 16 +
.../components/providers/ThemeProvider.tsx | 44 +
.../search/AdvancedModelsDialog.tsx | 103 +
.../search/SaveToNotebooksDialog.tsx | 121 +
.../components/search/StreamingResponse.tsx | 173 +
frontend/src/components/source/ChatPanel.tsx | 345 +
.../src/components/source/MessageActions.tsx | 117 +
.../src/components/source/ModelSelector.tsx | 165 +
.../src/components/source/SessionManager.tsx | 252 +
.../components/source/SourceDetailContent.tsx | 744 ++
.../src/components/source/SourceDialog.tsx | 57 +
.../components/source/SourceInsightDialog.tsx | 60 +
.../components/sources/AddSourceButton.tsx | 44 +
.../components/sources/AddSourceDialog.tsx | 426 +
frontend/src/components/sources/README.md | 144 +
.../src/components/sources/SourceCard.tsx | 390 +
frontend/src/components/sources/index.ts | 3 +
.../sources/steps/NotebooksStep.tsx | 42 +
.../sources/steps/ProcessingStep.tsx | 120 +
.../sources/steps/SourceTypeStep.tsx | 161 +
frontend/src/components/ui/accordion.tsx | 82 +
frontend/src/components/ui/alert-dialog.tsx | 157 +
frontend/src/components/ui/alert.tsx | 59 +
frontend/src/components/ui/badge.tsx | 46 +
frontend/src/components/ui/button.tsx | 59 +
frontend/src/components/ui/card.tsx | 92 +
frontend/src/components/ui/checkbox-list.tsx | 85 +
frontend/src/components/ui/checkbox.tsx | 32 +
frontend/src/components/ui/collapsible.tsx | 33 +
frontend/src/components/ui/command.tsx | 184 +
frontend/src/components/ui/dialog.tsx | 143 +
frontend/src/components/ui/dropdown-menu.tsx | 257 +
frontend/src/components/ui/form-section.tsx | 37 +
frontend/src/components/ui/input.tsx | 21 +
frontend/src/components/ui/label.tsx | 24 +
.../src/components/ui/markdown-editor.tsx | 41 +
frontend/src/components/ui/popover.tsx | 48 +
frontend/src/components/ui/progress.tsx | 31 +
frontend/src/components/ui/radio-group.tsx | 45 +
frontend/src/components/ui/scroll-area.tsx | 58 +
frontend/src/components/ui/select.tsx | 185 +
frontend/src/components/ui/separator.tsx | 28 +
frontend/src/components/ui/sonner.tsx | 30 +
frontend/src/components/ui/tabs.tsx | 66 +
frontend/src/components/ui/textarea.tsx | 18 +
frontend/src/components/ui/tooltip.tsx | 61 +
.../src/components/ui/wizard-container.tsx | 104 +
frontend/src/lib/api/chat.ts | 71 +
frontend/src/lib/api/client.ts | 62 +
frontend/src/lib/api/embedding.ts | 79 +
frontend/src/lib/api/insights.ts | 34 +
frontend/src/lib/api/models.ts | 38 +
frontend/src/lib/api/notebooks.ts | 33 +
frontend/src/lib/api/notes.ts | 28 +
frontend/src/lib/api/podcasts.ts | 113 +
frontend/src/lib/api/query-client.ts | 33 +
frontend/src/lib/api/search.ts | 62 +
frontend/src/lib/api/settings.ts | 14 +
frontend/src/lib/api/source-chat.ts | 85 +
frontend/src/lib/api/sources.ts | 108 +
frontend/src/lib/api/transformations.ts | 50 +
frontend/src/lib/config.ts | 124 +
frontend/src/lib/hooks/use-ask.ts | 163 +
frontend/src/lib/hooks/use-auth.ts | 68 +
frontend/src/lib/hooks/use-insights.ts | 11 +
frontend/src/lib/hooks/use-modal-manager.ts | 46 +
frontend/src/lib/hooks/use-models.ts | 110 +
frontend/src/lib/hooks/use-navigation.ts | 13 +
frontend/src/lib/hooks/use-notebooks.ts | 91 +
frontend/src/lib/hooks/use-notes.ts | 95 +
frontend/src/lib/hooks/use-podcasts.ts | 369 +
frontend/src/lib/hooks/use-search.ts | 31 +
frontend/src/lib/hooks/use-settings.ts | 35 +
frontend/src/lib/hooks/use-sources.ts | 249 +
frontend/src/lib/hooks/use-toast.ts | 23 +
frontend/src/lib/hooks/use-transformations.ts | 147 +
frontend/src/lib/hooks/use-version-check.ts | 70 +
frontend/src/lib/hooks/useNotebookChat.ts | 291 +
frontend/src/lib/hooks/useSourceChat.ts | 243 +
frontend/src/lib/stores/auth-store.ts | 222 +
frontend/src/lib/stores/navigation-store.ts | 99 +
frontend/src/lib/stores/sidebar-store.ts | 21 +
frontend/src/lib/stores/theme-store.ts | 61 +
frontend/src/lib/theme-script.ts | 18 +
frontend/src/lib/types/api.ts | 223 +
frontend/src/lib/types/auth.ts | 10 +
frontend/src/lib/types/common.ts | 12 +
frontend/src/lib/types/config.ts | 24 +
frontend/src/lib/types/models.ts | 30 +
frontend/src/lib/types/podcasts.ts | 133 +
frontend/src/lib/types/search.ts | 56 +
frontend/src/lib/types/transformations.ts | 42 +
frontend/src/lib/utils.ts | 6 +
frontend/src/lib/utils/source-references.tsx | 238 +
frontend/src/middleware.ts | 19 +
frontend/tailwind.config.ts | 16 +
frontend/tsconfig.json | 27 +
migrations/7.surrealql | 18 +-
migrations/8.surrealql | 12 +
migrations/8_down.surrealql | 11 +
migrations/9.surrealql | 66 +
migrations/9_down.surrealql | 63 +
mypy.ini | 2 +
open_notebook/database/async_migrate.py | 4 +
open_notebook/domain/base.py | 7 +-
open_notebook/domain/models.py | 23 +-
open_notebook/domain/notebook.py | 110 +-
open_notebook/graphs/ask.py | 22 +-
open_notebook/graphs/chat.py | 51 +-
open_notebook/graphs/prompt.py | 3 +-
open_notebook/graphs/source.py | 38 +-
open_notebook/graphs/source_chat.py | 214 +
open_notebook/graphs/transformation.py | 13 +-
open_notebook/utils.py | 271 -
open_notebook/utils/README.md | 188 +
open_notebook/utils/__init__.py | 34 +
open_notebook/utils/context_builder.py | 502 +
open_notebook/utils/text_utils.py | 141 +
open_notebook/utils/token_utils.py | 38 +
open_notebook/utils/version_utils.py | 108 +
pages/10_⚙️_Settings.py | 22 +-
pages/11_🔧_Advanced.py | 262 +
pages/2_📒_Notebooks.py | 12 +-
pages/3_🔍_Ask_and_Search.py | 35 +-
pages/5_🎙️_Podcasts.py | 9 +-
pages/7_🤖_Models.py | 31 +-
pages/8_💱_Transformations.py | 18 +-
pages/components/note_panel.py | 4 +-
pages/components/source_insight.py | 8 +-
pages/components/source_panel.py | 11 +-
pages/stream_app/auth.py | 1 +
pages/stream_app/chat.py | 95 +-
pages/stream_app/note.py | 4 +-
pages/stream_app/source.py | 14 +-
pages/stream_app/utils.py | 89 +-
prompts/source_chat.jinja | 63 +
pyproject.toml | 14 +-
setup_guide/docker-compose.yml | 3 +-
specs/chat-message-actions.md | 245 +
specs/fix-notebook-chat-context-bug.md | 270 +
.../remove-insights-count-from-details-tab.md | 50 +
supervisord.conf | 6 +-
supervisord.single.conf | 6 +-
tests/test_source_chat.py | 296 +
tests/test_source_chat_api.py | 223 +
uv.lock | 2031 ++--
319 files changed, 46747 insertions(+), 7408 deletions(-)
create mode 100644 .claude/commands/quality.md
delete mode 100644 .claude/sessions/api_troubleshoot/migration_plan.md
delete mode 100644 .claude/sessions/migrate_surrealdb/architecture.md
delete mode 100644 .claude/sessions/migrate_surrealdb/context.md
delete mode 100644 .claude/sessions/migrate_surrealdb/plan.md
delete mode 100644 .claude/sessions/migrate_surrealdb/requirements.txt
delete mode 100644 .claude/sessions/oss-136/architecture.md
delete mode 100644 .claude/sessions/oss-136/context.md
delete mode 100644 .claude/sessions/oss-136/plan.md
delete mode 100644 .claude/sessions/oss-136/test.md
delete mode 100644 .claude/sessions/podcast_page/architecture.md
delete mode 100644 .claude/sessions/podcast_page/context.md
delete mode 100644 .claude/sessions/podcast_page/plan.md
delete mode 100644 .claude/sessions/podcast_page/requirements.txt
create mode 100644 CONFIGURATION.md
create mode 100644 MIGRATION.md
create mode 100644 api/chat_service.py
create mode 100644 api/routers/auth.py
create mode 100644 api/routers/chat.py
create mode 100644 api/routers/config.py
create mode 100644 api/routers/embedding_rebuild.py
create mode 100644 api/routers/source_chat.py
create mode 100644 batch_fix_services.py
create mode 100644 commands/embedding_commands.py
create mode 100644 commands/source_commands.py
delete mode 100644 doc_outline.md
create mode 100644 docker-compose.dev.yml
create mode 100644 docs/migration/streamlit-to-nextjs.md
create mode 100644 frontend/.claude/logs/post_tool_use.json
create mode 100644 frontend/.claude/logs/pre_tool_use.json
create mode 100644 frontend/.gitignore
create mode 100644 frontend/components.json
create mode 100644 frontend/eslint.config.mjs
create mode 100644 frontend/next.config.ts
create mode 100644 frontend/package-lock.json
create mode 100644 frontend/package.json
create mode 100644 frontend/postcss.config.mjs
create mode 100644 frontend/public/file.svg
create mode 100644 frontend/public/globe.svg
create mode 100644 frontend/public/logo.svg
create mode 100644 frontend/public/next.svg
create mode 100644 frontend/public/vercel.svg
create mode 100644 frontend/public/window.svg
create mode 100644 frontend/src/app/(auth)/login/page.tsx
create mode 100644 frontend/src/app/(dashboard)/advanced/components/RebuildEmbeddings.tsx
create mode 100644 frontend/src/app/(dashboard)/advanced/components/SystemInfo.tsx
create mode 100644 frontend/src/app/(dashboard)/advanced/page.tsx
create mode 100644 frontend/src/app/(dashboard)/layout.tsx
create mode 100644 frontend/src/app/(dashboard)/models/components/AddModelForm.tsx
create mode 100644 frontend/src/app/(dashboard)/models/components/DefaultModelsSection.tsx
create mode 100644 frontend/src/app/(dashboard)/models/components/EmbeddingModelChangeDialog.tsx
create mode 100644 frontend/src/app/(dashboard)/models/components/ModelTypeSection.tsx
create mode 100644 frontend/src/app/(dashboard)/models/components/ProviderStatus.tsx
create mode 100644 frontend/src/app/(dashboard)/models/page.tsx
create mode 100644 frontend/src/app/(dashboard)/notebooks/[id]/page.tsx
create mode 100644 frontend/src/app/(dashboard)/notebooks/components/ChatColumn.tsx
create mode 100644 frontend/src/app/(dashboard)/notebooks/components/NoteEditorDialog.tsx
create mode 100644 frontend/src/app/(dashboard)/notebooks/components/NotebookCard.tsx
create mode 100644 frontend/src/app/(dashboard)/notebooks/components/NotebookHeader.tsx
create mode 100644 frontend/src/app/(dashboard)/notebooks/components/NotebookList.tsx
create mode 100644 frontend/src/app/(dashboard)/notebooks/components/NotesColumn.tsx
create mode 100644 frontend/src/app/(dashboard)/notebooks/components/SourcesColumn.tsx
create mode 100644 frontend/src/app/(dashboard)/notebooks/page.tsx
create mode 100644 frontend/src/app/(dashboard)/page.tsx
create mode 100644 frontend/src/app/(dashboard)/podcasts/page.tsx
create mode 100644 frontend/src/app/(dashboard)/search/page.tsx
create mode 100644 frontend/src/app/(dashboard)/settings/components/SettingsForm.tsx
create mode 100644 frontend/src/app/(dashboard)/settings/page.tsx
create mode 100644 frontend/src/app/(dashboard)/sources/[id]/page.tsx
create mode 100644 frontend/src/app/(dashboard)/sources/page.tsx
create mode 100644 frontend/src/app/(dashboard)/transformations/components/DefaultPromptEditor.tsx
create mode 100644 frontend/src/app/(dashboard)/transformations/components/TransformationCard.tsx
create mode 100644 frontend/src/app/(dashboard)/transformations/components/TransformationEditorDialog.tsx
create mode 100644 frontend/src/app/(dashboard)/transformations/components/TransformationPlayground.tsx
create mode 100644 frontend/src/app/(dashboard)/transformations/components/TransformationsList.tsx
create mode 100644 frontend/src/app/(dashboard)/transformations/page.tsx
create mode 100644 frontend/src/app/favicon.ico
create mode 100644 frontend/src/app/globals.css
create mode 100644 frontend/src/app/layout.tsx
create mode 100644 frontend/src/app/page.tsx
create mode 100644 frontend/src/components/auth/LoginForm.tsx
create mode 100644 frontend/src/components/common/AddButton.tsx
create mode 100644 frontend/src/components/common/ConfirmDialog.tsx
create mode 100644 frontend/src/components/common/ConnectionGuard.tsx
create mode 100644 frontend/src/components/common/ContextIndicator.tsx
create mode 100644 frontend/src/components/common/ContextToggle.tsx
create mode 100644 frontend/src/components/common/EmptyState.tsx
create mode 100644 frontend/src/components/common/ErrorBoundary.tsx
create mode 100644 frontend/src/components/common/InlineEdit.tsx
create mode 100644 frontend/src/components/common/LoadingSpinner.tsx
create mode 100644 frontend/src/components/common/ModelSelector.tsx
create mode 100644 frontend/src/components/common/ThemeToggle.tsx
create mode 100644 frontend/src/components/errors/ConnectionErrorOverlay.tsx
create mode 100644 frontend/src/components/layout/AppShell.tsx
create mode 100644 frontend/src/components/layout/AppSidebar.tsx
create mode 100644 frontend/src/components/notebooks/CreateNotebookDialog.tsx
create mode 100644 frontend/src/components/podcasts/EpisodeCard.tsx
create mode 100644 frontend/src/components/podcasts/EpisodeProfilesPanel.tsx
create mode 100644 frontend/src/components/podcasts/EpisodesTab.tsx
create mode 100644 frontend/src/components/podcasts/GeneratePodcastDialog.tsx
create mode 100644 frontend/src/components/podcasts/SpeakerProfilesPanel.tsx
create mode 100644 frontend/src/components/podcasts/TemplatesTab.tsx
create mode 100644 frontend/src/components/podcasts/forms/EpisodeProfileFormDialog.tsx
create mode 100644 frontend/src/components/podcasts/forms/SpeakerProfileFormDialog.tsx
create mode 100644 frontend/src/components/providers/ModalProvider.tsx
create mode 100644 frontend/src/components/providers/QueryProvider.tsx
create mode 100644 frontend/src/components/providers/ThemeProvider.tsx
create mode 100644 frontend/src/components/search/AdvancedModelsDialog.tsx
create mode 100644 frontend/src/components/search/SaveToNotebooksDialog.tsx
create mode 100644 frontend/src/components/search/StreamingResponse.tsx
create mode 100644 frontend/src/components/source/ChatPanel.tsx
create mode 100644 frontend/src/components/source/MessageActions.tsx
create mode 100644 frontend/src/components/source/ModelSelector.tsx
create mode 100644 frontend/src/components/source/SessionManager.tsx
create mode 100644 frontend/src/components/source/SourceDetailContent.tsx
create mode 100644 frontend/src/components/source/SourceDialog.tsx
create mode 100644 frontend/src/components/source/SourceInsightDialog.tsx
create mode 100644 frontend/src/components/sources/AddSourceButton.tsx
create mode 100644 frontend/src/components/sources/AddSourceDialog.tsx
create mode 100644 frontend/src/components/sources/README.md
create mode 100644 frontend/src/components/sources/SourceCard.tsx
create mode 100644 frontend/src/components/sources/index.ts
create mode 100644 frontend/src/components/sources/steps/NotebooksStep.tsx
create mode 100644 frontend/src/components/sources/steps/ProcessingStep.tsx
create mode 100644 frontend/src/components/sources/steps/SourceTypeStep.tsx
create mode 100644 frontend/src/components/ui/accordion.tsx
create mode 100644 frontend/src/components/ui/alert-dialog.tsx
create mode 100644 frontend/src/components/ui/alert.tsx
create mode 100644 frontend/src/components/ui/badge.tsx
create mode 100644 frontend/src/components/ui/button.tsx
create mode 100644 frontend/src/components/ui/card.tsx
create mode 100644 frontend/src/components/ui/checkbox-list.tsx
create mode 100644 frontend/src/components/ui/checkbox.tsx
create mode 100644 frontend/src/components/ui/collapsible.tsx
create mode 100644 frontend/src/components/ui/command.tsx
create mode 100644 frontend/src/components/ui/dialog.tsx
create mode 100644 frontend/src/components/ui/dropdown-menu.tsx
create mode 100644 frontend/src/components/ui/form-section.tsx
create mode 100644 frontend/src/components/ui/input.tsx
create mode 100644 frontend/src/components/ui/label.tsx
create mode 100644 frontend/src/components/ui/markdown-editor.tsx
create mode 100644 frontend/src/components/ui/popover.tsx
create mode 100644 frontend/src/components/ui/progress.tsx
create mode 100644 frontend/src/components/ui/radio-group.tsx
create mode 100644 frontend/src/components/ui/scroll-area.tsx
create mode 100644 frontend/src/components/ui/select.tsx
create mode 100644 frontend/src/components/ui/separator.tsx
create mode 100644 frontend/src/components/ui/sonner.tsx
create mode 100644 frontend/src/components/ui/tabs.tsx
create mode 100644 frontend/src/components/ui/textarea.tsx
create mode 100644 frontend/src/components/ui/tooltip.tsx
create mode 100644 frontend/src/components/ui/wizard-container.tsx
create mode 100644 frontend/src/lib/api/chat.ts
create mode 100644 frontend/src/lib/api/client.ts
create mode 100644 frontend/src/lib/api/embedding.ts
create mode 100644 frontend/src/lib/api/insights.ts
create mode 100644 frontend/src/lib/api/models.ts
create mode 100644 frontend/src/lib/api/notebooks.ts
create mode 100644 frontend/src/lib/api/notes.ts
create mode 100644 frontend/src/lib/api/podcasts.ts
create mode 100644 frontend/src/lib/api/query-client.ts
create mode 100644 frontend/src/lib/api/search.ts
create mode 100644 frontend/src/lib/api/settings.ts
create mode 100644 frontend/src/lib/api/source-chat.ts
create mode 100644 frontend/src/lib/api/sources.ts
create mode 100644 frontend/src/lib/api/transformations.ts
create mode 100644 frontend/src/lib/config.ts
create mode 100644 frontend/src/lib/hooks/use-ask.ts
create mode 100644 frontend/src/lib/hooks/use-auth.ts
create mode 100644 frontend/src/lib/hooks/use-insights.ts
create mode 100644 frontend/src/lib/hooks/use-modal-manager.ts
create mode 100644 frontend/src/lib/hooks/use-models.ts
create mode 100644 frontend/src/lib/hooks/use-navigation.ts
create mode 100644 frontend/src/lib/hooks/use-notebooks.ts
create mode 100644 frontend/src/lib/hooks/use-notes.ts
create mode 100644 frontend/src/lib/hooks/use-podcasts.ts
create mode 100644 frontend/src/lib/hooks/use-search.ts
create mode 100644 frontend/src/lib/hooks/use-settings.ts
create mode 100644 frontend/src/lib/hooks/use-sources.ts
create mode 100644 frontend/src/lib/hooks/use-toast.ts
create mode 100644 frontend/src/lib/hooks/use-transformations.ts
create mode 100644 frontend/src/lib/hooks/use-version-check.ts
create mode 100644 frontend/src/lib/hooks/useNotebookChat.ts
create mode 100644 frontend/src/lib/hooks/useSourceChat.ts
create mode 100644 frontend/src/lib/stores/auth-store.ts
create mode 100644 frontend/src/lib/stores/navigation-store.ts
create mode 100644 frontend/src/lib/stores/sidebar-store.ts
create mode 100644 frontend/src/lib/stores/theme-store.ts
create mode 100644 frontend/src/lib/theme-script.ts
create mode 100644 frontend/src/lib/types/api.ts
create mode 100644 frontend/src/lib/types/auth.ts
create mode 100644 frontend/src/lib/types/common.ts
create mode 100644 frontend/src/lib/types/config.ts
create mode 100644 frontend/src/lib/types/models.ts
create mode 100644 frontend/src/lib/types/podcasts.ts
create mode 100644 frontend/src/lib/types/search.ts
create mode 100644 frontend/src/lib/types/transformations.ts
create mode 100644 frontend/src/lib/utils.ts
create mode 100644 frontend/src/lib/utils/source-references.tsx
create mode 100644 frontend/src/middleware.ts
create mode 100644 frontend/tailwind.config.ts
create mode 100644 frontend/tsconfig.json
create mode 100644 migrations/8.surrealql
create mode 100644 migrations/8_down.surrealql
create mode 100644 migrations/9.surrealql
create mode 100644 migrations/9_down.surrealql
create mode 100644 open_notebook/graphs/source_chat.py
delete mode 100644 open_notebook/utils.py
create mode 100644 open_notebook/utils/README.md
create mode 100644 open_notebook/utils/__init__.py
create mode 100644 open_notebook/utils/context_builder.py
create mode 100644 open_notebook/utils/text_utils.py
create mode 100644 open_notebook/utils/token_utils.py
create mode 100644 open_notebook/utils/version_utils.py
create mode 100644 pages/11_🔧_Advanced.py
create mode 100644 prompts/source_chat.jinja
create mode 100644 specs/chat-message-actions.md
create mode 100644 specs/fix-notebook-chat-context-bug.md
create mode 100644 specs/remove-insights-count-from-details-tab.md
create mode 100644 tests/test_source_chat.py
create mode 100644 tests/test_source_chat_api.py
diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md
index e960b98..4fd01cf 100644
--- a/.claude/CLAUDE.md
+++ b/.claude/CLAUDE.md
@@ -105,6 +105,25 @@ await repo_query("SELECT * FROM table WHERE field = $value", {"value": "example"
await repo_delete(record_id)
```
+### Database Migrations
+
+Database schema migrations run **automatically** when the API starts up. The migration system:
+- Uses `AsyncMigrationManager` from `/open_notebook/database/async_migrate.py`
+- Runs in the FastAPI `lifespan` event handler in `/api/main.py`
+- Checks current database version against available migrations in `/migrations/`
+- Executes pending migrations sequentially on startup
+- Tracks migration state in the `_sbl_migrations` table
+- Fails fast if migrations encounter errors (preventing API startup with outdated schema)
+
+**Important**: Database migrations are now handled by the API. The Streamlit UI migration check (`pages/stream_app/utils.py:check_migration()`) is deprecated and does nothing. Always ensure the API is running before using the React frontend or Streamlit UI.
+
+**Troubleshooting**:
+- If the API fails to start, check logs for migration errors
+- Verify SurrealDB is running: `docker compose ps surrealdb`
+- Check database connection settings in `.env`
+- Migration files must exist in `/migrations/` directory
+- For manual migration rollback, use down migration files (not automated)
+
## Content Processing Pipeline
1. Content ingestion (files, URLs, text) via `/open_notebook/graphs/source.py`
diff --git a/.claude/commands/quality.md b/.claude/commands/quality.md
new file mode 100644
index 0000000..8598731
--- /dev/null
+++ b/.claude/commands/quality.md
@@ -0,0 +1,4 @@
+
+Please run the linter on the @frontend/ app
+
+And run ruff on the python files.
\ No newline at end of file
diff --git a/.claude/sessions/api_troubleshoot/migration_plan.md b/.claude/sessions/api_troubleshoot/migration_plan.md
deleted file mode 100644
index 61531fe..0000000
--- a/.claude/sessions/api_troubleshoot/migration_plan.md
+++ /dev/null
@@ -1,319 +0,0 @@
-# API Migration Plan: Direct Domain Calls to API Calls
-
-## Project Context
-
-The Open Notebook project has undergone a significant architectural migration from direct domain model access to a proper API-based architecture. The project consists of:
-
-1. **Domain Layer**: Core business logic and data models (in `open_notebook/domain/`)
-2. **API Layer**: FastAPI-based REST API endpoints (in `api/`)
-3. **Streamlit Frontend**: User interface components (in `pages/`)
-
-During the development process, a comprehensive API layer was built to provide proper separation of concerns, better error handling, and standardized interfaces. However, it appears that some Streamlit components were not fully migrated to use the API layer and are still making direct calls to domain models using `asyncio.run()`.
-
-This creates several issues:
-- **Architectural inconsistency**: Some parts use APIs while others bypass them
-- **Potential data consistency problems**: Direct domain calls might bypass API validation and business logic
-- **Maintenance difficulties**: Changes to domain models could break Streamlit components unexpectedly
-- **Performance issues**: Direct async calls in Streamlit can cause blocking behavior
-
-## Migration Strategy
-
-This document systematically identifies every instance where Streamlit components directly call domain models and provides the exact API replacement. The goal is to ensure that ALL frontend interactions go through the API layer, maintaining proper architectural boundaries.
-
-## Overview
-This document maps all instances where the Streamlit app is directly calling domain models instead of using the API layer. Each entry includes the current implementation and the recommended API replacement.
-
-## Migration Mappings
-
-### 1. **pages/components/source_panel.py**
-
-#### Line 18: Get Source by ID
-**Current:**
-```python
-source: Source = asyncio.run(Source.get(source_id))
-```
-**Should be:**
-```python
-from api.client import api_client
-source = api_client.get_source(source_id)
-```
-**API Endpoint:** `GET /api/sources/{source_id}`
-
-#### Line 62: Get All Transformations
-**Current:**
-```python
-transformations = asyncio.run(Transformation.get_all(order_by="name asc"))
-```
-**Should be:**
-```python
-from api.transformations_service import transformations_service
-transformations = transformations_service.get_all_transformations()
-```
-**API Endpoint:** `GET /api/transformations`
-
-#### Line 83: Get Embedding Model
-**Current:**
-```python
-embedding_model = asyncio.run(model_manager.get_embedding_model())
-```
-**Should be:**
-```python
-from api.models_service import models_service
-default_models = models_service.get_default_models()
-embedding_model = default_models.get("embedding")
-```
-**API Endpoint:** `GET /api/models/defaults`
-
-#### Line 91: Check Embedded Chunks
-**Current:**
-```python
-if not asyncio.run(source.get_embedded_chunks()) and st.button(
-```
-**Should be:**
-```python
-# Use the source object already fetched from API that includes embedded_chunks field
-if not source.embedded_chunks and st.button(
-```
-**API Endpoint:** `GET /api/sources/{source_id}` (uses embedded_chunks field)
-
-### 2. **pages/components/note_panel.py**
-
-#### Line 16: Get Embedding Model
-**Current:**
-```python
-if not asyncio.run(model_manager.get_embedding_model()):
-```
-**Should be:**
-```python
-from api.models_service import models_service
-default_models = models_service.get_default_models()
-if not default_models.get("embedding"):
-```
-**API Endpoint:** `GET /api/models/defaults`
-
-#### Line 20: Get Note by ID
-**Current:**
-```python
-note: Note = asyncio.run(Note.get(note_id))
-```
-**Should be:**
-```python
-from api.client import api_client
-note = api_client.get_note(note_id)
-```
-**API Endpoint:** `GET /api/notes/{note_id}`
-
-### 3. **pages/components/model_selector.py**
-
-#### Line 21: Get Models by Type
-**Current:**
-```python
-models = asyncio.run(Model.get_models_by_type(model_type))
-```
-**Should be:**
-```python
-from api.models_service import models_service
-models = models_service.get_models(type=model_type)
-```
-**API Endpoint:** `GET /api/models?type={model_type}`
-
-### 4. **pages/stream_app/utils.py**
-
-#### Line 122: Get Default Models Instance
-**Current:**
-```python
-default_models = asyncio.run(DefaultModels.get_instance())
-```
-**Should be:**
-```python
-from api.models_service import models_service
-default_models = models_service.get_default_models()
-```
-**API Endpoint:** `GET /api/models/defaults`
-
-### 5. **pages/stream_app/chat.py**
-
-#### Line 89: Get All Episode Profiles
-**Current:**
-```python
-episode_profiles = asyncio.run(EpisodeProfile.get_all())
-```
-**Should be:**
-```python
-from api.client import api_client
-episode_profiles = api_client.get_episode_profiles()
-```
-**API Endpoint:** `GET /api/episode-profiles`
-
-### 6. **pages/stream_app/source.py**
-
-#### Line 30: Get Speech to Text Model
-**Current:**
-```python
-if not asyncio.run(model_manager.get_speech_to_text()):
-```
-**Should be:**
-```python
-from api.models_service import models_service
-default_models = models_service.get_default_models()
-if not default_models.get("speech_to_text"):
-```
-**API Endpoint:** `GET /api/models/defaults`
-
-#### Line 40: Get All Transformations
-**Current:**
-```python
-transformations = asyncio.run(Transformation.get_all())
-```
-**Should be:**
-```python
-from api.transformations_service import transformations_service
-transformations = transformations_service.get_all_transformations()
-```
-**API Endpoint:** `GET /api/transformations`
-
-#### Line 167: Get Source Insights
-**Current:**
-```python
-insights = asyncio.run(source.get_insights())
-```
-**Should be:**
-```python
-from api.insights_service import insights_service
-insights = insights_service.get_source_insights(source.id)
-```
-**API Endpoint:** `GET /api/sources/{source_id}/insights`
-
-### 7. **pages/stream_app/note.py**
-
-#### Line 20: Get Embedding Model
-**Current:**
-```python
-if not asyncio.run(model_manager.get_embedding_model()):
-```
-**Should be:**
-```python
-from api.models_service import models_service
-default_models = models_service.get_default_models()
-if not default_models.get("embedding"):
-```
-**API Endpoint:** `GET /api/models/defaults`
-
-### 7. **pages/3_🔍_Ask_and_Search.py**
-
-#### Line 66: Get Embedding Model
-**Current:**
-```python
-embedding_model = asyncio.run(model_manager.get_embedding_model())
-```
-**Should be:**
-```python
-from api.models_service import models_service
-default_models = models_service.get_default_models()
-embedding_model = default_models.get("embedding")
-```
-**API Endpoint:** `GET /api/models/defaults`
-
-### 8. **pages/2_📒_Notebooks.py**
-
-#### Line 75: Get Notebook Sources
-**Current:**
-```python
-sources = asyncio.run(current_notebook.get_sources())
-```
-**Should be:**
-```python
-from api.sources_service import sources_service
-sources = sources_service.get_sources(notebook_id=current_notebook.id)
-```
-**API Endpoint:** `GET /api/sources?notebook_id={notebook_id}`
-
-#### Line 76: Get Notebook Notes
-**Current:**
-```python
-notes = asyncio.run(current_notebook.get_notes())
-```
-**Should be:**
-```python
-from api.notes_service import notes_service
-notes = notes_service.get_notes(notebook_id=current_notebook.id)
-```
-**API Endpoint:** `GET /api/notes?notebook_id={notebook_id}`
-
-### 9. **pages/5_🎙️_Podcasts.py**
-
-#### Line 428: Get Text to Speech Models
-**Current:**
-```python
-text_to_speech_models = asyncio.run(Model.get_models_by_type("text_to_speech"))
-```
-**Should be:**
-```python
-from api.models_service import models_service
-text_to_speech_models = models_service.get_models(type="text_to_speech")
-```
-**API Endpoint:** `GET /api/models?type=text_to_speech`
-
-#### Line 429: Get Language Models
-**Current:**
-```python
-text_models = asyncio.run(Model.get_models_by_type("language"))
-```
-**Should be:**
-```python
-from api.models_service import models_service
-text_models = models_service.get_models(type="language")
-```
-**API Endpoint:** `GET /api/models?type=language`
-
-## Missing APIs
-
-✅ **All required APIs are already implemented!**
-
-The Source API already properly exposes embedded chunks information through the `embedded_chunks` field in both `SourceResponse` and `SourceListResponse` models.
-
-## Implementation Notes
-
-1. All `asyncio.run()` calls should be removed since the API client handles async operations internally
-2. Import statements need to be updated to use API services instead of domain models
-3. Error handling should be added for API calls
-4. Consider caching frequently accessed data like default models
-5. The API client should handle authentication and error responses consistently
-
-## Completed Tasks
-
-✅ **API Analysis Complete**: All required APIs are implemented and available
-✅ **Migration Plan Created**: Comprehensive mapping of 20 violations across 9 files
-✅ **Source API Verification**: Confirmed embedded_chunks field is properly exposed
-✅ **SourceWithMetadata Pattern**: Created clean wrapper for domain objects with API metadata
-✅ **Complete API Migration**: All 27 violations across 11 files successfully migrated
-✅ **Episode Profiles Service**: Created new API service for podcast episode profiles
-✅ **Final Verification**: Independent audit confirmed 100% migration completion
-✅ **Post-Audit Fixes**: Fixed 3 additional violations found during final review
-✅ **Architecture Consistency**: All Streamlit components now use API layer exclusively
-
-## Remaining Tasks
-
-1. ✅ ~~**Systematically replace each direct domain call with its API equivalent**~~ (20/20 violations completed)
-2. **Remove unused domain model imports** after migration (optional cleanup)
-3. **Test each component after migration** to ensure functionality is preserved
-
-## Implementation Status
-
-### Phase 1: Critical Components
-- [x] pages/components/source_panel.py (4 violations) ✅
-- [x] pages/components/note_panel.py (2 violations) ✅
-- [x] pages/components/model_selector.py (1 violation) ✅
-
-### Phase 2: Core Streamlit Pages
-- [x] pages/2_📒_Notebooks.py (2 violations) ✅
-- [x] pages/3_🔍_Ask_and_Search.py (1 violation) ✅
-- [x] pages/5_🎙️_Podcasts.py (2 violations) ✅
-
-### Phase 3: Supporting Pages
-- [x] pages/stream_app/source.py (3 violations) ✅
-- [x] pages/stream_app/note.py (1 violation) ✅
-- [x] pages/stream_app/utils.py (1 violation) ✅
-- [x] pages/stream_app/chat.py (1 violation) ✅
-
-**Progress: 27/27 violations fixed (100%) 🎉**
\ No newline at end of file
diff --git a/.claude/sessions/migrate_surrealdb/architecture.md b/.claude/sessions/migrate_surrealdb/architecture.md
deleted file mode 100644
index 313b777..0000000
--- a/.claude/sessions/migrate_surrealdb/architecture.md
+++ /dev/null
@@ -1,358 +0,0 @@
-# SurrealDB Migration Architecture
-
-## High-Level Overview
-
-### Before Migration
-```
-┌─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐
-│ Application Layer │
-├─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤
-│ FastAPI Services │ Streamlit Pages │ Domain Models (base.py, models.py, notebook.py) │ Migration System │ Utils (surreal_clean) │ Background Tasks │
-├─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤
-│ Synchronous Database Layer │
-├─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤
-│ repository.py: repo_query, repo_create, repo_upsert, repo_update, repo_delete, repo_relate │ migrate.py: MigrationManager (sync) │ @contextmanager │
-├─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤
-│ sdblpy (SurrealSyncConnection) │
-├─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤
-│ SurrealDB Database │
-└─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘
-```
-
-### After Migration
-```
-┌─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐
-│ Application Layer │
-├─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤
-│ FastAPI Services │ Streamlit Pages (nest_asyncio) │ Domain Models (async/await) │ Migration System (async) │ Background Tasks (async) │
-├─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤
-│ Asynchronous Database Layer │
-├─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤
-│ new.py: repo_query, repo_create, repo_upsert, repo_update, repo_delete, repo_relate, repo_insert │ migrate.py: AsyncMigrationManager │ @asynccontextmanager │
-├─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤
-│ surrealdb (AsyncSurreal) │
-├─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤
-│ SurrealDB Database │
-└─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘
-```
-
-## Affected Components and Dependencies
-
-### 1. Database Layer (Core Infrastructure)
-
-#### 1.1 Repository Replacement
-- **Replace**: `open_notebook/database/repository.py`
-- **With**: `open_notebook/database/new.py` (rename to `repository.py`)
-- **Changes**:
- - All functions become async
- - Connection management via `@asynccontextmanager`
- - Improved error handling and logging
- - Automatic timestamp management
- - Built-in RecordID parsing
-
-#### 1.2 Migration System Redesign
-- **Replace**: `open_notebook/database/migrate.py`
-- **With**: New async migration system based on sblpy patterns
-- **Components**:
- - `AsyncMigrationManager` - Main migration controller
- - `AsyncMigration` - Individual migration wrapper
- - `AsyncMigrationRunner` - Migration execution engine
- - `db_processes` - Database version management
- - `sql_adapter` - SQL file processing
-
-### 2. Domain Models (Data Access Layer)
-
-#### 2.1 Base Model (`open_notebook/domain/base.py`)
-- **Critical Changes**:
- - All methods become async: `get_all()`, `get()`, `save()`, `delete()`, `relate()`
- - `RecordModel.__init__()` and `update()` become async
- - Add proper async context handling
- - Maintain backward compatibility for method signatures
-
-#### 2.2 Domain Models (`open_notebook/domain/models.py`)
-- **Changes**:
- - `Model.get_models_by_type()` becomes async
- - All model instantiation becomes async
-
-#### 2.3 Notebook Models (`open_notebook/domain/notebook.py`)
-- **Complex Changes**:
- - All property getters become async methods
- - `text_search()` and `vector_search()` functions become async
- - Complex query methods require async handling
- - Embedding and vectorization operations become async
-
-### 3. Application Layer
-
-#### 3.1 FastAPI Services (API Layer)
-- **Files**: `api/models_service.py`, `api/notebook_service.py`, `api/notes_service.py`
-- **Changes**:
- - All endpoints remain async (FastAPI already supports this)
- - Add proper async/await for database calls
- - Update error handling for async operations
-
-#### 3.2 FastAPI Routers
-- **Directory**: `api/routers/`
-- **Changes**:
- - Update all route handlers to properly await database operations
- - Ensure proper async context management
- - Add async error handling
-
-#### 3.3 Streamlit Pages (UI Layer)
-- **Directory**: `pages/`
-- **Changes**:
- - Import and apply `nest_asyncio` at the top of each file
- - Wrap async database calls with `asyncio.run()`
- - Maintain synchronous interface for Streamlit components
- - Add proper error handling for async operations
-
-### 4. Environment Configuration
-
-#### 4.1 Environment Variable Compatibility
-- **Current**: `SURREAL_ADDRESS`, `SURREAL_PORT`, `SURREAL_USER`, `SURREAL_PASS`
-- **New**: `SURREAL_URL`, `SURREAL_USER`, `SURREAL_PASSWORD`
-- **Strategy**:
- - Check for new format first
- - Fall back to old format and convert
- - Provide clear migration documentation
-
-#### 4.2 Connection String Conversion
-```python
-# Old format detection and conversion
-if not os.getenv("SURREAL_URL") and os.getenv("SURREAL_ADDRESS"):
- url = f"http://{os.getenv('SURREAL_ADDRESS')}:{os.getenv('SURREAL_PORT')}"
- os.environ["SURREAL_URL"] = url
- os.environ["SURREAL_PASSWORD"] = os.getenv("SURREAL_PASS")
-```
-
-## External Dependencies
-
-### 4.1 New Dependencies
-- `surrealdb` - Official SurrealDB Python client (already added)
-- `nest_asyncio` - For Streamlit async compatibility
-
-### 4.2 Removed Dependencies
-- `sdblpy` - Custom lightweight client (remove from dependencies)
-
-### 4.3 Updated Utilities
-- Remove `surreal_clean` function from `utils.py` (no longer needed)
-- Update any code that depends on `surreal_clean`
-
-## Implementation Patterns
-
-### 5.1 Async Context Management
-```python
-# Old pattern
-@contextmanager
-def db_connection():
- connection = SurrealSyncConnection(...)
- try:
- yield connection
- finally:
- connection.socket.close()
-
-# New pattern
-@asynccontextmanager
-async def db_connection():
- db = AsyncSurreal(os.environ["SURREAL_URL"])
- await db.signin({"username": ..., "password": ...})
- await db.use(namespace, database)
- try:
- yield db
- finally:
- await db.close()
-```
-
-### 5.2 Domain Model Async Conversion
-```python
-# Old pattern
-class RecordModel:
- def save(self):
- if hasattr(self, 'id') and self.id:
- return repo_update(self.id, self.model_dump())
- else:
- return repo_create(self.table_name, self.model_dump())
-
-# New pattern
-class RecordModel:
- async def save(self):
- if hasattr(self, 'id') and self.id:
- return await repo_update(self.table_name, self.id, self.model_dump())
- else:
- return await repo_create(self.table_name, self.model_dump())
-```
-
-### 5.3 SQL Safety and Parameterized Queries
-```python
-# Old pattern (SQL injection risk)
-srcs = repo_query(f"""
- select * omit source.full_text from (
- select in as source from reference where out={self.id}
- fetch source
-) order by source.updated desc
-""")
-
-# New pattern (SQL safe with parameters)
-srcs = await repo_query("""
- select * omit source.full_text from (
- select in as source from reference where out=$id
- fetch source
-) order by source.updated desc
-""", {"id": ensure_record_id(self.id)})
-```
-
-### 5.4 Streamlit Async Integration
-```python
-# Pattern for Streamlit pages
-import nest_asyncio
-nest_asyncio.apply()
-
-import asyncio
-import streamlit as st
-
-async def load_data():
- return await some_async_database_call()
-
-# In Streamlit app
-data = asyncio.run(load_data())
-st.write(data)
-```
-
-## Migration System Architecture
-
-### 6.1 Async Migration Components
-
-#### AsyncMigrationManager
-- Manages database connections and migration state
-- Handles version checking and migration execution
-- Provides async interface for all migration operations
-
-#### AsyncMigration
-- Wraps individual migration files
-- Supports creation from files, strings, or lists
-- Handles async execution with proper error handling
-
-#### AsyncMigrationRunner
-- Executes migrations in sequence
-- Manages version bumping and rollbacks
-- Provides incremental migration capabilities
-
-### 6.2 Migration Database Schema
-```sql
--- Migration tracking table (same as sblpy)
-CREATE TABLE _sbl_migrations;
-DEFINE FIELD version ON TABLE _sbl_migrations TYPE int;
-DEFINE FIELD applied_at ON TABLE _sbl_migrations TYPE datetime;
-```
-
-### 6.3 Migration File Structure
-```
-migrations/
-├── 1.surrealql # Up migration
-├── 1_down.surrealql # Down migration
-├── 2.surrealql
-├── 2_down.surrealql
-└── ...
-```
-
-## Constraints and Assumptions
-
-### 7.1 Technical Constraints
-- Maintain exact same API interface for all domain models
-- Preserve all existing functionality
-- Support both old and new environment variable formats
-- Ensure Streamlit pages continue to work without major changes
-
-### 7.2 Performance Assumptions
-- Async operations will improve overall performance
-- Connection pooling will be handled by the official client
-- Memory usage may increase slightly due to async overhead
-
-### 7.3 Compatibility Assumptions
-- All existing SurrealQL queries will continue to work
-- RecordID handling will be improved but maintain compatibility
-- Migration files will not need to be modified
-
-## Trade-offs and Alternatives
-
-### 8.1 Chosen Approach: Complete Async Migration
-**Pros**:
-- Modern, future-proof architecture
-- Better performance and scalability
-- Official client support and features
-- Cleaner code with better error handling
-
-**Cons**:
-- Requires updating all database-related code
-- Potential for introducing bugs during conversion
-- Learning curve for async patterns
-
-### 8.2 Alternative: Hybrid Approach
-**Pros**:
-- Gradual migration possible
-- Lower risk of breaking changes
-- Easier to test incrementally
-
-**Cons**:
-- More complex codebase during transition
-- Potential for inconsistencies
-- Longer development time
-
-### 8.3 Alternative: Wrapper Layer
-**Pros**:
-- Minimal changes to existing code
-- Quick implementation
-- Easy rollback
-
-**Cons**:
-- Performance overhead
-- Doesn't leverage async benefits
-- Technical debt accumulation
-
-## Implementation Files
-
-### 8.1 Files to Edit
-1. `open_notebook/database/new.py` → `open_notebook/database/repository.py`
-2. `open_notebook/database/migrate.py` (complete rewrite)
-3. `open_notebook/domain/base.py` (async conversion)
-4. `open_notebook/domain/models.py` (async conversion)
-5. `open_notebook/domain/notebook.py` (async conversion)
-6. All files in `api/` directory (~10 files)
-7. All files in `pages/` directory (~15 files)
-8. All files in `pages/stream_app/` directory (~10 files)
-9. `open_notebook/utils.py` (remove surreal_clean)
-
-### 8.2 Files to Create
-1. `open_notebook/database/async_migrate.py` (new async migration system)
-2. Environment compatibility helpers (if needed)
-
-### 8.3 Files to Remove
-1. `open_notebook/database/repository.py` (old version)
-2. References to `sdblpy` in `pyproject.toml`
-
-## Risk Mitigation
-
-### 9.1 Data Safety
-- Test all operations on development database first
-- Backup production database before migration
-- Verify all CRUD operations work correctly
-
-### 9.2 Code Quality
-- Comprehensive manual testing after each component
-- Verify all async/await patterns are correct
-- Test error handling and edge cases
-
-### 9.3 Performance
-- Monitor database connection efficiency
-- Test with realistic data volumes
-- Verify memory usage patterns
-
-## Success Metrics
-
-1. **Functionality**: All existing features work identically
-2. **Performance**: No degradation in response times
-3. **Reliability**: Proper error handling and logging
-4. **Maintainability**: Clean async/await patterns throughout
-5. **Compatibility**: Environment variables work in both formats
-6. **Migration**: Database migrations work reliably
-
-This architecture provides a comprehensive roadmap for migrating from the lightweight sdblpy client to the official SurrealDB Python client while maintaining all existing functionality and improving the overall system architecture.
\ No newline at end of file
diff --git a/.claude/sessions/migrate_surrealdb/context.md b/.claude/sessions/migrate_surrealdb/context.md
deleted file mode 100644
index be6951f..0000000
--- a/.claude/sessions/migrate_surrealdb/context.md
+++ /dev/null
@@ -1,110 +0,0 @@
-# SurrealDB Migration Context
-
-## Why This Is Being Built
-
-We are migrating from sdblpy (lightweight SurrealDB client) to the official SurrealDB Python client for better functionality, long-term support, and access to the full feature set of SurrealDB.
-
-## Expected Outcome
-
-- Complete replacement of the database layer from synchronous to asynchronous operations
-- Maintain all existing functionality while improving performance and reliability
-- Modernize the codebase to use official SurrealDB client
-- Ensure seamless user experience with no data loss or functionality regression
-
-## Technical Approach
-
-### 1. Database Layer Migration
-- Replace `open_notebook/database/repository.py` with `open_notebook/database/new.py`
-- Convert all database operations from synchronous to asynchronous
-- Update all domain models to use async/await syntax
-
-### 2. Environment Variable Compatibility
-- Maintain backward compatibility by checking which environment variables are configured
-- Convert `SURREAL_ADDRESS` + `SURREAL_PORT` to `SURREAL_URL` format when needed
-- Support both old and new environment variable formats
-
-### 3. Streamlit Integration
-- Use `asyncio.run()` for async database calls in Streamlit pages
-- Import `nest_asyncio` and run `apply()` method before anything else in all Streamlit pages
-- Ensure all Streamlit functionality remains intact
-
-### 4. Migration System
-- Reimplement migration system using async SurrealDB client
-- Inspect source code at `../../../experimentos/surreal-lite-py` for patterns
-- Maintain existing migration file structure and functionality
-
-### 5. API and Domain Models
-- Update all FastAPI endpoints to properly handle async database calls
-- Modify domain models (`base.py`, `models.py`, `notebook.py`) to use async patterns
-- Ensure all relationships and complex queries continue to work
-
-## Key Differences Between Old and New Systems
-
-### Database Functions
-- **Old**: All synchronous functions (repo_create, repo_query, etc.)
-- **New**: All async functions with improved error handling and automatic timestamps
-
-### Environment Variables
-- **Old**: `SURREAL_ADDRESS`, `SURREAL_PORT`, `SURREAL_USER`, `SURREAL_PASS`
-- **New**: `SURREAL_URL`, `SURREAL_USER`, `SURREAL_PASSWORD`
-
-### Connection Management
-- **Old**: `@contextmanager` for sync connections
-- **New**: `@asynccontextmanager` for async connections with proper cleanup
-
-### Data Processing
-- **Old**: Manual data cleaning required (`surreal_clean` function)
-- **New**: Built-in data handling, no manual cleaning needed
-
-## Migration Scope
-
-### Files Requiring Direct Changes (~40+ files)
-1. **Core Domain Models**: `base.py`, `models.py`, `notebook.py`
-2. **API Services**: All FastAPI endpoints and services
-3. **Streamlit Pages**: All pages and components
-4. **Migration System**: `migrate.py` replacement
-5. **Database Layer**: Replace `repository.py` with `new.py`
-
-### Testing Strategy
-- Manual testing approach after completing each major component
-- Test all database operations, API endpoints, and Streamlit functionality
-- Verify data integrity and performance
-
-## Dependencies and Constraints
-
-### New Dependencies
-- Official `surrealdb` Python client (already added)
-- `nest_asyncio` for Streamlit compatibility
-
-### Removed Dependencies
-- `sdblpy` (custom lightweight client)
-- `surreal_clean` utility function (no longer needed)
-
-### Constraints
-- Must maintain all existing functionality
-- No data loss during migration
-- Minimal disruption to user workflows
-- Backward compatibility for environment variables
-
-## Success Criteria
-
-1. All database operations work with async/await pattern
-2. All API endpoints function correctly
-3. All Streamlit pages load and operate normally
-4. Migration system works with new async client
-5. Environment variables support both old and new formats
-6. No functionality regression
-7. Improved performance and reliability
-
-## Risks and Mitigation
-
-### Risks
-- Async conversion might introduce subtle bugs
-- Streamlit async integration complexity
-- Migration system compatibility issues
-
-### Mitigation
-- Thorough manual testing of each component
-- Incremental migration approach
-- Maintain environment variable compatibility
-- Careful inspection of surreal-lite-py source for migration patterns
\ No newline at end of file
diff --git a/.claude/sessions/migrate_surrealdb/plan.md b/.claude/sessions/migrate_surrealdb/plan.md
deleted file mode 100644
index 7290239..0000000
--- a/.claude/sessions/migrate_surrealdb/plan.md
+++ /dev/null
@@ -1,898 +0,0 @@
-# SurrealDB Migration Implementation Plan
-
-## Overview
-
-This plan breaks down the migration from `sdblpy` to the official `surrealdb` Python client into manageable phases of approximately 2 hours each. Each phase is designed to be independent, testable, and builds upon the previous phase.
-
-**Total Estimated Time**: 12-14 hours across 6-7 sessions
-**Risk Level**: Medium-High (significant architecture changes)
-**Rollback Strategy**: Git branches for each phase
-
----
-
-## Phase 1: Foundation & Database Layer Migration (2 hours)
-
-### 🎯 Goals
-- Replace the synchronous database layer with async implementation
-- Create environment variable compatibility layer
-- Establish the foundation for all subsequent migrations
-
-### 📁 Files to Change
-1. `open_notebook/database/repository.py` - Replace with async version
-2. `open_notebook/database/migrate.py` - Create async migration system
-3. `pyproject.toml` - Remove sdblpy dependency
-4. `.env.example` - Add new environment variable examples
-
-### 🔧 Specific Implementation Steps
-
-#### 1.1 Environment Variable Compatibility
-```python
-# Add to repository.py or new config.py
-def get_database_url():
- """Get database URL with backward compatibility"""
- surreal_url = os.getenv("SURREAL_URL")
- if surreal_url:
- return surreal_url
-
- # Fallback to old format - WebSocket URL format
- address = os.getenv("SURREAL_ADDRESS", "localhost")
- port = os.getenv("SURREAL_PORT", "8000")
- return f"ws://{address}/rpc:{port}"
-
-def get_database_password():
- """Get password with backward compatibility"""
- return os.getenv("SURREAL_PASSWORD") or os.getenv("SURREAL_PASS")
-```
-
-#### 1.2 Replace Database Layer
-- Copy `database/new.py` → `database/repository.py`
-- Update connection configuration to use compatibility functions
-- Ensure all function signatures match existing API
-
-#### 1.3 Async Migration System
-Create `database/async_migrate.py`:
-```python
-class AsyncMigrationManager:
- def __init__(self):
- self.url = get_database_url()
- self.password = get_database_password()
- # ... async connection setup
-
- async def get_current_version(self) -> int:
- # Async version of migration tracking
-
- async def run_migration_up(self):
- # Async migration execution
-```
-
-#### 1.4 Update Dependencies
-- Remove `sdblpy` from pyproject.toml
-- Dependencies `surrealdb` and `nest-asyncio` are already properly configured
-
-### ✅ Testing Strategy
-1. Test database connection with both old and new env vars
-2. Verify basic CRUD operations work
-3. Test migration system initialization
-4. Confirm no import errors in application
-
-### ⚠️ Critical Notes
-- **DO NOT** update any domain models in this phase
-- Keep existing function signatures identical
-- Test thoroughly before proceeding to Phase 2
-- **STOP** at end of phase and request human approval before continuing
-
----
-
-## Phase 2: Base Domain Model Migration (2.5 hours)
-
-### 🎯 Goals
-- Convert base classes (`ObjectModel`, `RecordModel`) to async
-- Update simple domain models
-- Establish async patterns for inheritance
-
-### 📁 Files to Change
-1. `open_notebook/domain/base.py` - Convert to async
-2. `open_notebook/domain/models.py` - Update ModelManager to async
-
-### 🔧 Specific Implementation Steps
-
-#### 2.1 Async Base Classes
-Convert `ObjectModel` and `RecordModel`:
-```python
-class ObjectModel(BaseModel):
- # ... existing code ...
-
- async def save(self):
- """Async save method"""
- data = self.model_dump() # Pydantic v2 syntax
- if hasattr(self, 'id') and self.id:
- result = await repo_update(self.table_name, self.id, data)
- else:
- result = await repo_create(self.table_name, data)
- # Update self with returned data
- return self
-
- async def delete(self):
- """Async delete method"""
- if hasattr(self, 'id') and self.id:
- return await repo_delete(ensure_record_id(self.id))
- raise ValueError("Cannot delete object without ID")
-
- @classmethod
- async def get_all(cls, limit: int = 1000):
- """Async get all method"""
- result = await repo_query(f"SELECT * FROM {cls.table_name} LIMIT $limit", {"limit": limit})
- return [cls(**item) for item in result]
-
- @classmethod
- async def get(cls, id: str):
- """Async get by ID method"""
- result = await repo_query("SELECT * FROM $id", {"id": ensure_record_id(f"{cls.table_name}:{id}")})
- if result:
- return cls(**result[0])
- return None
-```
-
-#### 2.2 Convert Simple Models
-Update these models to use async base methods:
-- `ContentSettings` (RecordModel)
-- `DefaultModels` (RecordModel)
-- `DefaultPrompts` (RecordModel)
-- `Transformation` (ObjectModel)
-
-#### 2.3 Update ModelManager
-```python
-class ModelManager:
- async def get_models_by_type(self, model_type: str):
- """Async model retrieval"""
- return await repo_query(
- "SELECT * FROM model WHERE type = $type",
- {"type": model_type}
- )
-
- # Update caching to be async-safe
-```
-
-### ✅ Testing Strategy
-1. Test base class CRUD operations
-2. Verify inheritance works correctly
-3. Test simple model operations
-4. Check ModelManager functionality
-
-### ⚠️ Critical Notes
-- This phase establishes the async pattern for all other models
-- Property methods that use database queries will need attention in future phases
-- Keep backward compatibility for method names
-- **STOP** at end of phase and request human approval before continuing
-
----
-
-## Phase 3: Medium Complexity Domain Models (2 hours)
-
-### 🎯 Goals
-- Convert medium complexity models to async
-- Handle property to async method conversion
-- Update SQL queries to use parameterized syntax
-
-### 📁 Files to Change
-1. `open_notebook/domain/notebook.py` - Convert Notebook, Note, ChatSession
-2. Update all property methods to async methods
-
-### 🔧 Specific Implementation Steps
-
-#### 3.1 Convert Property Methods to Async Methods
-```python
-class Notebook(ObjectModel):
- # Old property
- @property
- def sources(self):
- return repo_query(f"SELECT * FROM source WHERE notebook_id = '{self.id}'")
-
- # New async method
- async def get_sources(self):
- return await repo_query(
- "SELECT * FROM source WHERE notebook_id = $id",
- {"id": ensure_record_id(self.id)}
- )
-
- # Update all properties: sources, notes, chat_sessions
-```
-
-#### 3.2 Security: Parameterized Queries
-Convert all f-string queries to parameterized:
-```python
-# OLD (Security risk)
-result = await repo_query(f"SELECT * FROM reference WHERE out={self.id}")
-
-# NEW (Secure)
-result = await repo_query(
- "SELECT * FROM reference WHERE out=$id",
- {"id": ensure_record_id(self.id)}
-)
-```
-
-#### 3.3 Convert Models
-- `Notebook` - Convert properties to async methods
-- `Note` - Update save with embedding logic
-- `ChatSession` - Simple conversion
-- `SourceEmbedding` - Simple with one relationship
-- `SourceInsight` - Simple with one relationship
-
-### ✅ Testing Strategy
-1. Test each model's CRUD operations
-2. Verify relationship queries work
-3. Test parameterized query security
-4. Check embedding functionality
-
-### ⚠️ Critical Notes
-- **BREAKING CHANGE**: Properties become async methods (`.sources` → `await .get_sources()`)
-- All SQL queries must be parameterized for security
-- Document property → method name changes
-- **STOP** at end of phase and request human approval before continuing
-
----
-
-## Phase 4: Source and Search Migration (2.5 hours)
-
-### 🎯 Goals
-- Convert the most complex model (Source) with vectorization
-- Handle ThreadPoolExecutor integration with async
-- Update search functions
-
-### 📁 Files to Change
-1. `open_notebook/domain/notebook.py` - Source model and search functions
-
-### 🔧 Specific Implementation Steps
-
-#### 4.1 Source Model Vectorization
-```python
-class Source(ObjectModel):
- async def vectorize(self):
- """Complex async vectorization with ThreadPoolExecutor"""
- # Keep ThreadPoolExecutor for CPU-bound embedding work
- loop = asyncio.get_event_loop()
-
- with ThreadPoolExecutor() as executor:
- # Run CPU-intensive embedding in thread pool
- embedding_task = loop.run_in_executor(
- executor, self._generate_embeddings
- )
- embeddings = await embedding_task
-
- # Async database operations
- for chunk_data in embeddings:
- await repo_create("source_embedding", chunk_data)
-
- def _generate_embeddings(self):
- """Sync method for CPU-bound embedding work"""
- # Existing embedding logic stays synchronous
- pass
-
- async def add_insight(self, insight_text: str):
- """Async insight creation"""
- return await repo_create("source_insight", {
- "source_id": self.id,
- "content": insight_text
- })
-```
-
-#### 4.2 Update Search Functions
-```python
-async def text_search(query: str, notebook_id: str = None):
- """Async text search with parameterized queries"""
- conditions = ["content CONTAINS $query"]
- params = {"query": query}
-
- if notebook_id:
- conditions.append("notebook_id = $notebook_id")
- params["notebook_id"] = ensure_record_id(notebook_id)
-
- sql = f"SELECT * FROM source WHERE {' AND '.join(conditions)}"
- return await repo_query(sql, params)
-
-async def vector_search(query: str, limit: int = 10):
- """Async vector search"""
- # Implementation with async database calls
-```
-
-### ✅ Testing Strategy
-1. Test Source model CRUD operations
-2. Verify vectorization process works
-3. Test search functions with various queries
-4. Check ThreadPoolExecutor integration
-
-### ⚠️ Critical Notes
-- ThreadPoolExecutor pattern for CPU-bound work
-- Async/sync boundary management crucial
-- Search functions are heavily used - test thoroughly
-- **STOP** at end of phase and request human approval before continuing
-
----
-
-## Phase 5: API Layer Migration (1.5 hours)
-
-### 🎯 Goals
-- Update all FastAPI endpoints to properly await domain operations
-- Update service classes to use async domain methods
-- Ensure proper error handling
-
-### 📁 Files to Change
-1. `api/notebook_service.py` - Update service methods
-2. `api/notes_service.py` - Update service methods
-3. `api/models_service.py` - Update service methods
-4. All files in `api/routers/` - Update route handlers
-
-### 🔧 Specific Implementation Steps
-
-#### 5.1 Update Service Classes
-```python
-class NotebookService:
- async def get_notebook(self, notebook_id: str):
- """Update to use async domain methods"""
- notebook = await Notebook.get(notebook_id)
- if notebook:
- # Property methods become async method calls
- sources = await notebook.get_sources()
- notes = await notebook.get_notes()
- return {
- "notebook": notebook,
- "sources": sources,
- "notes": notes
- }
- return None
-
- async def create_notebook(self, data: dict):
- """Async notebook creation"""
- notebook = Notebook(**data)
- return await notebook.save()
-```
-
-#### 5.2 Update API Routers
-```python
-@router.get("/notebooks/{notebook_id}")
-async def get_notebook(notebook_id: str):
- """Ensure proper async/await usage"""
- service = NotebookService()
- result = await service.get_notebook(notebook_id) # Await added
- if result:
- return result
- raise HTTPException(status_code=404, detail="Notebook not found")
-```
-
-### ✅ Testing Strategy
-1. Test all API endpoints manually
-2. Verify proper error handling
-3. Check response formats remain consistent
-4. Test with various data scenarios
-
-### ⚠️ Critical Notes
-- FastAPI endpoints are already async, just need proper await calls
-- Service layer acts as adapter between API and domain
-- Maintain existing API response formats
-- **STOP** at end of phase and request human approval before continuing
-
----
-
-## Phase 6: Streamlit Integration (2 hours)
-
-### 🎯 Goals
-- Add `nest_asyncio` to all Streamlit pages
-- Wrap domain model calls with `asyncio.run()`
-- Update complex UI operations
-
-### 📁 Files to Change
-1. All files in `pages/` directory (~15 files)
-2. All files in `pages/stream_app/` directory (~10 files)
-3. Files in `pages/components/` directory (~5 files)
-
-### 🔧 Specific Implementation Steps
-
-#### 6.1 Standard Streamlit Page Pattern
-```python
-# Add to top of every Streamlit file
-import nest_asyncio
-nest_asyncio.apply()
-
-import asyncio
-import streamlit as st
-from open_notebook.domain.notebook import Notebook
-
-# Async data loading
-async def load_notebooks():
- return await Notebook.get_all()
-
-async def load_notebook_details(notebook_id):
- notebook = await Notebook.get(notebook_id)
- if notebook:
- sources = await notebook.get_sources()
- notes = await notebook.get_notes()
- return notebook, sources, notes
- return None, [], []
-
-# Streamlit app code
-def main():
- st.title("My Page")
-
- # Wrap async calls
- notebooks = asyncio.run(load_notebooks())
-
- if st.selectbox("Select Notebook", notebooks):
- notebook_id = st.session_state.selected_notebook
- notebook, sources, notes = asyncio.run(load_notebook_details(notebook_id))
-
- # Display data...
-
-if __name__ == "__main__":
- main()
-```
-
-#### 6.2 Handle Service Layer Calls
-For pages using service layer HTTP calls:
-```python
-# These remain mostly unchanged since they use HTTP
-service = NotebookService()
-response = requests.get(f"/api/notebooks/{notebook_id}")
-```
-
-#### 6.3 Complex Chat Integration
-```python
-# pages/stream_app/chat.py - Special handling
-async def process_chat_message(message: str, notebook_id: str):
- # LangGraph operations are already async
- result = await chat_graph.astream({
- "message": message,
- "notebook_id": notebook_id
- })
- return result
-
-# In Streamlit
-if user_input:
- response = asyncio.run(process_chat_message(user_input, notebook_id))
-```
-
-### ✅ Testing Strategy
-1. Test each Streamlit page loads correctly
-2. Verify all async operations work
-3. Check session state management
-4. Test complex chat functionality
-
-### ⚠️ Critical Notes
-- Some pages already use `nest_asyncio` - check before adding
-- Service layer HTTP calls don't need changes
-- Chat system needs special attention due to streaming
-- **STOP** at end of phase and request human approval before continuing
-
----
-
-## Phase 7: Migration System & Cleanup (1 hour)
-
-### 🎯 Goals
-- Update migration system to use async database client
-- Remove obsolete code and dependencies
-- Final testing and documentation
-
-### 📁 Files to Change
-1. `open_notebook/database/migrate.py` - Finalize async migration system
-2. `open_notebook/utils.py` - Remove `surreal_clean` function
-3. `pages/stream_app/utils.py` - Update migration check
-4. Documentation updates
-
-### 🔧 Specific Implementation Steps
-
-#### 7.1 Finalize Async Migration System
-```python
-class AsyncMigrationManager:
- async def run_migration_up(self):
- """Complete async migration implementation"""
- current_version = await self.get_current_version()
-
- if self.needs_migration:
- for i in range(current_version, len(self.up_migrations)):
- migration = self.up_migrations[i]
- async with db_connection() as conn:
- await conn.query(migration.sql)
- await self.bump_version()
-
- async def needs_migration(self) -> bool:
- current = await self.get_current_version()
- return current < len(self.up_migrations)
-```
-
-#### 7.2 Remove Obsolete Code
-- Remove `surreal_clean` function from `utils.py`
-- Update any code that imported `surreal_clean`
-- Clean up unused imports
-
-#### 7.3 Update Migration Check
-```python
-# pages/stream_app/utils.py
-async def check_migration():
- """Async migration check"""
- manager = AsyncMigrationManager()
- if await manager.needs_migration():
- await manager.run_migration_up()
-```
-
-### ✅ Testing Strategy
-1. Test migration system works end-to-end
-2. Verify application starts without errors
-3. Test all major functionality paths
-4. Performance check
-
-### ⚠️ Critical Notes
-- **STOP** at end of phase and request human approval
-- Mark migration as complete in plan.md
-
----
-
-## 🚨 Risk Mitigation Strategies
-
-### Git Strategy
-- Work directly on current branch (no additional branches needed)
-- Human will review and commit after each phase completion
-- Agent must request human approval before proceeding to next phase
-
-### Testing Approach
-- Manual testing after each phase
-- Focus on CRUD operations, API endpoints, and UI functionality
-- Test with realistic data volumes
-- Performance monitoring
-
-### Rollback Plan
-- Each phase is designed to be independently rollback-able
-- Keep environment variable compatibility for easy switching
-- Maintain backup of current working state
-
----
-
-## 📋 Success Criteria
-
-### Phase Completion Criteria
-- [ ] All code compiles without errors
-- [ ] No breaking changes to external API interfaces
-- [ ] All manual tests pass
-- [ ] Performance is maintained or improved
-- [ ] Environment variables work in both formats
-
-### Final Success Metrics
-- [ ] All existing functionality preserved
-- [ ] Improved security with parameterized queries
-- [ ] Clean async/await patterns throughout
-- [ ] Official SurrealDB client integration complete
-- [ ] Migration system working with async client
-- [ ] Documentation updated
-
----
-
-## 🎯 Implementation Notes
-
-### Session Planning
-- **Session 1**: Phase 1 (Foundation)
-- **Session 2**: Phase 2 + start Phase 3 (Base models)
-- **Session 3**: Complete Phase 3 + Phase 4 (Complex models)
-- **Session 4**: Phase 5 + Phase 6 (API + Streamlit)
-- **Session 5**: Phase 7 + final testing (Cleanup)
-
-### Dependencies Between Phases
-- Phase 2 depends on Phase 1 (database layer)
-- Phase 3 builds on Phase 2 (base classes)
-- Phase 4 completes domain model migration
-- Phases 5-6 can be done in parallel if needed
-- Phase 7 requires all previous phases
-
-### Breaking Changes Documentation
-- Properties become async methods (documented in each phase)
-- Import changes (minimal, mostly internal)
-- Environment variable additions (backward compatible)
-
-This plan provides a systematic approach to migrating the entire codebase while minimizing risk and maintaining functionality throughout the process.
-
----
-
-## 📝 Phase Completion Tracking
-
-### Phase Status
-- [x] **Phase 1**: Foundation & Database Layer Migration - ✅ **COMPLETED**
-- [x] **Phase 2**: Base Domain Model Migration - ✅ **COMPLETED**
-- [x] **Phase 3**: Medium Complexity Domain Models - ✅ **COMPLETED**
-- [x] **Phase 4**: Complex Domain Models - ✅ **COMPLETED**
-- [x] **Phase 5**: API Layer Migration - ✅ **COMPLETED**
-- [x] **Phase 6**: Streamlit Integration - ✅ **COMPLETED**
-- [x] **Phase 7**: Migration System & Cleanup - ✅ **COMPLETED**
-
-### Important Notes for Agent
-- **ALWAYS STOP** at the end of each phase and request human approval
-- **UPDATE** this plan.md file after each successful phase:
- - Mark phase as complete with ✅
- - Add any lessons learned or additional notes
- - Update next steps if requirements change
-- **ASK HUMAN** to review and commit changes before proceeding
-- **DO NOT** proceed to next phase without explicit human approval
-
----
-
-## 📋 Phase 1 Completion Summary
-
-**✅ PHASE 1 COMPLETED SUCCESSFULLY**
-
-### What Was Accomplished
-1. **Environment Compatibility Layer**: Created `get_database_url()` and `get_database_password()` functions that support both old and new environment variable formats
-2. **Async Database Layer**: Replaced `repository.py` with async version using official SurrealDB client
-3. **Migration System**: Created complete async migration system with backward-compatible sync wrapper
-4. **Dependencies Updated**: Removed `sdblpy` dependency, confirmed `surrealdb` and `nest-asyncio` are properly configured
-5. **Environment Configuration**: Updated `.env.example` with new format examples
-
-### Files Modified
-- `open_notebook/database/repository.py` - Replaced with async version
-- `open_notebook/database/repository_old.py` - Backup of original
-- `open_notebook/database/async_migrate.py` - New async migration system
-- `open_notebook/database/migrate.py` - Updated to use async system with sync wrapper
-- `pyproject.toml` - Removed sdblpy dependency
-- `.env.example` - Added new environment variable format
-
-### Testing Results
-- ✅ Environment compatibility functions work correctly
-- ✅ URL generation from old format: `ws://localhost/rpc:8000`
-- ✅ Password compatibility works with both formats
-- ✅ All repository function imports successful
-- ✅ Migration system imports working
-- ✅ Domain models show expected async/sync mismatch (to be fixed in Phase 2)
-
-### Ready for Phase 2
-The foundation is now in place. Domain models currently show expected errors when trying to use async repository functions synchronously. This will be resolved in Phase 2 when we convert the base domain models to async.
-
-**🛑 STOPPING FOR HUMAN APPROVAL** - Please review and commit these changes before proceeding to Phase 2.
-
----
-
-## 📋 Phase 2 Completion Summary
-
-**✅ PHASE 2 COMPLETED SUCCESSFULLY**
-
-### What Was Accomplished
-1. **ObjectModel Async Conversion**: Converted all base methods to async (`get_all`, `get`, `save`, `delete`, `relate`)
-2. **RecordModel Async Conversion**: Updated singleton pattern with async initialization (`get_instance`, `update`, `patch`)
-3. **Model Class Updates**: Made `get_models_by_type()` async and updated ModelManager methods
-4. **Security Improvements**: Ensured all user-input queries use parameterized syntax
-5. **Embedding Integration**: Updated async embedding model access in save() method
-
-### Files Modified
-- `open_notebook/domain/base.py` - Complete async conversion of ObjectModel and RecordModel
-- `open_notebook/domain/models.py` - Async conversion of Model class and ModelManager
-
-### Key Changes
-- **Breaking Change**: All domain model methods are now async (callers must use `await`)
-- **Pattern Change**: RecordModel uses `await ClassName.get_instance()` instead of `ClassName()`
-- **Security**: All database queries use parameterized syntax to prevent SQL injection
-- **ModelManager**: All model retrieval methods are now async
-
-### Testing Results
-- ✅ All imports successful
-- ✅ ObjectModel methods are async (get_all, get, save, delete, relate)
-- ✅ RecordModel methods are async (get_instance, update, patch)
-- ✅ Model class methods are async (get_models_by_type, get_all, get)
-- ✅ ModelManager methods are async (get_model, get_default_model, get_embedding_model, refresh_defaults)
-- ✅ Parameterized queries implemented for security
-
-### Ready for Phase 3
-The async foundation is now complete. All base classes properly support async operations and establish the pattern for domain model inheritance. Phase 3 can now proceed to convert medium complexity domain models.
-
-**🛑 STOPPING FOR HUMAN APPROVAL** - Please review and commit these changes before proceeding to Phase 3.
-
----
-
-## 📋 Phase 3 Completion Summary
-
-**✅ PHASE 3 COMPLETED SUCCESSFULLY**
-
-### What Was Accomplished
-1. **Notebook Properties → Async Methods**: Converted `sources`, `notes`, `chat_sessions` properties to `get_sources()`, `get_notes()`, `get_chat_sessions()` async methods
-2. **Source Class Complex Methods**: Updated `vectorize()`, `add_insight()`, `get_context()`, `get_embedded_chunks()`, `get_insights()`, and `add_to_notebook()` to async
-3. **Simple Model Updates**: Converted `SourceEmbedding.get_source()`, `SourceInsight.get_source()`, `SourceInsight.save_as_note()`, `Note.add_to_notebook()`, `ChatSession.relate_to_notebook()` to async
-4. **Search Functions**: Made `text_search()` and `vector_search()` async with proper embedding model access
-5. **Security & Cleanup**: Parameterized all queries, removed `surreal_clean` usage, updated async embedding model access
-
-### Files Modified
-- `open_notebook/domain/notebook.py` - Complete async conversion of all medium complexity models and functions
-
-### Key Changes
-- **Breaking Change**: All property access becomes async method calls
-- **ThreadPoolExecutor Integration**: `vectorize()` properly combines CPU-bound embedding work with async database operations
-- **Security**: All database queries use parameterized syntax to prevent SQL injection
-- **Clean Architecture**: Removed `surreal_clean` dependency - no longer needed with official client
-
-### Property → Method Mapping
-- `notebook.sources` → `await notebook.get_sources()`
-- `notebook.notes` → `await notebook.get_notes()`
-- `notebook.chat_sessions` → `await notebook.get_chat_sessions()`
-- `source.insights` → `await source.get_insights()`
-- `source.embedded_chunks` → `await source.get_embedded_chunks()`
-- `source_embedding.source` → `await source_embedding.get_source()`
-- `source_insight.source` → `await source_insight.get_source()`
-
-### Testing Results
-- ✅ All imports successful
-- ✅ All Notebook async methods working (get_sources, get_notes, get_chat_sessions)
-- ✅ All Source async methods working (get_context, get_embedded_chunks, get_insights, vectorize, add_insight, add_to_notebook)
-- ✅ All relationship model async methods working (SourceEmbedding, SourceInsight)
-- ✅ All search functions async (text_search, vector_search)
-- ✅ Security: surreal_clean successfully removed
-- ✅ Parameterized queries implemented
-
-### Ready for Phase 4
-All medium complexity domain models now use async patterns. The core business logic models (Notebook, Source, Note, etc.) are fully async and secure. Phase 4 can now proceed to handle any remaining complex domain models and edge cases.
-
-**🛑 STOPPING FOR HUMAN APPROVAL** - Please review and commit these changes before proceeding to Phase 4.
-
----
-
-## 📋 Phase 4 Completion Summary
-
-**✅ PHASE 4 COMPLETED SUCCESSFULLY**
-
-### What Was Accomplished
-1. **Async Embedding Calls**: Converted all sync `.embed()` calls to async `.aembed()` throughout the codebase
-2. **Source.vectorize() Optimization**: Replaced ThreadPoolExecutor with `asyncio.gather()` for proper async concurrent processing
-3. **Search Functions**: Fully async text_search() and vector_search() with async embedding generation
-4. **Graph Integration**: Updated graphs/source.py functions to use async source operations with proper await calls
-5. **Code Cleanup**: Removed all `surreal_clean` usage - no longer needed with official SurrealDB client
-
-### Files Modified
-- `open_notebook/domain/notebook.py` - Fixed Source.vectorize(), Source.add_insight(), vector_search()
-- `open_notebook/domain/base.py` - Fixed ObjectModel.save() embedding calls
-- `open_notebook/graphs/source.py` - Updated save_source(), transform_content() to async, removed surreal_clean
-- `pages/stream_app/note.py` - Removed surreal_clean usage
-
-### Key Technical Changes
-- **Vectorization Performance**: Switched from ThreadPoolExecutor to `asyncio.gather()` for better async performance
-- **Async Boundary Management**: All embedding operations now properly use async calls
-- **Graph Workflows**: All source operations in LangGraph workflows now async-compatible
-- **Security**: Maintained parameterized queries while updating to async patterns
-
-### Testing Results
-- ✅ All imports successful
-- ✅ All async method signatures correct
-- ✅ Class instantiation working
-- ✅ No syntax or import errors
-- ✅ Source.vectorize(), Source.add_insight(), search functions, and graph workflows all async
-
-### Ready for Phase 5
-All complex domain model operations are now fully async. The core business logic is complete and ready for API layer migration. Graph workflows properly integrate with async domain methods.
-
-**🛑 STOPPING FOR HUMAN APPROVAL** - Please review and commit these changes before proceeding to Phase 5.
-
----
-
-## 📋 Phase 5 Completion Summary
-
-**✅ PHASE 5 COMPLETED SUCCESSFULLY**
-
-### What Was Accomplished
-1. **Router Layer Complete Migration**: Updated all 9 router files to use async domain model methods
-2. **Property Access Conversion**: Converted all property access to async method calls (e.g., `notebook.sources` → `await notebook.get_sources()`)
-3. **Domain Model Method Updates**: All `get()`, `save()`, `delete()`, and special methods now use `await`
-4. **Search Function Updates**: Both `text_search()` and `vector_search()` functions converted to async
-5. **RecordModel Pattern Updates**: Updated singleton pattern calls to `await Model.get_instance()`
-
-### Files Modified
-- `api/routers/notebooks.py` - All Notebook CRUD operations converted to async
-- `api/routers/notes.py` - All Note CRUD operations + property access (`notebook.notes` → `await notebook.get_notes()`)
-- `api/routers/sources.py` - All Source CRUD operations + insights access (`source.insights` → `await source.get_insights()`)
-- `api/routers/context.py` - Property access converted to async methods + all Source/Note lookups
-- `api/routers/embedding.py` - Source/Note get and vectorize methods converted to async
-- `api/routers/models.py` - Model CRUD + DefaultModels singleton pattern converted to async
-- `api/routers/search.py` - Search functions converted to async
-- `api/routers/settings.py` - ContentSettings singleton pattern converted to async
-- `api/routers/transformations.py` - Transformation CRUD operations converted to async
-
-### Key Changes Made
-- **Breaking Change**: All router endpoints now properly await domain model operations
-- **Property → Method Conversion**: Critical property access converted to async methods:
- - `notebook.sources` → `await notebook.get_sources()`
- - `notebook.notes` → `await notebook.get_notes()`
- - `source.insights` → `await source.get_insights()`
-- **RecordModel Updates**: Singleton access pattern updated:
- - `DefaultModels()` → `await DefaultModels.get_instance()`
- - `ContentSettings()` → `await ContentSettings.get_instance()`
-- **Search Functions**: Both text and vector search now async
-- **Model Manager**: Refresh operations converted to async
-
-### Testing Results
-- ✅ All router imports successful
-- ✅ All domain model imports successful
-- ✅ Main API app imports successfully
-- ✅ No syntax or import errors detected
-- ✅ FastAPI endpoints remain async-compatible
-- ✅ Error handling patterns preserved
-
-### Ready for Phase 6
-The API layer is now fully compatible with async domain models. All FastAPI endpoints properly await domain operations, and the property → method conversions are complete. The API maintains all existing functionality while using the new async patterns.
-
-**🛑 STOPPING FOR HUMAN APPROVAL** - Please review and commit these changes before proceeding to Phase 6.
-
----
-
-## 📋 Phase 6 Completion Summary
-
-**✅ PHASE 6 COMPLETED SUCCESSFULLY**
-
-### What Was Accomplished
-1. **nest_asyncio Integration**: Added `nest_asyncio.apply()` to all Streamlit files requiring async domain model access
-2. **Property → Method Conversion**: Converted all property access to async method calls throughout Streamlit UI:
- - `notebook.sources` → `asyncio.run(notebook.get_sources())`
- - `notebook.notes` → `asyncio.run(notebook.get_notes())`
- - `notebook.chat_sessions` → `asyncio.run(notebook.get_chat_sessions())`
- - `source.insights` → `asyncio.run(source.get_insights())`
- - `source.embedded_chunks` → `asyncio.run(source.get_embedded_chunks())`
-3. **Domain Model Calls**: Wrapped all direct domain model operations with `asyncio.run()`:
- - `ObjectModel.get()` → `asyncio.run(ObjectModel.get())`
- - `Source.get()` → `asyncio.run(Source.get())`
- - `Note.save()` → `asyncio.run(note.save())`
- - `ChatSession.get()` → `asyncio.run(ChatSession.get())`
-4. **RecordModel Pattern Updates**: Updated singleton pattern calls:
- - `DefaultModels()` → `asyncio.run(DefaultModels.get_instance())`
- - All RecordModel access now uses async get_instance()
-5. **Bug Fix**: Fixed RecordModel._load_from_db() to handle both list and dict responses from SurrealDB queries
-
-### Files Modified
-- `app_home.py` - Added nest_asyncio, converted ObjectModel.get() to async
-- `pages/2_📒_Notebooks.py` - Added nest_asyncio, converted property access to async methods
-- `pages/stream_app/utils.py` - Fixed migration check and model manager calls to async
-- `pages/components/source_panel.py` - Updated Source.get() and property access to async
-- `pages/components/note_panel.py` - Added nest_asyncio, converted Note.get() to async
-- `pages/components/source_insight.py` - Added nest_asyncio, converted all domain calls to async
-- `pages/components/source_embedding_panel.py` - Added nest_asyncio, converted all domain calls to async
-- `pages/stream_app/note.py` - Added nest_asyncio, converted save/relate calls to async
-- `pages/stream_app/chat.py` - Added nest_asyncio, converted chat_sessions property to async
-- `pages/3_🔍_Ask_and_Search.py` - Added nest_asyncio, converted Notebook.get_all() and Note operations to async
-- `pages/5_🎙️_Podcasts.py` - Added nest_asyncio, converted Model.get_models_by_type() to async
-- `open_notebook/domain/base.py` - Fixed RecordModel._load_from_db() for SurrealDB compatibility
-
-### Key Technical Changes
-- **Streamlit Async Pattern**: All Streamlit files now use `nest_asyncio.apply()` + `asyncio.run()` pattern
-- **Property Access Elimination**: All property access converted to explicit async method calls
-- **Database Compatibility**: Fixed RecordModel loading to handle new SurrealDB client response format
-- **Service Layer Preservation**: HTTP-based service calls remained unchanged (no async conversion needed)
-
-### Testing Results
-- ✅ All Streamlit files import successfully
-- ✅ Domain model async operations working
-- ✅ nest_asyncio integration functional
-- ✅ RecordModel singleton pattern working with async
-- ✅ No import or syntax errors detected
-
-### Ready for Phase 7
-All Streamlit pages now properly integrate with async domain models. The UI layer maintains identical functionality while using the new async patterns. Only Phase 7 (Migration System & Cleanup) remains to complete the full migration.
-
-**🛑 STOPPING FOR HUMAN APPROVAL** - Please review and commit these changes before proceeding to Phase 7.
-
----
-
-## 📋 Phase 7 Completion Summary
-
-**✅ PHASE 7 COMPLETED SUCCESSFULLY**
-
-### What Was Accomplished
-1. **Code Cleanup**: Removed obsolete `surreal_clean` function from `utils.py` (lines 103-123)
-2. **Migration System Verification**: Confirmed async migration system is working correctly with sync wrapper for Streamlit
-3. **Environment Compatibility**: Verified both old and new environment variable formats work correctly
-4. **Documentation**: Updated phase tracking to mark all phases complete
-
-### Files Modified
-- `open_notebook/utils.py` - Removed obsolete surreal_clean function
-
-### Key Observations
-- Migration system was already fully implemented in Phase 1 and is working correctly
-- Environment variable compatibility layer properly handles both formats
-- All previous cleanup was done incrementally during Phases 1-6
-- No issues found during testing
-
-### Migration Complete! 🎉
-The entire SurrealDB migration from `sdblpy` to the official `surrealdb` Python client is now complete. The codebase has been successfully modernized with:
-- Full async/await support throughout
-- Official SurrealDB client integration
-- Improved security with parameterized queries
-- Maintained backward compatibility for environment variables
-- Clean architecture with proper separation of concerns
-
-**🛑 FINAL STOP** - The migration is complete! Please review and commit these final changes.
\ No newline at end of file
diff --git a/.claude/sessions/migrate_surrealdb/requirements.txt b/.claude/sessions/migrate_surrealdb/requirements.txt
deleted file mode 100644
index 878f1b4..0000000
--- a/.claude/sessions/migrate_surrealdb/requirements.txt
+++ /dev/null
@@ -1,15 +0,0 @@
-This project uses SurrealDB as its database engine and we have been using a lightweight client: sdblpy = { git = "https://github.com/lfnovo/surreal-lite-py" }
-
-We are now migrating to the official SurrealDB Python client (surrealdb).
-
-The main difference is that surrealdb is a full SurrealDB client, while sdblpy is a lightweight client that only provides a subset of the features.
-
-I have already prepared the new library helpers we will use at /Users/luisnovo/dev/projetos/open-notebook/open-notebook/open_notebook/database/new.py
-
-There are 3 challenges with this project:
- - The new library is an asynchronous library and most of our database code is based in sync operations. We need to decide how to handle this.
- - The old client has a pretty useful migration feature that we use in /Users/luisnovo/dev/projetos/open-notebook/open-notebook/open_notebook/database/migrate.py - we will need to find a way to inspect this feature and rewrite it for us to use
- - The new client doesn't need the clean function we use in /Users/luisnovo/dev/projetos/open-notebook/open-notebook/open_notebook/utils.py - surreal_clean - since it already handles its own cleaning when used correctly
-
-This will be a pretty hefty refactoring, but it will be worth it in the end.
-
diff --git a/.claude/sessions/oss-136/architecture.md b/.claude/sessions/oss-136/architecture.md
deleted file mode 100644
index d7cc6a6..0000000
--- a/.claude/sessions/oss-136/architecture.md
+++ /dev/null
@@ -1,454 +0,0 @@
-# OSS-136 Epic: Podcast Engine + Background Infrastructure - Architecture
-
-## 🏗️ High-Level System Architecture
-
-### Current State (Before Changes)
-```
-┌─────────────────────────────────────────────────────────────────────────────────────┐
-│ Current System │
-├─────────────────────────────────────────────────────────────────────────────────────┤
-│ Streamlit UI (pages/5_🎙️_Podcasts.py) │
-│ ├─ Complex 15+ field forms │
-│ ├─ Synchronous processing (blocks UI) │
-│ └─ Direct podcast generation call │
-│ │
-│ Domain Layer (open_notebook/plugins/podcasts.py) │
-│ ├─ PodcastConfig (complex model) │
-│ ├─ PodcastEpisode (simple model) │
-│ └─ Direct podcastfy library usage │
-│ │
-│ Database (SurrealDB) │
-│ ├─ podcast_config (schemaless, complex) │
-│ └─ podcast_episode (basic fields) │
-└─────────────────────────────────────────────────────────────────────────────────────┘
-```
-
-### Target State (After Implementation)
-```
-┌─────────────────────────────────────────────────────────────────────────────────────┐
-│ New Podcast Engine System │
-├─────────────────────────────────────────────────────────────────────────────────────┤
-│ Streamlit UI (Simplified) │
-│ ├─ Episode Profile selector (3-click workflow) │
-│ ├─ Basic job status display │
-│ └─ Non-blocking async submission │
-│ │
-│ FastAPI Layer (New) │
-│ ├─ POST /api/podcasts/generate │
-│ ├─ GET /api/podcasts/jobs/{job_id} │
-│ ├─ GET /api/episode-profiles │
-│ └─ GET /api/speaker-profiles │
-│ │
-│ Service Layer (New) │
-│ ├─ PodcastService (async operations) │
-│ ├─ EpisodeProfileService (profile management) │
-│ └─ SpeakerProfileService (speaker management) │
-│ │
-│ Background Processing (New) │
-│ ├─ Surreal-Commands Worker │
-│ ├─ Podcast-Creator Integration │
-│ └─ LangGraph Workflow │
-│ │
-│ Database (Enhanced) │
-│ ├─ episode_profile (new schema) │
-│ ├─ speaker_profile (new schema) │
-│ ├─ podcast_episode (enhanced) │
-│ ├─ command (surreal-commands) │
-│ └─ podcast_config (legacy, for migration) │
-└─────────────────────────────────────────────────────────────────────────────────────┘
-```
-
-## 🔄 Phase-by-Phase Architecture
-
-### Phase 1: Async Foundation (OSS-137)
-
-#### 1.1 Surreal-Commands Integration
-```python
-# New: api/commands/podcast_commands.py
-from surreal_commands import command
-from pydantic import BaseModel
-from typing import Optional
-
-class PodcastGenerationInput(BaseModel):
- notebook_id: str
- episode_profile_name: str
- episode_name: str
- briefing_suffix: Optional[str] = None
-
-class PodcastGenerationOutput(BaseModel):
- success: bool
- episode_id: str
- audio_file_path: Optional[str]
- error_message: Optional[str]
-
-@command("generate_podcast")
-async def generate_podcast_command(
- input_data: PodcastGenerationInput
-) -> PodcastGenerationOutput:
- # Integration with podcast-creator library
- # Return structured results
- pass
-```
-
-#### 1.2 Worker Process Integration
-```bash
-# supervisord.conf addition
-[program:worker]
-command=uv run --env-file .env python -m surreal_commands.worker
-environment=SURREAL_COMMANDS_MODULES="api.commands.podcast_commands"
-stdout_logfile=/dev/stdout
-stderr_logfile=/dev/stderr
-autorestart=true
-```
-
-#### 1.3 FastAPI Job Management
-```python
-# New: api/routers/podcasts.py
-from fastapi import APIRouter, HTTPException
-from surreal_commands import submit_command, get_command_status
-
-router = APIRouter()
-
-@router.post("/podcasts/generate")
-async def generate_podcast(request: PodcastGenerationRequest):
- cmd_id = submit_command(
- "api.commands.podcast_commands",
- "generate_podcast",
- request.model_dump()
- )
- return {"job_id": cmd_id, "status": "submitted"}
-
-@router.get("/podcasts/jobs/{job_id}")
-async def get_podcast_job_status(job_id: str):
- status = await get_command_status(job_id)
- return {"job_id": job_id, "status": status.status, "result": status.result}
-```
-
-### Phase 2: Engine Integration (OSS-138)
-
-#### 2.1 Episode Profile Models
-```python
-# New: open_notebook/domain/podcast.py
-from typing import ClassVar, Optional
-from pydantic import Field
-from open_notebook.domain.base import ObjectModel
-
-class EpisodeProfile(ObjectModel):
- table_name: ClassVar[str] = "episode_profile"
- name: str
- description: Optional[str] = None
- speaker_config: str # Reference to speaker profile
- outline_provider: str
- outline_model: str
- transcript_provider: str
- transcript_model: str
- default_briefing: str
- num_segments: int = Field(default=5)
- migrated_from_podcast_config: Optional[str] = None
-
-class SpeakerProfile(ObjectModel):
- table_name: ClassVar[str] = "speaker_profile"
- name: str
- description: Optional[str] = None
- tts_provider: str
- tts_model: str
- speakers: list # Array of speaker objects
- migrated_from_podcast_config: Optional[str] = None
-
-class PodcastEpisode(ObjectModel):
- table_name: ClassVar[str] = "podcast_episode"
- name: str
- episode_profile: str # Reference to episode profile used
- generation_metadata: dict # Store generation parameters
- text: str
- audio_file: str
- command: Optional[str] = None # Link to surreal-commands job
-```
-
-#### 2.2 Podcast-Creator Integration
-```python
-# Enhanced: api/commands/podcast_commands.py
-from podcast_creator import create_podcast, configure
-from open_notebook.domain.podcast import EpisodeProfile, SpeakerProfile
-from open_notebook.domain.notebook import Notebook
-
-@command("generate_podcast")
-async def generate_podcast_command(
- input_data: PodcastGenerationInput
-) -> PodcastGenerationOutput:
- try:
- # Load episode profile
- episode_profile = await EpisodeProfile.get_by_name(input_data.episode_profile_name)
- speaker_profile = await SpeakerProfile.get_by_name(episode_profile.speaker_config)
-
- # Get notebook context
- notebook = await Notebook.get_by_id(input_data.notebook_id)
- context = await notebook.get_context()
-
- # Configure podcast-creator
- configure("speakers_config", {
- "profiles": {
- speaker_profile.name: {
- "tts_provider": speaker_profile.tts_provider,
- "tts_model": speaker_profile.tts_model,
- "speakers": speaker_profile.speakers
- }
- }
- })
-
- # Generate briefing
- briefing = episode_profile.default_briefing
- if input_data.briefing_suffix:
- briefing += f"\n\n{input_data.briefing_suffix}"
-
- # Create podcast
- result = await create_podcast(
- content=str(context),
- briefing=briefing,
- episode_name=input_data.episode_name,
- output_dir=f"data/podcasts/episodes/{input_data.episode_name}",
- speaker_config=speaker_profile.name,
- outline_provider=episode_profile.outline_provider,
- outline_model=episode_profile.outline_model,
- transcript_provider=episode_profile.transcript_provider,
- transcript_model=episode_profile.transcript_model,
- num_segments=episode_profile.num_segments
- )
-
- # Save episode record
- episode = PodcastEpisode(
- name=input_data.episode_name,
- episode_profile=episode_profile.name,
- generation_metadata={
- "briefing": briefing,
- "context_size": len(str(context)),
- "num_segments": episode_profile.num_segments
- },
- text=str(context),
- audio_file=result["final_output_file_path"]
- )
- await episode.save()
-
- return PodcastGenerationOutput(
- success=True,
- episode_id=episode.id,
- audio_file_path=result["final_output_file_path"]
- )
-
- except Exception as e:
- return PodcastGenerationOutput(
- success=False,
- episode_id=None,
- error_message=str(e)
- )
-```
-
-### Phase 3: UI Modernization (OSS-139)
-
-#### 3.1 Simplified Streamlit Interface
-```python
-# Enhanced: pages/5_🎙️_Podcasts.py
-import asyncio
-import streamlit as st
-from open_notebook.domain.podcast import EpisodeProfile, SpeakerProfile, PodcastEpisode
-from api.podcast_service import PodcastService
-
-# Simple episode profile selector
-episode_profiles = asyncio.run(EpisodeProfile.get_all())
-profile_names = [ep.name for ep in episode_profiles]
-
-selected_profile = st.selectbox("Choose Episode Profile", profile_names)
-episode_name = st.text_input("Episode Name")
-briefing_suffix = st.text_area("Additional Instructions (optional)")
-
-if st.button("Generate Podcast"):
- # Submit async job
- job_id = await PodcastService.submit_generation_job(
- notebook_id=st.session_state.current_notebook_id,
- episode_profile_name=selected_profile,
- episode_name=episode_name,
- briefing_suffix=briefing_suffix
- )
- st.success(f"Podcast generation started. Job ID: {job_id}")
-
-# Display episodes with job status
-episodes = asyncio.run(PodcastEpisode.get_all_with_job_status())
-for episode in episodes:
- with st.container():
- st.write(f"**{episode.name}** - Status: {episode.job_status}")
- if episode.job_status == "completed":
- st.audio(episode.audio_file)
-```
-
-#### 3.2 Episode Profile Management
-```python
-# New: pages/components/episode_profile_manager.py
-class EpisodeProfileManager:
- @staticmethod
- def create_default_profiles():
- """Create default episode profiles for common use cases"""
- profiles = [
- {
- "name": "tech_discussion",
- "description": "Technical discussion between experts",
- "speaker_config": "tech_experts",
- "default_briefing": "Create an engaging technical discussion about the provided content..."
- },
- {
- "name": "solo_expert",
- "description": "Single expert explaining complex topics",
- "speaker_config": "solo_expert",
- "default_briefing": "Explain the content in an accessible, educational way..."
- },
- # More profiles...
- ]
- return profiles
-```
-
-### Phase 4: Data Migration (OSS-141)
-
-#### 4.1 Migration Strategy
-```python
-# New: migrations/7.surrealql (handled by Luis)
-# Create new tables
-DEFINE TABLE episode_profile SCHEMAFULL;
-DEFINE TABLE speaker_profile SCHEMAFULL;
-# ... field definitions
-
-# Migration script (handled by Luis)
-# Translate old podcast_config fields to new format
-# Create default profiles based on common configurations
-```
-
-## 🔗 Component Dependencies & Relationships
-
-### External Dependencies
-```toml
-# pyproject.toml additions
-dependencies = [
- "surreal-commands>=1.0.0",
- "podcast-creator>=0.2.0",
- # ... existing dependencies
-]
-```
-
-### Internal Component Flow
-```
-┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐
-│ Streamlit UI │───▶│ FastAPI │───▶│ Service │
-│ (3-click) │ │ (async) │ │ Layer │
-└─────────────────┘ └─────────────────┘ └─────────────────┘
- │
- ▼
-┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐
-│ SurrealDB │◀───│ Background │◀───│ Surreal- │
-│ (job status) │ │ Worker │ │ Commands │
-└─────────────────┘ └─────────────────┘ └─────────────────┘
- │
- ▼
- ┌─────────────────┐
- │ Podcast- │
- │ Creator │
- │ (LangGraph) │
- └─────────────────┘
-```
-
-## 🎯 Design Patterns & Best Practices
-
-### 1. Async-First Architecture
-- All new components use async/await patterns
-- Consistent with existing codebase patterns
-- Non-blocking UI experience
-
-### 2. Domain-Driven Design
-- Clear separation: Domain models, Service layer, API layer
-- Follows existing `ObjectModel` patterns
-- Consistent with current architecture
-
-### 3. Command Pattern
-- Surreal-commands for background processing
-- Structured input/output models
-- Error handling and status tracking
-
-### 4. Configuration Management
-- Episode Profiles for simplified user experience
-- Speaker Profiles for reusable voice configurations
-- Migration-friendly design
-
-## 📁 File Structure & Modifications
-
-### New Files to Create
-```
-api/
-├── commands/
-│ └── podcast_commands.py # Surreal-commands integration
-├── routers/
-│ └── podcasts.py # FastAPI podcast endpoints
-└── podcast_service.py # Service layer for podcast operations
-
-open_notebook/
-└── domain/
- └── podcast.py # New domain models (Episode/Speaker Profiles)
-
-supervisord.conf # Add worker process configuration
-```
-
-### Files to Modify
-```
-api/main.py # Add podcast router
-pages/5_🎙️_Podcasts.py # Simplified UI implementation
-open_notebook/plugins/podcasts.py # Enhanced with new models
-```
-
-### Files to Migrate (Phase 4)
-```
-migrations/7.surrealql # New schema (handled by Luis)
-migrations/7_down.surrealql # Rollback script
-```
-
-## ⚡ Performance & Scalability
-
-### Async Processing Benefits
-- **Non-blocking UI**: Users can continue working while podcasts generate
-- **Scalable Design**: Foundation for future background processing
-- **Resource Management**: Worker process isolation
-
-### Database Optimization
-- **Structured Schema**: Move from schemaless to schemafull for better performance
-- **Efficient Queries**: Profile-based lookups vs complex configuration parsing
-- **Status Tracking**: Simple relationship-based job status
-
-## 🛡️ Error Handling & Monitoring
-
-### Command Error Handling
-```python
-@command("generate_podcast")
-async def generate_podcast_command(input_data: PodcastGenerationInput):
- try:
- # ... podcast generation logic
- return PodcastGenerationOutput(success=True, ...)
- except ValidationError as e:
- return PodcastGenerationOutput(success=False, error_message=f"Invalid input: {e}")
- except Exception as e:
- logger.error(f"Podcast generation failed: {e}")
- return PodcastGenerationOutput(success=False, error_message=str(e))
-```
-
-### Status Monitoring
-- Command status tracking via surreal-commands
-- Simple UI updates through database relationships
-- Structured error messages for debugging
-
-## 🔄 Migration Strategy
-
-### Backward Compatibility
-- Existing `podcast_config` table remains during migration
-- Gradual migration of user configurations
-- Fallback mechanisms for legacy data
-
-### Data Translation
-- Old configuration fields mapped to new Episode Profile format
-- Default profiles created for common use cases
-- Migration script handles complex configurations
-
-This architecture provides a solid foundation for the podcast engine while maintaining consistency with existing codebase patterns and ensuring a smooth migration path.
\ No newline at end of file
diff --git a/.claude/sessions/oss-136/context.md b/.claude/sessions/oss-136/context.md
deleted file mode 100644
index 2ede0bc..0000000
--- a/.claude/sessions/oss-136/context.md
+++ /dev/null
@@ -1,133 +0,0 @@
-# OSS-136 Epic: Podcast Engine + Background Infrastructure - Context
-
-## 🎯 Project Vision
-Create a proprietary podcast generation engine that serves as Open Notebook's competitive differentiator against Google Notebook LM, while establishing the foundation for all background processing using proven open-source libraries.
-
-## 📋 Current Implementation Analysis
-
-### Existing System (to be replaced)
-- **Technology**: Uses `podcastfy` library (synchronous)
-- **Database**: `podcast_config` (complex 15+ fields) and `podcast_episode` tables
-- **UI**: Complex Streamlit forms with manual field configuration
-- **Processing**: Synchronous - blocks UI during generation
-- **Location**: `open_notebook/plugins/podcasts.py` and `pages/5_🎙️_Podcasts.py`
-
-### Key Current Features
-- Multiple TTS providers (OpenAI, Anthropic, Google, ElevenLabs)
-- Detailed speaker configuration (roles, personalities, voices)
-- Conversation styles and dialogue structures
-- Episode management and audio playback
-
-## 🚀 Strategic Value & Competitive Advantages
-
-### Democratization Impact
-- **User Choice**: Flexible 1-4 speakers vs Google's fixed 2-host format
-- **Model Freedom**: User selects LLM + TTS providers via Esperanto integration
-- **Local Privacy**: Complete support for local audio models and processing
-- **Customization**: Rich speaker personalities, backstories, and editable prompts
-
-### Technical Foundation
-- **Battle-tested Infrastructure**: Proven surreal-commands for background processing
-- **Professional Engine**: Production-ready podcast-creator library with advanced features
-- **Ecosystem Consistency**: LangChain Runnable patterns across all async operations
-- **Scalable Architecture**: Foundation for Content Composer, Deep Research, and future workflows
-
-## 🔄 Implementation Strategy (Updated Based on Clarifications)
-
-### Phase 1: Async Foundation (OSS-137)
-- **Technology**: Surreal-commands integration in same container
-- **Worker**: Single worker using existing supervisord.conf
-- **Processing**: Async job queue with SurrealDB backend
-- **Status**: Simple status via podcast_episode → command relationship
-
-### Phase 2: Engine Integration (OSS-138)
-- **Technology**: Podcast-creator library with Episode Profiles
-- **Migration**: From 15+ fields to simplified 3-click workflow
-- **Compatibility**: Translation of old fields into new system (briefing concatenation)
-- **Profiles**: Default Episode and Speaker profiles for common use cases
-
-### Phase 3: UI Modernization (OSS-139)
-- **Focus**: Simplified Episode Profile selector + basic job status
-- **Approach**: Build UI after async foundation is ready
-- **No**: Real-time updates, WebSockets, complex status tracking
-- **Yes**: Simple page refresh for status updates, preparing for React migration
-
-### Phase 4: Data Migration (OSS-141)
-- **Timing**: Last phase, handled in parallel by Luis
-- **Strategy**: Automatic translation of existing configs to Episode Profiles
-- **Compatibility**: Heavy customizations handled by migration script
-- **Database**: New tables for episode_profile and speaker_profile
-
-## 🔧 Technical Architecture
-
-### New Database Schema (Migration 7)
-```sql
--- episode_profile table
-DEFINE TABLE episode_profile SCHEMAFULL;
-DEFINE FIELD name ON TABLE episode_profile TYPE string;
-DEFINE FIELD description ON TABLE episode_profile TYPE option;
-DEFINE FIELD speaker_config ON TABLE episode_profile TYPE string;
-DEFINE FIELD outline_provider ON TABLE episode_profile TYPE string;
-DEFINE FIELD outline_model ON TABLE episode_profile TYPE string;
-DEFINE FIELD transcript_provider ON TABLE episode_profile TYPE string;
-DEFINE FIELD transcript_model ON TABLE episode_profile TYPE string;
-DEFINE FIELD default_briefing ON TABLE episode_profile TYPE string;
-DEFINE FIELD num_segments ON TABLE episode_profile TYPE int;
-
--- speaker_profile table
-DEFINE TABLE speaker_profile SCHEMAFULL;
-DEFINE FIELD name ON TABLE speaker_profile TYPE string;
-DEFINE FIELD tts_provider ON TABLE speaker_profile TYPE string;
-DEFINE FIELD tts_model ON TABLE speaker_profile TYPE string;
-DEFINE FIELD speakers ON TABLE speaker_profile TYPE array;
-```
-
-### Component Integration
-- **Surreal-Commands**: Async job processing with SurrealDB LIVE queries
-- **Podcast-Creator**: Episode Profiles with LangGraph workflow
-- **FastAPI**: New async endpoints for podcast generation
-- **Streamlit**: Simplified UI with Episode Profile selection
-
-### Worker Architecture
-- **Container**: Same container as main app
-- **Supervisor**: Existing supervisord.conf with new worker service
-- **Scalability**: Single worker only (surreal-commands current limitation)
-- **Processing**: Background job queue with status tracking
-
-## 🎯 Success Metrics
-
-### Technical Metrics
-- **Generation Time**: ~2-3 minutes for professional quality
-- **Concurrency**: Non-blocking UI during generation
-- **Flexibility**: 1-4 speaker support vs Google's 2-host limit
-- **Quality**: Professional podcast output with rich speaker personalities
-
-### User Experience Metrics
-- **Simplicity**: 3-click workflow (profile → name → generate)
-- **Accessibility**: Episode Profiles for non-technical users
-- **Transparency**: Clear job status without complex real-time updates
-- **Flexibility**: Custom profiles for advanced users
-
-## 📝 Implementation Notes
-
-### Constraints
-- **No Tests**: Testing will be handled in separate epic
-- **No Real-time**: Simple refresh-based status updates in Streamlit
-- **Single Worker**: Current surreal-commands limitation
-- **Migration**: Luis will handle DB schema and migration scripts
-
-### Dependencies
-- **Libraries**: surreal-commands and podcast-creator already proven
-- **Integration**: Esperanto for multi-provider support
-- **Infrastructure**: Existing SurrealDB and supervisord setup
-- **Migration**: Database schema changes handled in parallel
-
-### Key Files to Modify/Create
-- `api/routers/podcasts.py` - New FastAPI endpoints
-- `api/podcast_service.py` - Service layer for async operations
-- `pages/5_🎙️_Podcasts.py` - Simplified UI with Episode Profiles
-- `open_notebook/plugins/podcasts.py` - Updated models and logic
-- `supervisord.conf` - Worker process configuration
-- Migration scripts (handled by Luis)
-
-This implementation will establish Open Notebook as a superior alternative to Google Notebook LM while creating a robust foundation for future async processing features.
\ No newline at end of file
diff --git a/.claude/sessions/oss-136/plan.md b/.claude/sessions/oss-136/plan.md
deleted file mode 100644
index e9b6492..0000000
--- a/.claude/sessions/oss-136/plan.md
+++ /dev/null
@@ -1,1795 +0,0 @@
-# OSS-136 Epic: Podcast Engine + Background Infrastructure - Implementation Plan
-
-## Overview
-
-This plan breaks down the implementation of the new podcast engine and background infrastructure into manageable phases of approximately 3-4 hours each. Each phase is designed to be independent, testable, and builds upon the previous phase to create a competitive advantage against Google Notebook LM.
-
-**Total Estimated Time**: 14-16 hours across 4 phases
-**Risk Level**: Medium (new async architecture with proven libraries)
-**Rollback Strategy**: Independent commits for each phase
-**Dependencies**: surreal-commands, podcast-creator (both proven libraries)
-
-**Strategic Goal**: Create 1-4 speaker flexibility vs Google's 2-host limitation with simplified Episode Profile workflow
-
----
-
-## Phase 1: Async Foundation (OSS-137) - 4 hours
-
-Surreal Commands Library: https://github.com/lfnovo/surreal-commands
-Also available in Context7 and on /Users/luisnovo/dev/projetos/surreal-commands/surreal-commands
-
-### 🎯 Goals
-- Integrate surreal-commands for background job processing
-- Create generic command infrastructure with example commands
-- Set up worker process in existing container using supervisord
-- Add Makefile command to start worker in dev environment
-- Establish command-based architecture foundation for all future background processing
-
-### 📁 Files to Create/Change
-1. **NEW**: `commands/example_commands.py` - Generic command examples for testing (moved from /api/commands)
-2. **NEW**: `commands/__init__.py` - Commands module initialization
-3. **NEW**: `api/routers/commands.py` - Generic command execution endpoints
-4. **NEW**: `api/command_service.py` - Generic service layer for command operations
-5. **MODIFY**: `api/main.py` - Add commands router and import commands module
-6. **MODIFY**: `supervisord.conf` - Add worker process
-7. **MODIFY**: `pyproject.toml` - Add surreal-commands dependency
-8. **MODIFY**: `Makefile` - Add worker start/stop/restart commands
-9. **NEW**: `test_commands.sh` - Testing script for manual verification
-
-### 🔧 Specific Implementation Steps
-
-#### 1.1 Add Dependencies
-```toml
-# pyproject.toml - Add to dependencies array
-dependencies = [
- # ... existing dependencies
- "surreal-commands>=1.0.7",
-]
-```
-
-#### 1.2 Create Generic Command Infrastructure
-```python
-# commands/__init__.py
-"""Surreal-commands integration for Open Notebook"""
-
-# commands/example_commands.py
-from surreal_commands import command
-from pydantic import BaseModel
-from typing import Optional, List
-from loguru import logger
-import asyncio
-import time
-
-class TextProcessingInput(BaseModel):
- text: str
- operation: str = "uppercase" # uppercase, lowercase, word_count, reverse
- delay_seconds: Optional[int] = None # For testing async behavior
-
-class TextProcessingOutput(BaseModel):
- success: bool
- original_text: str
- processed_text: Optional[str] = None
- word_count: Optional[int] = None
- processing_time: float
- error_message: Optional[str] = None
-
-class DataAnalysisInput(BaseModel):
- numbers: List[float]
- analysis_type: str = "basic" # basic, detailed
- delay_seconds: Optional[int] = None
-
-class DataAnalysisOutput(BaseModel):
- success: bool
- analysis_type: str
- count: int
- sum: Optional[float] = None
- average: Optional[float] = None
- min_value: Optional[float] = None
- max_value: Optional[float] = None
- processing_time: float
- error_message: Optional[str] = None
-
-@command("process_text", app="open_notebook")
-async def process_text_command(input_data: TextProcessingInput) -> TextProcessingOutput:
- """
- Example command for text processing. Tests basic command functionality
- and demonstrates different processing types.
- """
- start_time = time.time()
-
- try:
- logger.info(f"Processing text with operation: {input_data.operation}")
-
- # Simulate processing delay if specified
- if input_data.delay_seconds:
- await asyncio.sleep(input_data.delay_seconds)
-
- processed_text = None
- word_count = None
-
- if input_data.operation == "uppercase":
- processed_text = input_data.text.upper()
- elif input_data.operation == "lowercase":
- processed_text = input_data.text.lower()
- elif input_data.operation == "reverse":
- processed_text = input_data.text[::-1]
- elif input_data.operation == "word_count":
- word_count = len(input_data.text.split())
- processed_text = f"Word count: {word_count}"
- else:
- raise ValueError(f"Unknown operation: {input_data.operation}")
-
- processing_time = time.time() - start_time
-
- return TextProcessingOutput(
- success=True,
- original_text=input_data.text,
- processed_text=processed_text,
- word_count=word_count,
- processing_time=processing_time
- )
-
- except Exception as e:
- processing_time = time.time() - start_time
- logger.error(f"Text processing failed: {e}")
- return TextProcessingOutput(
- success=False,
- original_text=input_data.text,
- processing_time=processing_time,
- error_message=str(e)
- )
-
-@command("analyze_data", app="open_notebook")
-async def analyze_data_command(input_data: DataAnalysisInput) -> DataAnalysisOutput:
- """
- Example command for data analysis. Tests command with complex input/output
- and demonstrates error handling.
- """
- start_time = time.time()
-
- try:
- logger.info(f"Analyzing {len(input_data.numbers)} numbers with {input_data.analysis_type} analysis")
-
- # Simulate processing delay if specified
- if input_data.delay_seconds:
- await asyncio.sleep(input_data.delay_seconds)
-
- if not input_data.numbers:
- raise ValueError("No numbers provided for analysis")
-
- count = len(input_data.numbers)
- sum_value = sum(input_data.numbers)
- average = sum_value / count
- min_value = min(input_data.numbers)
- max_value = max(input_data.numbers)
-
- processing_time = time.time() - start_time
-
- return DataAnalysisOutput(
- success=True,
- analysis_type=input_data.analysis_type,
- count=count,
- sum=sum_value,
- average=average,
- min_value=min_value,
- max_value=max_value,
- processing_time=processing_time
- )
-
- except Exception as e:
- processing_time = time.time() - start_time
- logger.error(f"Data analysis failed: {e}")
- return DataAnalysisOutput(
- success=False,
- analysis_type=input_data.analysis_type,
- count=0,
- processing_time=processing_time,
- error_message=str(e)
- )
-```
-
-#### 1.3 Create Generic Command Service Layer
-```python
-# api/command_service.py
-from typing import List, Optional, Dict, Any
-from loguru import logger
-from surreal_commands import submit_command, get_command_status
-from api.models import ErrorResponse
-
-class CommandService:
- """Generic service layer for command operations"""
-
- @staticmethod
- async def submit_command_job(
- module_name: str,
- command_name: str,
- command_args: Dict[str, Any],
- context: Optional[Dict[str, Any]] = None
- ) -> str:
- """Submit a generic command job for background processing"""
- try:
- cmd_id = submit_command(
- module_name,
- command_name,
- command_args,
- context=context
- )
- logger.info(f"Submitted command job: {cmd_id} for {module_name}.{command_name}")
- return cmd_id
-
- except Exception as e:
- logger.error(f"Failed to submit command job: {e}")
- raise
-
- @staticmethod
- async def get_command_status(job_id: str) -> Dict[str, Any]:
- """Get status of any command job"""
- try:
- status = await get_command_status(job_id)
- return {
- "job_id": job_id,
- "status": status.status if status else "unknown",
- "result": status.result if status else None,
- "error_message": status.error_message if status else None,
- "created": str(status.created) if status and status.created else None,
- "updated": str(status.updated) if status and status.updated else None,
- "progress": status.progress if status else None
- }
- except Exception as e:
- logger.error(f"Failed to get command status: {e}")
- raise
-
- @staticmethod
- async def list_command_jobs(
- module_filter: Optional[str] = None,
- command_filter: Optional[str] = None,
- status_filter: Optional[str] = None,
- limit: int = 50
- ) -> List[Dict[str, Any]]:
- """List command jobs with optional filtering"""
- # This will be implemented with proper SurrealDB queries
- # For now, return empty list as this is foundation phase
- return []
-
- @staticmethod
- async def cancel_command_job(job_id: str) -> bool:
- """Cancel a running command job"""
- try:
- # Implementation depends on surreal-commands cancellation support
- # For now, just log the attempt
- logger.info(f"Attempting to cancel job: {job_id}")
- return True
- except Exception as e:
- logger.error(f"Failed to cancel command job: {e}")
- raise
-```
-
-#### 1.4 Create Generic Command Endpoints
-```python
-# api/routers/commands.py
-from typing import List, Optional, Dict, Any
-from fastapi import APIRouter, HTTPException, Query
-from pydantic import BaseModel, Field
-from loguru import logger
-
-from api.command_service import CommandService
-from api.models import ErrorResponse
-
-router = APIRouter()
-
-class CommandExecutionRequest(BaseModel):
- command: str = Field(..., description="Command function name (e.g., 'process_text')")
- app: str = Field(..., description="Application name (e.g., 'open_notebook')")
- input: Dict[str, Any] = Field(..., description="Arguments to pass to the command")
-
-class CommandJobResponse(BaseModel):
- job_id: str
- status: str
- message: str
-
-class CommandJobStatusResponse(BaseModel):
- job_id: str
- status: str
- result: Optional[Dict[str, Any]] = None
- error_message: Optional[str] = None
- created: Optional[str] = None
- updated: Optional[str] = None
- progress: Optional[Dict[str, Any]] = None
-
-@router.post("/commands/jobs", response_model=CommandJobResponse)
-async def execute_command(request: CommandExecutionRequest):
- """
- Submit a command for background processing.
- Returns immediately with job ID for status tracking.
- """
- # parameters
- "command": "generate_podcast",
- "app": "open_notebook",
- "input": { "notebook_id": "123", "episode_profile": "tech" }
-
-
-@router.get("/commands/{job_id}", response_model=CommandJobStatusResponse)
-async def get_command_job_status(job_id: str):
- """Get the status of a specific command job"""
- try:
- status_data = await CommandService.get_command_status(job_id)
- return CommandJobStatusResponse(**status_data)
-
- except Exception as e:
- logger.error(f"Error fetching job status: {str(e)}")
- raise HTTPException(
- status_code=500,
- detail=f"Failed to fetch job status: {str(e)}"
- )
-
-@router.get("/commands/jobs", response_model=List[Dict[str, Any]])
-async def list_command_jobs(
- command_filter: Optional[str] = Query(None, description="Filter by command name"),
- status_filter: Optional[str] = Query(None, description="Filter by status"),
- limit: int = Query(50, description="Maximum number of jobs to return")
-):
- """List command jobs with optional filtering"""
- try:
- jobs = await CommandService.list_command_jobs(
- command_filter=command_filter,
- status_filter=status_filter,
- limit=limit
- )
- return jobs
-
- except Exception as e:
- logger.error(f"Error listing command jobs: {str(e)}")
- raise HTTPException(
- status_code=500,
- detail=f"Failed to list command jobs: {str(e)}"
- )
-
-@router.delete("/commands/jobs/{job_id}")
-async def cancel_command_job(job_id: str):
- """Cancel a running command job"""
- try:
- success = await CommandService.cancel_command_job(job_id)
- return {"job_id": job_id, "cancelled": success}
-
- except Exception as e:
- logger.error(f"Error cancelling command job: {str(e)}")
- raise HTTPException(
- status_code=500,
- detail=f"Failed to cancel command job: {str(e)}"
- )
-
-```
-
-#### 1.5 Add Router to Main App
-```python
-# api/main.py - Add import and router
-from api.routers import notebooks, search, models, transformations, notes, embedding, settings, context, sources, insights
-from api.routers import commands as commands_router
-
-# Import commands to register them in the API process
-try:
- import commands.example_commands
- from loguru import logger
- logger.info("Commands imported in API process")
-except Exception as e:
- from loguru import logger
- logger.error(f"Failed to import commands in API process: {e}")
-
-# Add to router includes (after line 31)
-app.include_router(commands_router.router, prefix="/api", tags=["commands"])
-```
-
-#### 1.6 Configure Worker Process
-```bash
-# supervisord.conf - Add after [program:api] section
-[program:worker]
-command=uv run --env-file .env surreal-commands-worker --import-modules commands.example_commands
-stdout_logfile=/dev/stdout
-stdout_logfile_maxbytes=0
-stderr_logfile=/dev/stderr
-stderr_logfile_maxbytes=0
-autorestart=true
-```
-
-#### 1.7 Add Makefile Commands
-```makefile
-# Makefile - Add worker management commands
-.PHONY: worker worker-start worker-stop worker-restart
-
-worker: worker-start
-
-worker-start:
- @echo "Starting surreal-commands worker..."
- uv run --env-file .env surreal-commands-worker --import-modules commands.example_commands
-
-worker-stop:
- @echo "Stopping surreal-commands worker..."
- pkill -f "surreal-commands-worker" || true
-
-worker-restart: worker-stop
- @sleep 2
- @$(MAKE) worker-start
-
-```
-
-### ✅ Testing Strategy
-1. **Dependencies**: Verify surreal-commands installs correctly
-2. **Worker Process**: Test worker starts and registers example commands successfully
-3. **API Endpoints**: Test generic command submission and status retrieval
-4. **Command Execution**: Verify example commands execute and return expected results
-5. **Error Handling**: Test error scenarios and proper error responses
-6. **Async Behavior**: Test commands with delays to verify non-blocking execution
-
-### 🧪 Manual Testing Commands
-```bash
-# 1. Install dependencies
-uv sync
-
-# 2. Start SurrealDB
-make database
-
-# 3. Start API and worker separately for testing
-# Terminal 1: Start API
-make api
-
-# Terminal 2: Start worker
-make worker
-
-# 4. Test example command endpoints (shortcuts)
-curl -X POST "http://localhost:5055/api/commands/jobs" \
- -H "Content-Type: application/json" \
- -d '{
- params
- }
- }'
-
-
-# 6. Check job status (use job_id from responses)
-curl "http://localhost:5055/api/commands/jobs/{job_id}"
-
-# 7. List all command jobs
-curl "http://localhost:5055/api/commands/jobs"
-
-# 8. Test worker with supervisord (production mode)
-docker compose up
-
-# 9. Test Makefile commands
-make worker-start
-make worker-stop
-make worker-restart
-```
-
-### ⚠️ Critical Notes
-- **Worker Process**: Single worker only (surreal-commands current limitation)
-- **Environment Setup**: Ensure SurrealDB is running before starting worker
-- **Testing Required**: Thoroughly test async job submission and status tracking
-- **🛑 STOP**: Request human approval before proceeding to Phase 2
-
----
-
-## Phase 2: Engine Integration (OSS-138) - 4 hours
-
-### 📚 Dependencies
-- Surreal Commands Library: https://github.com/lfnovo/surreal-commands
-- Available in Context7 and on /Users/luisnovo/dev/projetos/surreal-commands/surreal-commands
-- Podcast Creator Library: https://github.com/lfnovo/podcast-creator
-- Available in Context7 and on /Users/luisnovo/dev/projetos/podcast-creator/podcast-creator
-
-### 🎯 Goals
-- Integrate podcast-creator library with Episode Profiles
-- Create domain models for Episode and Speaker profiles
-- Implement real podcast generation with LangGraph workflow
-- Replace placeholder implementation with production-ready engine
-
-### 📁 Files to Create/Change
-1. **NEW**: `open_notebook/domain/podcast.py` - Episode, Speaker, PodcastEpisode models
-2. **NEW**: `api/routers/episode_profiles.py` - Episode profile management endpoints
-3. **NEW**: `api/routers/speaker_profiles.py` - Speaker profile management endpoints
-4. **MODIFY**: `commands/podcast_commands.py` - Real podcast generation implementation
-5. **MODIFY**: `api/main.py` - Add new routers
-6. **DELETE AT THE END**: `plugins/podcasts.py` - Old Podcast module that we are replacing
-
-
-### 🔧 Before you start
-
-Database models have already been created
-
-Referer to the file 7.surrealql to see that has already been created.
-
-
-### 🔧 Specific Implementation Steps
-
-
-#### 2.1 Create Domain Models
-```python
-# open_notebook/domain/podcast.py
-from typing import ClassVar, Optional, List, Dict, Any
-from pydantic import Field, validator
-from open_notebook.domain.base import ObjectModel
-
-class EpisodeProfile(ObjectModel):
- """
- Episode Profile - Simplified podcast configuration.
- Replaces complex 15+ field configuration with user-friendly profiles.
- """
- table_name: ClassVar[str] = "episode_profile"
-
- name: str = Field(..., description="Unique profile name")
- description: Optional[str] = Field(None, description="Profile description")
- speaker_config: str = Field(..., description="Reference to speaker profile name")
- outline_provider: str = Field(..., description="AI provider for outline generation")
- outline_model: str = Field(..., description="AI model for outline generation")
- transcript_provider: str = Field(..., description="AI provider for transcript generation")
- transcript_model: str = Field(..., description="AI model for transcript generation")
- default_briefing: str = Field(..., description="Default briefing template")
- num_segments: int = Field(default=5, description="Number of podcast segments")
-
- @validator('num_segments')
- def validate_segments(cls, v):
- if not 3 <= v <= 20:
- raise ValueError('Number of segments must be between 3 and 20')
- return v
-
- @classmethod
- async def get_by_name(cls, name: str) -> Optional['EpisodeProfile']:
- """Get episode profile by name"""
- from open_notebook.database.repository import repo_query, ensure_record_id
- result = await repo_query(
- "SELECT * FROM episode_profile WHERE name = $name",
- {"name": name}
- )
- if result:
- return cls(**result[0])
- return None
-
-class SpeakerProfile(ObjectModel):
- """
- Speaker Profile - Voice and personality configuration.
- Supports 1-4 speakers for flexible podcast formats.
- """
- table_name: ClassVar[str] = "speaker_profile"
-
- name: str = Field(..., description="Unique profile name")
- description: Optional[str] = Field(None, description="Profile description")
- tts_provider: str = Field(..., description="TTS provider (openai, elevenlabs, etc.)")
- tts_model: str = Field(..., description="TTS model name")
- speakers: List[Dict[str, Any]] = Field(..., description="Array of speaker configurations")
-
- @validator('speakers')
- def validate_speakers(cls, v):
- if not 1 <= len(v) <= 4:
- raise ValueError('Must have between 1 and 4 speakers')
-
- required_fields = ['name', 'voice_id', 'backstory', 'personality']
- for speaker in v:
- for field in required_fields:
- if field not in speaker:
- raise ValueError(f'Speaker missing required field: {field}')
- return v
-
- @classmethod
- async def get_by_name(cls, name: str) -> Optional['SpeakerProfile']:
- """Get speaker profile by name"""
- from open_notebook.database.repository import repo_query
- result = await repo_query(
- "SELECT * FROM speaker_profile WHERE name = $name",
- {"name": name}
- )
- if result:
- return cls(**result[0])
- return None
-
-from surrealdb import RecordID
-
-class PodcastEpisode(ObjectModel):
- """Enhanced PodcastEpisode with job tracking and metadata"""
- table_name: ClassVar[str] = "episode"
-
- name: str
- episode_profile: str = Field(..., description="Episode profile used")
- generation_metadata: Dict[str, Any] = Field(default_factory=dict, description="Generation parameters")
- briefing: str = Field(..., description="Full briefing used for generation")
- text: str = Field(..., description="Source content")
- audio_file: Optional[str] = Field(None, description="Path to generated audio file")
- transcript_file: Optional[str] = Field(None, description="Path to transcript file")
- outline_file: Optional[str] = Field(None, description="Path to outline file")
- command: Optional[Union[str, RecordID]] = Field(None, description="Link to surreal-commands job")
-
- async def get_job_status(self) -> Optional[str]:
- """Get the status of the associated command"""
- if not self.command:
- return None
-
- from surreal_commands import get_command_status
- try:
- status = await get_command_status(self.command)
- return status.status if status else "unknown"
- except Exception:
- return "unknown"
-```
-
-#### 2.2 - Load the episode_profile and speaker_profile objects from SurrealDB into podcast-creator using its configure methods and Create the command
-
-Look for a reference on commands/example_commands.py or look in the surreal-commands documentation for more details on how to create a command
-
-Your command will get the speaker_profile, episode_profile, episode_name, additional_briefing and content as input and will generate the podcast episode
-set output_dir as os.environ.get("DATA_DIR", "/podcasts")
-
-The command will call the generate_podcast method from podcast_creator with the following parameters:
-
-- output_dir
-- episode_profile
-- episode_name
-- additional_briefing
-- content
-
-```python
-
-# commands/podcast_commands.py
-from podcast_creator import configure
-
-# get the profiles
-episode_profiles = await repo_query("select * from episode_profile")
-speaker_profiles = await repo_query("select * from speaker_profile")
-
-# transform the surrealdb array into a dictionary so you can pass them to config like this:
-
-episode_profiles_dict = {profile["name"]: profile for profile in episode_profiles}
-speaker_profiles_dict = {profile["name"]: profile for profile in speaker_profiles}
-
-# Define custom episode profiles
-configure("episode_config", {
- "profiles": episode_profiles_dict
-})
-
-configure("speaker_config", {
- "profiles": speaker_profiles_dict
-})
-
-
-# commands/podcast_commands.py - Replace placeholder with real implementation
-from podcast_creator import create_podcast, configure
-from open_notebook.domain.podcast import EpisodeProfile, SpeakerProfile, PodcastEpisode
-from open_notebook.domain.notebook import Notebook
-from pathlib import Path
-import json
-
-@command("generate_podcast")
-async def generate_podcast_command(
- input_data: PodcastGenerationInput
-) -> PodcastGenerationOutput:
- """
- Real podcast generation using podcast-creator library with Episode Profiles
- """
- try:
- logger.info(f"Starting podcast generation for episode: {input_data.episode_name}")
-
- # 1. Load Episode and Speaker profiles
- episode_profile = await EpisodeProfile.get_by_name(input_data.episode_profile_name)
- speaker_profile = await SpeakerProfile.get_by_name(episode_profile.speaker_config)
-
- # 4. Generate briefing
- briefing = episode_profile.default_briefing
- if input_data.briefing_suffix:
- briefing += f"\n\nAdditional instructions: {input_data.briefing_suffix}"
-
- # 5. Create output directory
- output_dir = Path(f"{os.environ.get('DATA_DIR', '/podcasts')}/episodes/{input_data.episode_name}")
- output_dir.mkdir(parents=True, exist_ok=True)
-
- # 6. Generate podcast using podcast-creator
- result = await create_podcast(
- content=input_data.content,
- briefing=briefing,
- episode_name=input_data.episode_name,
- output_dir=str(output_dir),
- speaker_profile=speaker_profile.name,
- podcast_profile=episode_profile.name,
-
- )
-
- # 7. Save episode record
- episode = PodcastEpisode(
- name=input_data.episode_name,
- episode_profile=episode_profile.model_dump(),
- speaker_profile=speaker_profile.model_dump(),
- briefing=briefing,
- content=str(context),
- audio_file=result.get("final_output_file_path"),
- transcript=result.get("transcript"),
- outline=result.get("outline")
- )
- await episode.save()
-
- logger.info(f"Successfully generated podcast episode: {episode.id}")
-
- return PodcastGenerationOutput(
- success=True,
- episode_id=str(episode.id),
- audio_file_path=result.get("final_output_file_path"),
- )
-
- except Exception as e:
- logger.error(f"Podcast generation failed: {e}")
- return PodcastGenerationOutput(
- success=False,
- error_message=str(e)
- )
-
-```
-
-#### 2.3 - Create the API endpoint for podcast generation and the esrvice that will service the API and submit the command to surreal-commands
-
-POST /podcast/episode
-
-
-
-#### 2.4 Create Profile Management Endpoints
-```python
-# api/routers/episode_profiles.py
-from typing import List
-from fastapi import APIRouter, HTTPException
-from pydantic import BaseModel, Field
-from open_notebook.domain.podcast import EpisodeProfile
-from api.models import ErrorResponse
-
-router = APIRouter()
-
-class EpisodeProfileResponse(BaseModel):
- id: str
- name: str
- description: str
- speaker_config: str
- outline_provider: str
- outline_model: str
- transcript_provider: str
- transcript_model: str
- default_briefing: str
- num_segments: int
-
-@router.get("/episode-profiles", response_model=List[EpisodeProfileResponse])
-async def list_episode_profiles():
- """List all available episode profiles"""
- try:
- profiles = await EpisodeProfile.get_all(order_by="name asc")
- return [
- EpisodeProfileResponse(
- id=profile.id,
- name=profile.name,
- description=profile.description or "",
- speaker_config=profile.speaker_config,
- outline_provider=profile.outline_provider,
- outline_model=profile.outline_model,
- transcript_provider=profile.transcript_provider,
- transcript_model=profile.transcript_model,
- default_briefing=profile.default_briefing,
- num_segments=profile.num_segments
- )
- for profile in profiles
- ]
- except Exception as e:
- raise HTTPException(
- status_code=500,
- detail=f"Failed to fetch episode profiles: {str(e)}"
- )
-
-# api/routers/speaker_profiles.py
-from typing import List, Dict, Any
-from fastapi import APIRouter, HTTPException
-from pydantic import BaseModel
-from open_notebook.domain.podcast import SpeakerProfile
-
-router = APIRouter()
-
-class SpeakerProfileResponse(BaseModel):
- id: str
- name: str
- description: str
- tts_provider: str
- tts_model: str
- speakers: List[Dict[str, Any]]
-
-@router.get("/speaker-profiles", response_model=List[SpeakerProfileResponse])
-async def list_speaker_profiles():
- """List all available speaker profiles"""
- try:
- profiles = await SpeakerProfile.get_all(order_by="name asc")
- return [
- SpeakerProfileResponse(
- id=profile.id,
- name=profile.name,
- description=profile.description or "",
- tts_provider=profile.tts_provider,
- tts_model=profile.tts_model,
- speakers=profile.speakers
- )
- for profile in profiles
- ]
- except Exception as e:
- raise HTTPException(
- status_code=500,
- detail=f"Failed to fetch speaker profiles: {str(e)}"
- )
-```
-
-### ✅ Testing Strategy
-1. **Profile Management**: Test episode and speaker profile CRUD operations
-2. **Real Generation**: Test end-to-end podcast generation through the API -> surreal-commands -> podcast-creator
-3. **Error Handling**: Test various failure scenarios (missing profiles, invalid content)
-4. **Integration**: Verify podcast-creator integration with Episode Profiles
-
-
-### 🧪 Manual Testing Commands
-```bash
-
-# 2. List available profiles
-curl "http://localhost:5055/api/episode-profiles"
-curl "http://localhost:5055/api/speaker-profiles"
-
-# 3. Generate real podcast
-curl -X POST "http://localhost:5055/api/podcasts/episodes" \
- -H "Content-Type: application/json" \
- -d '{
- "episode_profile_name": "tech_discussion",
- "content": "My first episode",
- "episode_name": "my_first_episode"
- "briefing_suffix": "Additional instructions blabla"
- "speaker_profile_name": "tech_experts"
- }'
-
-# 4. Monitor job progress
-curl "http://localhost:5055/api/commands/jobs/{job_id}"
-```
-
-### ⚠️ Critical Notes
-- **Real Audio Generation**: This phase produces actual podcast audio files (~2-3 minutes)
-- **Error Recovery**: Implement proper cleanup on generation failure
-- **🛑 STOP**: Request human approval before proceeding to Phase 3
-
----
-
-## Phase 3: UI Modernization (OSS-139) - 3 hours
-
-### 🎯 Goals
-- Simplify Streamlit UI from 15+ fields to 3-click workflow (Profile → Name → Generate)
-- Display podcast episodes with job status via database relationships
-- Implement non-blocking podcast generation UX
-- Prepare UI foundation for future React migration
-
-### 📁 Files to Create/Change
-1. **MODIFY**: `pages/5_🎙️_Podcasts.py` - Complete UI overhaul (make a backup before starting)
-2. **NEW**: `pages/components/episode_profile_selector.py` - Profile selection component
-3. **NEW**: `pages/components/podcast_status_display.py` - Status display component
-4. **MODIFY**: `pages/stream_app/chat.py` - Update podcast tab integration
-
-### 🔧 Specific Implementation Steps
-
-#### 3.1 Create Profile Selection Component
-```python
-# pages/components/episode_profile_selector.py
-import asyncio
-import streamlit as st
-from typing import List, Optional
-from open_notebook.domain.podcast import EpisodeProfile, SpeakerProfile
-
-class EpisodeProfileSelector:
- """Component for selecting episode profiles with preview"""
-
- @staticmethod
- async def render() -> Optional[str]:
- """Render episode profile selector and return selected profile name"""
-
- # Load available profiles
- profiles = asyncio.run(EpisodeProfile.get_all(order_by="name asc"))
-
- if not profiles:
- st.error("No episode profiles available. Please contact administrator.")
- return None
-
- # Create profile options with descriptions
- profile_options = {}
- for profile in profiles:
- display_name = f"{profile.name} - {profile.description}" if profile.description else profile.name
- profile_options[display_name] = profile.name
-
- # Profile selection
- selected_display = st.selectbox(
- "Choose Episode Profile",
- options=list(profile_options.keys()),
- help="Select a pre-configured podcast style"
- )
-
- if selected_display:
- selected_name = profile_options[selected_display]
- selected_profile = next(p for p in profiles if p.name == selected_name)
-
- # Show profile preview
- with st.expander("📝 Profile Details", expanded=False):
- st.write(f"**Description:** {selected_profile.description or 'No description'}")
- st.write(f"**Speaker Configuration:** {selected_profile.speaker_config}")
- st.write(f"**Segments:** {selected_profile.num_segments}")
- st.write(f"**AI Models:** {selected_profile.outline_provider}/{selected_profile.outline_model} (outline), {selected_profile.transcript_provider}/{selected_profile.transcript_model} (transcript)")
-
- # Show speaker preview
- speaker_profile = asyncio.run(SpeakerProfile.get_by_name(selected_profile.speaker_config))
- if speaker_profile:
- st.write(f"**Speakers ({len(speaker_profile.speakers)}):**")
- for speaker in speaker_profile.speakers:
- st.write(f"- **{speaker['name']}**: {speaker['personality']}")
-
- with st.container():
- st.text_area(
- "Default Briefing:",
- value=selected_profile.default_briefing,
- height=100,
- disabled=True
- )
-
- return selected_name
-
- return None
-```
-
-#### 3.2 Create Status Display Component
-```python
-# pages/components/podcast_status_display.py
-import asyncio
-import streamlit as st
-from typing import List
-from datetime import datetime
-from open_notebook.domain.podcast import PodcastEpisode
-import humanize
-
-class PodcastStatusDisplay:
- """Component for displaying podcast episodes with job status"""
-
- @staticmethod
- async def render(notebook_id: Optional[str] = None):
- """Render podcast episodes with status"""
-
- # Get episodes with job status
- episodes = await PodcastStatusDisplay._get_episodes_with_status(notebook_id)
-
- if not episodes:
- st.info("No podcast episodes found. Generate your first episode above!")
- return
-
- st.subheader(f"📻 Podcast Episodes ({len(episodes)})")
-
- # Group by status for better organization
- status_groups = {
- "completed": [],
- "running": [],
- "failed": [],
- "pending": []
- }
-
- for episode in episodes:
- status = episode.get("job_status", "unknown")
- if status == "completed":
- status_groups["completed"].append(episode)
- elif status in ["running", "processing"]:
- status_groups["running"].append(episode)
- elif status == "failed":
- status_groups["failed"].append(episode)
- else:
- status_groups["pending"].append(episode)
-
- # Display running jobs first
- if status_groups["running"]:
- st.write("🔄 **Currently Processing**")
- for episode in status_groups["running"]:
- PodcastStatusDisplay._render_episode_card(episode, show_audio=False)
-
- # Display completed episodes
- if status_groups["completed"]:
- st.write("✅ **Completed Episodes**")
- for episode in status_groups["completed"]:
- PodcastStatusDisplay._render_episode_card(episode, show_audio=True)
-
- # Display failed jobs
- if status_groups["failed"]:
- st.write("❌ **Failed Episodes**")
- for episode in status_groups["failed"]:
- PodcastStatusDisplay._render_episode_card(episode, show_audio=False)
-
- # Display pending jobs
- if status_groups["pending"]:
- st.write("⏳ **Pending Episodes**")
- for episode in status_groups["pending"]:
- PodcastStatusDisplay._render_episode_card(episode, show_audio=False)
-
- @staticmethod
- def _render_episode_card(episode_data: dict, show_audio: bool = True):
- """Render individual episode card"""
- with st.container():
- st.markdown("---")
-
- col1, col2, col3 = st.columns([3, 1, 1])
-
- with col1:
- status_emoji = {
- "completed": "✅",
- "running": "🔄",
- "failed": "❌",
- "pending": "⏳"
- }.get(episode_data.get("job_status", "unknown"), "❓")
-
- st.write(f"{status_emoji} **{episode_data['name']}**")
- st.caption(f"Profile: {episode_data.get('episode_profile', 'Unknown')}")
-
- with col2:
- if episode_data.get("created"):
- created_date = datetime.fromisoformat(episode_data["created"].replace('Z', '+00:00'))
- st.caption(f"Created: {humanize.naturaltime(created_date)}")
-
- with col3:
- # Refresh button for non-completed episodes
- if episode_data.get("job_status") not in ["completed", "failed"]:
- if st.button("🔄", key=f"refresh_{episode_data['id']}", help="Refresh status"):
- st.rerun()
-
- # Show audio player for completed episodes
- if show_audio and episode_data.get("audio_file"):
- try:
- st.audio(episode_data["audio_file"], format="audio/mpeg")
- except Exception:
- st.error("Audio file not found or corrupted")
-
- # Show error message for failed episodes
- if episode_data.get("job_status") == "failed" and episode_data.get("error_message"):
- st.error(f"Error: {episode_data['error_message']}")
-
- # Show metadata in expander
- with st.expander(f"Details - {episode_data['name']}", expanded=False):
- metadata = episode_data.get("generation_metadata", {})
- if metadata:
- st.json(metadata)
-
- if episode_data.get("briefing"):
- st.text_area(
- "Briefing Used:",
- value=episode_data["briefing"],
- height=100,
- disabled=True,
- key=f"briefing_{episode_data['id']}"
- )
-
- @staticmethod
- async def _get_episodes_with_status(notebook_id: Optional[str] = None) -> List[dict]:
- """Get episodes with their job status"""
- from open_notebook.database.repository import repo_query
-
- # Query episodes with command status
- if notebook_id:
- query = """
- SELECT *,
- command.status AS job_status,
- command.error_message AS error_message
- FROM podcast_episode
- WHERE notebook_id = $notebook_id
- ORDER BY created DESC
- """
- params = {"notebook_id": notebook_id}
- else:
- query = """
- SELECT *,
- command.status AS job_status,
- command.error_message AS error_message
- FROM podcast_episode
- ORDER BY created DESC
- """
- params = {}
-
- result = await repo_query(query, params)
- return result
-```
-
-#### 3.3 Modernize Main Podcast Page
-```python
-# pages/5_🎙️_Podcasts.py - Complete rewrite
-import asyncio
-import streamlit as st
-import nest_asyncio
-from pages.stream_app.utils import setup_page
-from pages.components.episode_profile_selector import EpisodeProfileSelector
-from pages.components.podcast_status_display import PodcastStatusDisplay
-from api.podcast_service import PodcastService, DefaultProfiles
-
-nest_asyncio.apply()
-
-setup_page("🎙️ Podcasts", only_check_mandatory_models=False)
-
-# Page title and description
-st.title("🎙️ Podcast Generator")
-st.markdown("""
-Create professional podcasts from your notebook content using AI-powered Episode Profiles.
-Choose from pre-configured styles or create custom profiles for your unique podcast format.
-""")
-
-# Initialize default profiles if needed
-if st.button("🔧 Initialize Default Profiles", help="Create default episode and speaker profiles"):
- with st.spinner("Creating default profiles..."):
- try:
- asyncio.run(DefaultProfiles.create_default_episode_profiles())
- asyncio.run(DefaultProfiles.create_default_speaker_profiles())
- st.success("✅ Default profiles created successfully!")
- except Exception as e:
- st.error(f"Failed to create default profiles: {e}")
-
-st.markdown("---")
-
-# Main podcast generation section
-st.subheader("🎬 Generate New Episode")
-
-# Check if we have a current notebook
-current_notebook_id = st.session_state.get("current_notebook_id")
-if not current_notebook_id:
- st.warning("⚠️ Please select a notebook first from the main page.")
- st.stop()
-
-col1, col2 = st.columns([2, 1])
-
-with col1:
- # Episode Profile Selection (3-click workflow starts here)
- selected_profile = asyncio.run(EpisodeProfileSelector.render())
-
- if selected_profile:
- # Episode Name Input
- episode_name = st.text_input(
- "Episode Name",
- placeholder="e.g., Tech Discussion on AI Trends",
- help="Choose a descriptive name for your podcast episode"
- )
-
- # Optional briefing suffix
- briefing_suffix = st.text_area(
- "Additional Instructions (Optional)",
- placeholder="Add specific instructions for this episode...",
- height=100,
- help="Customize the briefing for this specific episode"
- )
-
-with col2:
- st.markdown("### 📋 Generation Checklist")
- st.markdown(f"""
- - {'✅' if selected_profile else '⏳'} **Episode Profile**: {selected_profile or 'Not selected'}
- - {'✅' if episode_name else '⏳'} **Episode Name**: {'Set' if episode_name else 'Required'}
- - {'✅' if current_notebook_id else '❌'} **Notebook Content**: {'Available' if current_notebook_id else 'Missing'}
- """)
-
-# Generate button (3-click workflow completion)
-if selected_profile and episode_name and current_notebook_id:
- st.markdown("---")
-
- # Estimated generation time
- st.info("⏱️ **Estimated generation time**: 2-3 minutes for professional quality podcast")
-
- if st.button("🚀 Generate Podcast", type="primary", use_container_width=True):
- with st.spinner("🎙️ Starting podcast generation..."):
- try:
- job_id = asyncio.run(PodcastService.submit_generation_job(
- notebook_id=current_notebook_id,
- episode_profile_name=selected_profile,
- episode_name=episode_name,
- briefing_suffix=briefing_suffix if briefing_suffix.strip() else None
- ))
-
- st.success(f"""
- ✅ **Podcast generation started!**
-
- **Job ID**: `{job_id}`
-
- Your podcast is being generated in the background. You can continue using Open Notebook while it processes.
- The episode will appear in the list below when completed.
- """)
-
- # Auto-refresh to show the new job
- st.rerun()
-
- except Exception as e:
- st.error(f"❌ Failed to start podcast generation: {e}")
-
-st.markdown("---")
-
-# Episodes display section
-asyncio.run(PodcastStatusDisplay.render(current_notebook_id))
-
-# Footer with helpful information
-st.markdown("---")
-with st.expander("ℹ️ How it works", expanded=False):
- st.markdown("""
- ### 🎯 3-Click Podcast Generation
-
- 1. **Choose Profile**: Select from pre-configured episode styles
- 2. **Name Episode**: Give your podcast a descriptive name
- 3. **Generate**: Click generate and continue using Open Notebook
-
- ### 🎨 Episode Profiles
- - **Tech Discussion**: 2 experts discussing technical topics
- - **Solo Expert**: Single expert explaining complex concepts
- - **Business Analysis**: Business-focused panel discussion
-
- ### 🔄 Background Processing
- - Podcasts generate in the background (2-3 minutes)
- - No need to wait - continue your research
- - Refresh the page to see updates
-
- ### 🎧 Professional Quality
- - Multiple AI models for outline and transcript generation
- - High-quality text-to-speech with personality-rich speakers
- - Support for 1-4 speakers (vs Google's 2-speaker limit)
- """)
-```
-
-#### 3.4 Update Chat Integration
-```python
-# pages/stream_app/chat.py - Update podcast tab (lines 76-132)
-with podcast_tab:
- st.markdown("### 🎙️ Quick Podcast Generation")
-
- # Simple profile selector for chat context
- episode_profiles = asyncio.run(EpisodeProfile.get_all())
- if episode_profiles:
- profile_names = [ep.name for ep in episode_profiles]
- selected_template = st.selectbox("Episode Profile", profile_names)
-
- episode_name = st.text_input("Episode Name", key="chat_episode_name")
-
- if episode_name and selected_template:
- if st.button("🚀 Generate from Chat Context"):
- try:
- job_id = asyncio.run(PodcastService.submit_generation_job(
- notebook_id=current_notebook.id,
- episode_profile_name=selected_template,
- episode_name=episode_name,
- briefing_suffix="Generate podcast from current chat context"
- ))
- st.success(f"Podcast generation started! Job ID: {job_id}")
- except Exception as e:
- st.error(f"Failed to generate podcast: {e}")
- else:
- st.warning("No episode profiles available. Please initialize default profiles.")
-
- st.page_link("pages/5_🎙️_Podcasts.py", label="🎙️ Go to Full Podcast Interface")
-```
-
-### ✅ Testing Strategy
-1. **Profile Selection**: Test episode profile selection and preview
-2. **3-Click Workflow**: Verify simplified generation process
-3. **Status Display**: Test job status updates and refresh functionality
-4. **Audio Playback**: Verify completed episodes play correctly
-5. **Error Handling**: Test UI behavior with failed generations
-6. **Chat Integration**: Test quick generation from chat context
-
-### 🧪 Manual Testing Scenarios
-```
-# Test 3-Click Workflow:
-1. Navigate to Podcasts page
-2. Select "tech_discussion" profile
-3. Enter episode name "Test Episode"
-4. Click "Generate Podcast"
-5. Verify job appears in status list
-6. Wait for completion and test audio playback
-
-# Test Status Updates:
-1. Generate multiple episodes
-2. Refresh page to see status updates
-3. Test failed episode error display
-4. Verify completed episodes show audio player
-
-# Test Profile Management:
-1. Initialize default profiles
-2. Verify all profiles load correctly
-3. Test profile preview information
-4. Verify speaker configuration display
-```
-
-### ⚠️ Critical Notes
-- **UI Simplification**: Massive reduction from 15+ fields to 3 clicks
-- **Non-blocking UX**: Users can continue working while podcasts generate
-- **No Real-time Updates**: Simple refresh-based status (preparing for React migration)
-- **Profile Dependencies**: Ensure default profiles are created before first use
-- **Audio Storage**: Verify audio files are accessible from Streamlit
-- **🛑 STOP**: Request human approval before proceeding to Phase 4
-
----
-
-## Phase 4: Data Migration (OSS-141) - 3 hours
-
-### 🎯 Goals
-- Create new database schema for Episode and Speaker profiles
-- Migrate existing podcast_config data to new Episode Profile format
-- Maintain backward compatibility during transition
-- Enable smooth rollback if needed
-
-### 📁 Files to Create/Change
-1. **NEW**: `migrations/7.surrealql` - New schema creation
-2. **NEW**: `migrations/7_down.surrealql` - Rollback script
-3. **NEW**: `api/migration_service.py` - Data migration utilities
-4. **NEW**: `api/routers/migration.py` - Migration management endpoints
-5. **MODIFY**: `api/main.py` - Add migration router
-
-### 🔧 Specific Implementation Steps
-
-#### 4.1 Create New Database Schema
-```sql
--- migrations/7.surrealql
-DEFINE TABLE IF NOT EXISTS episode_profile SCHEMAFULL;
-DEFINE FIELD IF NOT EXISTS name ON TABLE episode_profile TYPE string;
-DEFINE FIELD IF NOT EXISTS description ON TABLE episode_profile TYPE option;
-DEFINE FIELD IF NOT EXISTS speaker_config ON TABLE episode_profile TYPE string;
-DEFINE FIELD IF NOT EXISTS outline_provider ON TABLE episode_profile TYPE string;
-DEFINE FIELD IF NOT EXISTS outline_model ON TABLE episode_profile TYPE string;
-DEFINE FIELD IF NOT EXISTS transcript_provider ON TABLE episode_profile TYPE string;
-DEFINE FIELD IF NOT EXISTS transcript_model ON TABLE episode_profile TYPE string;
-DEFINE FIELD IF NOT EXISTS default_briefing ON TABLE episode_profile TYPE string;
-DEFINE FIELD IF NOT EXISTS num_segments ON TABLE episode_profile TYPE int DEFAULT 5;
-DEFINE FIELD IF NOT EXISTS migrated_from_podcast_config ON TABLE episode_profile TYPE option;
-DEFINE FIELD IF NOT EXISTS created ON TABLE episode_profile TYPE datetime DEFAULT time::now();
-DEFINE FIELD IF NOT EXISTS updated ON TABLE episode_profile TYPE datetime DEFAULT time::now();
-
--- Create Speaker Profile table
-DEFINE TABLE IF NOT EXISTS speaker_profile SCHEMAFULL;
-DEFINE FIELD IF NOT EXISTS name ON TABLE speaker_profile TYPE string;
-DEFINE FIELD IF NOT EXISTS description ON TABLE speaker_profile TYPE option;
-DEFINE FIELD IF NOT EXISTS tts_provider ON TABLE speaker_profile TYPE string;
-DEFINE FIELD IF NOT EXISTS tts_model ON TABLE speaker_profile TYPE string;
-DEFINE FIELD IF NOT EXISTS speakers ON TABLE speaker_profile TYPE array;
-DEFINE FIELD IF NOT EXISTS migrated_from_podcast_config ON TABLE speaker_profile TYPE option;
-DEFINE FIELD IF NOT EXISTS created ON TABLE speaker_profile TYPE datetime DEFAULT time::now();
-DEFINE FIELD IF NOT EXISTS updated ON TABLE speaker_profile TYPE datetime DEFAULT time::now();
-
--- Enhance PodcastEpisode table
-DEFINE TABLE IF NOT EXISTS episode SCHEMAFULL;
-DEFINE FIELD IF NOT EXISTS episode_profile ON TABLE episode TYPE string;
-DEFINE FIELD IF NOT EXISTS generation_metadata ON TABLE episode TYPE object;
-DEFINE FIELD IF NOT EXISTS briefing ON TABLE episode TYPE option;
-DEFINE FIELD IF NOT EXISTS transcript ON TABLE episode TYPE option
-## 📢 Open Notebook is under very active development
-
-> Open Notebook is under active development! We're moving fast and making improvements every week. Your feedback is incredibly valuable to me during this exciting phase and it gives me motivation to keep improving and building this amazing tool. Please feel free to star the project if you find it useful, and don't hesitate to reach out with any questions or suggestions. I'm excited to see how you'll use it and what ideas you'll bring to the project! Let's build something amazing together! 🚀
-
-## About The Project
+## A private, multi-model, 100% local, full-featured alternative to Notebook LM

-An open source, privacy-focused alternative to Google's Notebook LM. Why give Google more of our data when we can take control of our own research workflows?
-
In a world dominated by Artificial Intelligence, having the ability to think 🧠 and acquire new knowledge 💡, is a skill that should not be a privilege for a few, nor restricted to a single provider.
**Open Notebook empowers you to:**
@@ -56,6 +50,21 @@ In a world dominated by Artificial Intelligence, having the ability to think
Learn more about our project at [https://www.open-notebook.ai](https://www.open-notebook.ai)
+---
+
+## ⚠️ IMPORTANT: v1.0 Breaking Changes
+
+**If you're upgrading from a previous version**, please note:
+
+- 🏷️ **Docker tags have changed**: The `latest` tag is now **frozen** at the last Streamlit version
+- 🆕 **Use `v1-latest` tag** for the new React/Next.js version (recommended)
+- 🔌 **Port 5055 required**: You must expose port 5055 for the API to work
+- 📖 **Read the migration guide**: See [MIGRATION.md](MIGRATION.md) for detailed upgrade instructions
+
+**New users**: You can ignore this notice and proceed with the Quick Start below using the `v1-latest-single` tag.
+
+---
+
## 🆚 Open Notebook vs Google Notebook LM
| Feature | Open Notebook | Google Notebook LM | Advantage |
@@ -80,7 +89,7 @@ Learn more about our project at [https://www.open-notebook.ai](https://www.open-
### Built With
-[![Python][Python]][Python-url] [![SurrealDB][SurrealDB]][SurrealDB-url] [![LangChain][LangChain]][LangChain-url] [![Streamlit][Streamlit]][Streamlit-url]
+[![Python][Python]][Python-url] [![Next.js][Next.js]][Next-url] [![React][React]][React-url] [![SurrealDB][SurrealDB]][SurrealDB-url] [![LangChain][LangChain]][LangChain-url]
## 🚀 Quick Start
@@ -99,7 +108,7 @@ docker run -d \
-v ./notebook_data:/app/data \
-v ./surreal_data:/mydata \
-e OPENAI_API_KEY=your_key \
- lfnovo/open_notebook:latest-single
+ lfnovo/open_notebook:v1-latest-single
```
**What gets created:**
@@ -110,7 +119,7 @@ open-notebook/
```
**Access your installation:**
-- **🖥️ Main Interface**: http://localhost:8502 (Streamlit UI)
+- **🖥️ Main Interface**: http://localhost:8502 (Next.js UI)
- **🔧 API Access**: http://localhost:5055 (REST API)
- **📚 API Documentation**: http://localhost:5055/docs (Interactive Swagger UI)
@@ -212,13 +221,13 @@ Thanks to the [Esperanto](https://github.com/lfnovo/esperanto) library, we suppo
## 🗺️ Roadmap
### Upcoming Features
-- **React Frontend**: Modern React-based frontend to replace Streamlit
- **Live Front-End Updates**: Real-time UI updates for smoother experience
- **Async Processing**: Faster UI through asynchronous content processing
- **Cross-Notebook Sources**: Reuse research materials across projects
- **Bookmark Integration**: Connect with your favorite bookmarking apps
### Recently Completed ✅
+- **Next.js Frontend**: Modern React-based frontend with improved performance
- **Comprehensive REST API**: Full programmatic access to all functionality
- **Multi-Model Support**: 16+ AI providers including OpenAI, Anthropic, Ollama, LM Studio
- **Advanced Podcast Generator**: Professional multi-speaker podcasts with Episode Profiles
@@ -240,13 +249,13 @@ See the [open issues](https://github.com/lfnovo/open-notebook/issues) for a full
### Contributing
We welcome contributions! We're especially looking for help with:
-- **Frontend Development**: Help build a modern React-based UI (planned replacement for current Streamlit interface)
+- **Frontend Development**: Help improve our modern Next.js/React UI
- **Testing & Bug Fixes**: Make Open Notebook more robust
- **Feature Development**: Build the coolest research tool together
- **Documentation**: Improve guides and tutorials
-**Current Tech Stack**: Python, FastAPI, SurrealDB, Streamlit
-**Future Roadmap**: React frontend, enhanced real-time updates
+**Current Tech Stack**: Python, FastAPI, Next.js, React, SurrealDB
+**Future Roadmap**: Real-time updates, enhanced async processing
See our [Contributing Guide](CONTRIBUTING.md) for detailed information on how to get started.
@@ -294,8 +303,10 @@ Open Notebook is built on the shoulders of amazing open-source projects:
[linkedin-shield]: https://img.shields.io/badge/-LinkedIn-black.svg?style=for-the-badge&logo=linkedin&colorB=555
[linkedin-url]: https://linkedin.com/in/lfnovo
[product-screenshot]: images/screenshot.png
-[Streamlit]: https://img.shields.io/badge/Streamlit-FF4B4B?style=for-the-badge&logo=streamlit&logoColor=white
-[Streamlit-url]: https://streamlit.io/
+[Next.js]: https://img.shields.io/badge/Next.js-000000?style=for-the-badge&logo=next.js&logoColor=white
+[Next-url]: https://nextjs.org/
+[React]: https://img.shields.io/badge/React-61DAFB?style=for-the-badge&logo=react&logoColor=black
+[React-url]: https://reactjs.org/
[Python]: https://img.shields.io/badge/Python-3776AB?style=for-the-badge&logo=python&logoColor=white
[Python-url]: https://www.python.org/
[LangChain]: https://img.shields.io/badge/LangChain-3A3A3A?style=for-the-badge&logo=chainlink&logoColor=white
diff --git a/api/auth.py b/api/auth.py
index a9d51aa..04895c8 100644
--- a/api/auth.py
+++ b/api/auth.py
@@ -2,7 +2,7 @@ import os
from typing import Optional
from fastapi import HTTPException, Request
-from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
+from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
from starlette.middleware.base import BaseHTTPMiddleware
from starlette.responses import JSONResponse
@@ -27,6 +27,10 @@ class PasswordAuthMiddleware(BaseHTTPMiddleware):
if request.url.path in self.excluded_paths:
return await call_next(request)
+ # Skip authentication for CORS preflight requests (OPTIONS)
+ if request.method == "OPTIONS":
+ return await call_next(request)
+
# Check authorization header
auth_header = request.headers.get("Authorization")
@@ -66,7 +70,7 @@ class PasswordAuthMiddleware(BaseHTTPMiddleware):
security = HTTPBearer(auto_error=False)
-def check_api_password(credentials: HTTPAuthorizationCredentials = None) -> bool:
+def check_api_password(credentials: Optional[HTTPAuthorizationCredentials] = None) -> bool:
"""
Utility function to check API password.
Can be used as a dependency in individual routes if needed.
diff --git a/api/chat_service.py b/api/chat_service.py
new file mode 100644
index 0000000..2d02dcf
--- /dev/null
+++ b/api/chat_service.py
@@ -0,0 +1,172 @@
+"""
+Chat service for API operations.
+Provides async interface for chat functionality.
+"""
+import os
+from typing import Any, Dict, List, Optional
+
+import httpx
+from loguru import logger
+
+
+class ChatService:
+ """Service for chat-related API operations"""
+
+ def __init__(self):
+ self.base_url = os.getenv("API_BASE_URL", "http://127.0.0.1:5055")
+ # Add authentication header if password is set
+ self.headers = {}
+ password = os.getenv("OPEN_NOTEBOOK_PASSWORD")
+ if password:
+ self.headers["Authorization"] = f"Bearer {password}"
+
+ async def get_sessions(self, notebook_id: str) -> List[Dict[str, Any]]:
+ """Get all chat sessions for a notebook"""
+ try:
+ async with httpx.AsyncClient() as client:
+ response = await client.get(
+ f"{self.base_url}/api/chat/sessions",
+ params={"notebook_id": notebook_id},
+ headers=self.headers
+ )
+ response.raise_for_status()
+ return response.json()
+ except Exception as e:
+ logger.error(f"Error fetching chat sessions: {str(e)}")
+ raise
+
+ async def create_session(
+ self,
+ notebook_id: str,
+ title: Optional[str] = None,
+ model_override: Optional[str] = None,
+ ) -> Dict[str, Any]:
+ """Create a new chat session"""
+ try:
+ data: Dict[str, Any] = {"notebook_id": notebook_id}
+ if title is not None:
+ data["title"] = title
+ if model_override is not None:
+ data["model_override"] = model_override
+
+ async with httpx.AsyncClient() as client:
+ response = await client.post(
+ f"{self.base_url}/api/chat/sessions",
+ json=data,
+ headers=self.headers
+ )
+ response.raise_for_status()
+ return response.json()
+ except Exception as e:
+ logger.error(f"Error creating chat session: {str(e)}")
+ raise
+
+ async def get_session(self, session_id: str) -> Dict[str, Any]:
+ """Get a specific session with messages"""
+ try:
+ async with httpx.AsyncClient() as client:
+ response = await client.get(
+ f"{self.base_url}/api/chat/sessions/{session_id}",
+ headers=self.headers
+ )
+ response.raise_for_status()
+ return response.json()
+ except Exception as e:
+ logger.error(f"Error fetching session: {str(e)}")
+ raise
+
+ async def update_session(
+ self,
+ session_id: str,
+ title: Optional[str] = None,
+ model_override: Optional[str] = None,
+ ) -> Dict[str, Any]:
+ """Update session properties"""
+ try:
+ data: Dict[str, Any] = {}
+ if title is not None:
+ data["title"] = title
+ if model_override is not None:
+ data["model_override"] = model_override
+
+ if not data:
+ raise ValueError("At least one field must be provided to update a session")
+
+ async with httpx.AsyncClient() as client:
+ response = await client.put(
+ f"{self.base_url}/api/chat/sessions/{session_id}",
+ json=data,
+ headers=self.headers
+ )
+ response.raise_for_status()
+ return response.json()
+ except Exception as e:
+ logger.error(f"Error updating session: {str(e)}")
+ raise
+
+ async def delete_session(self, session_id: str) -> Dict[str, Any]:
+ """Delete a chat session"""
+ try:
+ async with httpx.AsyncClient() as client:
+ response = await client.delete(
+ f"{self.base_url}/api/chat/sessions/{session_id}",
+ headers=self.headers
+ )
+ response.raise_for_status()
+ return response.json()
+ except Exception as e:
+ logger.error(f"Error deleting session: {str(e)}")
+ raise
+
+ async def execute_chat(
+ self,
+ session_id: str,
+ message: str,
+ context: Dict[str, Any],
+ model_override: Optional[str] = None,
+ ) -> Dict[str, Any]:
+ """Execute a chat request"""
+ try:
+ data = {
+ "session_id": session_id,
+ "message": message,
+ "context": context
+ }
+ if model_override is not None:
+ data["model_override"] = model_override
+
+ async with httpx.AsyncClient(timeout=120.0) as client: # Longer timeout for chat
+ response = await client.post(
+ f"{self.base_url}/api/chat/execute",
+ json=data,
+ headers=self.headers
+ )
+ response.raise_for_status()
+ return response.json()
+ except Exception as e:
+ logger.error(f"Error executing chat: {str(e)}")
+ raise
+
+ async def build_context(self, notebook_id: str, context_config: Dict[str, Any]) -> Dict[str, Any]:
+ """Build context for a notebook"""
+ try:
+ data = {
+ "notebook_id": notebook_id,
+ "context_config": context_config
+ }
+
+ async with httpx.AsyncClient() as client:
+ response = await client.post(
+ f"{self.base_url}/api/chat/context",
+ json=data,
+ headers=self.headers
+ )
+ response.raise_for_status()
+ return response.json()
+ except Exception as e:
+ logger.error(f"Error building context: {str(e)}")
+ raise
+
+
+# Global instance
+chat_service = ChatService()
diff --git a/api/client.py b/api/client.py
index 20d0fdd..2ab2bd3 100644
--- a/api/client.py
+++ b/api/client.py
@@ -4,7 +4,7 @@ This module provides a client interface to interact with the Open Notebook API.
"""
import os
-from typing import Dict, List, Optional
+from typing import Any, Dict, List, Optional, Union
import httpx
from loguru import logger
@@ -24,7 +24,7 @@ class APIClient:
def _make_request(
self, method: str, endpoint: str, timeout: Optional[float] = None, **kwargs
- ) -> Dict:
+ ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
"""Make HTTP request to the API."""
url = f"{self.base_url}{endpoint}"
request_timeout = timeout if timeout is not None else self.timeout
@@ -56,28 +56,29 @@ class APIClient:
# Notebooks API methods
def get_notebooks(
self, archived: Optional[bool] = None, order_by: str = "updated desc"
- ) -> List[Dict]:
+ ) -> List[Dict[Any, Any]]:
"""Get all notebooks."""
- params = {"order_by": order_by}
+ params: Dict[str, Any] = {"order_by": order_by}
if archived is not None:
- params["archived"] = archived
+ params["archived"] = str(archived).lower()
- return self._make_request("GET", "/api/notebooks", params=params)
+ result = self._make_request("GET", "/api/notebooks", params=params)
+ return result if isinstance(result, list) else [result]
- def create_notebook(self, name: str, description: str = "") -> Dict:
+ def create_notebook(self, name: str, description: str = "") -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
"""Create a new notebook."""
data = {"name": name, "description": description}
return self._make_request("POST", "/api/notebooks", json=data)
- def get_notebook(self, notebook_id: str) -> Dict:
+ def get_notebook(self, notebook_id: str) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
"""Get a specific notebook."""
return self._make_request("GET", f"/api/notebooks/{notebook_id}")
- def update_notebook(self, notebook_id: str, **updates) -> Dict:
+ def update_notebook(self, notebook_id: str, **updates) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
"""Update a notebook."""
return self._make_request("PUT", f"/api/notebooks/{notebook_id}", json=updates)
- def delete_notebook(self, notebook_id: str) -> Dict:
+ def delete_notebook(self, notebook_id: str) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
"""Delete a notebook."""
return self._make_request("DELETE", f"/api/notebooks/{notebook_id}")
@@ -90,7 +91,7 @@ class APIClient:
search_sources: bool = True,
search_notes: bool = True,
minimum_score: float = 0.2,
- ) -> Dict:
+ ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
"""Search the knowledge base."""
data = {
"query": query,
@@ -108,7 +109,7 @@ class APIClient:
strategy_model: str,
answer_model: str,
final_answer_model: str,
- ) -> Dict:
+ ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
"""Ask the knowledge base a question (simple, non-streaming)."""
data = {
"question": question,
@@ -122,14 +123,15 @@ class APIClient:
)
# Models API methods
- def get_models(self, model_type: Optional[str] = None) -> List[Dict]:
+ def get_models(self, model_type: Optional[str] = None) -> List[Dict[Any, Any]]:
"""Get all models with optional type filtering."""
params = {}
if model_type:
params["type"] = model_type
- return self._make_request("GET", "/api/models", params=params)
+ result = self._make_request("GET", "/api/models", params=params)
+ return result if isinstance(result, list) else [result]
- def create_model(self, name: str, provider: str, model_type: str) -> Dict:
+ def create_model(self, name: str, provider: str, model_type: str) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
"""Create a new model."""
data = {
"name": name,
@@ -138,22 +140,23 @@ class APIClient:
}
return self._make_request("POST", "/api/models", json=data)
- def delete_model(self, model_id: str) -> Dict:
+ def delete_model(self, model_id: str) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
"""Delete a model."""
return self._make_request("DELETE", f"/api/models/{model_id}")
- def get_default_models(self) -> Dict:
+ def get_default_models(self) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
"""Get default model assignments."""
return self._make_request("GET", "/api/models/defaults")
- def update_default_models(self, **defaults) -> Dict:
+ def update_default_models(self, **defaults) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
"""Update default model assignments."""
return self._make_request("PUT", "/api/models/defaults", json=defaults)
# Transformations API methods
- def get_transformations(self) -> List[Dict]:
+ def get_transformations(self) -> List[Dict[Any, Any]]:
"""Get all transformations."""
- return self._make_request("GET", "/api/transformations")
+ result = self._make_request("GET", "/api/transformations")
+ return result if isinstance(result, list) else [result]
def create_transformation(
self,
@@ -162,7 +165,7 @@ class APIClient:
description: str,
prompt: str,
apply_default: bool = False,
- ) -> Dict:
+ ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
"""Create a new transformation."""
data = {
"name": name,
@@ -173,23 +176,23 @@ class APIClient:
}
return self._make_request("POST", "/api/transformations", json=data)
- def get_transformation(self, transformation_id: str) -> Dict:
+ def get_transformation(self, transformation_id: str) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
"""Get a specific transformation."""
return self._make_request("GET", f"/api/transformations/{transformation_id}")
- def update_transformation(self, transformation_id: str, **updates) -> Dict:
+ def update_transformation(self, transformation_id: str, **updates) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
"""Update a transformation."""
return self._make_request(
"PUT", f"/api/transformations/{transformation_id}", json=updates
)
- def delete_transformation(self, transformation_id: str) -> Dict:
+ def delete_transformation(self, transformation_id: str) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
"""Delete a transformation."""
return self._make_request("DELETE", f"/api/transformations/{transformation_id}")
def execute_transformation(
self, transformation_id: str, input_text: str, model_id: str
- ) -> Dict:
+ ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
"""Execute a transformation on input text."""
data = {
"transformation_id": transformation_id,
@@ -202,12 +205,13 @@ class APIClient:
)
# Notes API methods
- def get_notes(self, notebook_id: Optional[str] = None) -> List[Dict]:
+ def get_notes(self, notebook_id: Optional[str] = None) -> List[Dict[Any, Any]]:
"""Get all notes with optional notebook filtering."""
params = {}
if notebook_id:
params["notebook_id"] = notebook_id
- return self._make_request("GET", "/api/notes", params=params)
+ result = self._make_request("GET", "/api/notes", params=params)
+ return result if isinstance(result, list) else [result]
def create_note(
self,
@@ -215,7 +219,7 @@ class APIClient:
title: Optional[str] = None,
note_type: str = "human",
notebook_id: Optional[str] = None,
- ) -> Dict:
+ ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
"""Create a new note."""
data = {
"content": content,
@@ -227,61 +231,86 @@ class APIClient:
data["notebook_id"] = notebook_id
return self._make_request("POST", "/api/notes", json=data)
- def get_note(self, note_id: str) -> Dict:
+ def get_note(self, note_id: str) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
"""Get a specific note."""
return self._make_request("GET", f"/api/notes/{note_id}")
- def update_note(self, note_id: str, **updates) -> Dict:
+ def update_note(self, note_id: str, **updates) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
"""Update a note."""
return self._make_request("PUT", f"/api/notes/{note_id}", json=updates)
- def delete_note(self, note_id: str) -> Dict:
+ def delete_note(self, note_id: str) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
"""Delete a note."""
return self._make_request("DELETE", f"/api/notes/{note_id}")
# Embedding API methods
- def embed_content(self, item_id: str, item_type: str) -> Dict:
+ def embed_content(self, item_id: str, item_type: str, async_processing: bool = False) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
"""Embed content for vector search."""
data = {
"item_id": item_id,
"item_type": item_type,
+ "async_processing": async_processing,
}
# Use extended timeout for embedding operations
return self._make_request("POST", "/api/embed", json=data, timeout=120.0)
+ def rebuild_embeddings(
+ self,
+ mode: str = "existing",
+ include_sources: bool = True,
+ include_notes: bool = True,
+ include_insights: bool = True
+ ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+ """Rebuild embeddings in bulk."""
+ data = {
+ "mode": mode,
+ "include_sources": include_sources,
+ "include_notes": include_notes,
+ "include_insights": include_insights,
+ }
+ # Use extended timeout for rebuild operations (up to 10 minutes)
+ return self._make_request("POST", "/api/embeddings/rebuild", json=data, timeout=600.0)
+
+ def get_rebuild_status(self, command_id: str) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+ """Get status of a rebuild operation."""
+ return self._make_request("GET", f"/api/embeddings/rebuild/{command_id}/status")
+
# Settings API methods
- def get_settings(self) -> Dict:
+ def get_settings(self) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
"""Get all application settings."""
return self._make_request("GET", "/api/settings")
- def update_settings(self, **settings) -> Dict:
+ def update_settings(self, **settings) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
"""Update application settings."""
return self._make_request("PUT", "/api/settings", json=settings)
# Context API methods
def get_notebook_context(
self, notebook_id: str, context_config: Optional[Dict] = None
- ) -> Dict:
+ ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
"""Get context for a notebook."""
- data = {"notebook_id": notebook_id}
+ data: Dict[str, Any] = {"notebook_id": notebook_id}
if context_config:
data["context_config"] = context_config
- return self._make_request(
+ result = self._make_request(
"POST", f"/api/notebooks/{notebook_id}/context", json=data
)
+ return result if isinstance(result, dict) else {}
# Sources API methods
- def get_sources(self, notebook_id: Optional[str] = None) -> List[Dict]:
+ def get_sources(self, notebook_id: Optional[str] = None) -> List[Dict[Any, Any]]:
"""Get all sources with optional notebook filtering."""
params = {}
if notebook_id:
params["notebook_id"] = notebook_id
- return self._make_request("GET", "/api/sources", params=params)
+ result = self._make_request("GET", "/api/sources", params=params)
+ return result if isinstance(result, list) else [result]
def create_source(
self,
- notebook_id: str,
- source_type: str,
+ notebook_id: Optional[str] = None,
+ notebooks: Optional[List[str]] = None,
+ source_type: str = "text",
url: Optional[str] = None,
file_path: Optional[str] = None,
content: Optional[str] = None,
@@ -289,14 +318,24 @@ class APIClient:
transformations: Optional[List[str]] = None,
embed: bool = False,
delete_source: bool = False,
- ) -> Dict:
+ async_processing: bool = False,
+ ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
"""Create a new source."""
data = {
- "notebook_id": notebook_id,
"type": source_type,
"embed": embed,
"delete_source": delete_source,
+ "async_processing": async_processing,
}
+
+ # Handle backward compatibility for notebook_id vs notebooks
+ if notebooks:
+ data["notebooks"] = notebooks
+ elif notebook_id:
+ data["notebook_id"] = notebook_id
+ else:
+ raise ValueError("Either notebook_id or notebooks must be provided")
+
if url:
data["url"] = url
if file_path:
@@ -308,36 +347,41 @@ class APIClient:
if transformations:
data["transformations"] = transformations
- return self._make_request("POST", "/api/sources", json=data)
+ return self._make_request("POST", "/api/sources/json", json=data)
- def get_source(self, source_id: str) -> Dict:
+ def get_source(self, source_id: str) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
"""Get a specific source."""
return self._make_request("GET", f"/api/sources/{source_id}")
- def update_source(self, source_id: str, **updates) -> Dict:
+ def get_source_status(self, source_id: str) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+ """Get processing status for a source."""
+ return self._make_request("GET", f"/api/sources/{source_id}/status")
+
+ def update_source(self, source_id: str, **updates) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
"""Update a source."""
return self._make_request("PUT", f"/api/sources/{source_id}", json=updates)
- def delete_source(self, source_id: str) -> Dict:
+ def delete_source(self, source_id: str) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
"""Delete a source."""
return self._make_request("DELETE", f"/api/sources/{source_id}")
# Insights API methods
- def get_source_insights(self, source_id: str) -> List[Dict]:
+ def get_source_insights(self, source_id: str) -> List[Dict[Any, Any]]:
"""Get all insights for a specific source."""
- return self._make_request("GET", f"/api/sources/{source_id}/insights")
+ result = self._make_request("GET", f"/api/sources/{source_id}/insights")
+ return result if isinstance(result, list) else [result]
- def get_insight(self, insight_id: str) -> Dict:
+ def get_insight(self, insight_id: str) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
"""Get a specific insight."""
return self._make_request("GET", f"/api/insights/{insight_id}")
- def delete_insight(self, insight_id: str) -> Dict:
+ def delete_insight(self, insight_id: str) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
"""Delete a specific insight."""
return self._make_request("DELETE", f"/api/insights/{insight_id}")
def save_insight_as_note(
self, insight_id: str, notebook_id: Optional[str] = None
- ) -> Dict:
+ ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
"""Convert an insight to a note."""
data = {}
if notebook_id:
@@ -348,7 +392,7 @@ class APIClient:
def create_source_insight(
self, source_id: str, transformation_id: str, model_id: Optional[str] = None
- ) -> Dict:
+ ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
"""Create a new insight for a source by running a transformation."""
data = {"transformation_id": transformation_id}
if model_id:
@@ -358,11 +402,12 @@ class APIClient:
)
# Episode Profiles API methods
- def get_episode_profiles(self) -> List[Dict]:
+ def get_episode_profiles(self) -> List[Dict[Any, Any]]:
"""Get all episode profiles."""
- return self._make_request("GET", "/api/episode-profiles")
+ result = self._make_request("GET", "/api/episode-profiles")
+ return result if isinstance(result, list) else [result]
- def get_episode_profile(self, profile_name: str) -> Dict:
+ def get_episode_profile(self, profile_name: str) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
"""Get a specific episode profile by name."""
return self._make_request("GET", f"/api/episode-profiles/{profile_name}")
@@ -377,7 +422,7 @@ class APIClient:
transcript_model: str = "",
default_briefing: str = "",
num_segments: int = 5,
- ) -> Dict:
+ ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
"""Create a new episode profile."""
data = {
"name": name,
@@ -392,11 +437,11 @@ class APIClient:
}
return self._make_request("POST", "/api/episode-profiles", json=data)
- def update_episode_profile(self, profile_id: str, **updates) -> Dict:
+ def update_episode_profile(self, profile_id: str, **updates) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
"""Update an episode profile."""
return self._make_request("PUT", f"/api/episode-profiles/{profile_id}", json=updates)
- def delete_episode_profile(self, profile_id: str) -> Dict:
+ def delete_episode_profile(self, profile_id: str) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
"""Delete an episode profile."""
return self._make_request("DELETE", f"/api/episode-profiles/{profile_id}")
diff --git a/api/command_service.py b/api/command_service.py
index 3b7f64d..6e9506b 100644
--- a/api/command_service.py
+++ b/api/command_service.py
@@ -3,8 +3,6 @@ from typing import Any, Dict, List, Optional
from loguru import logger
from surreal_commands import get_command_status, submit_command
-from api.models import ErrorResponse
-
class CommandService:
"""Generic service layer for command operations"""
@@ -33,7 +31,9 @@ class CommandService:
command_args, # Input data
)
# Convert RecordID to string if needed
- cmd_id_str = str(cmd_id) if cmd_id else None
+ if not cmd_id:
+ raise ValueError("Failed to get cmd_id from submit_command")
+ cmd_id_str = str(cmd_id)
logger.info(
f"Submitted command job: {cmd_id_str} for {module_name}.{command_name}"
)
diff --git a/api/context_service.py b/api/context_service.py
index 6142177..3f6f63f 100644
--- a/api/context_service.py
+++ b/api/context_service.py
@@ -2,7 +2,7 @@
Context service layer using API.
"""
-from typing import Dict, Optional
+from typing import Any, Dict, List, Optional, Union
from loguru import logger
@@ -11,15 +11,15 @@ from api.client import api_client
class ContextService:
"""Service layer for context operations using API."""
-
+
def __init__(self):
logger.info("Using API for context operations")
-
+
def get_notebook_context(
self,
notebook_id: str,
context_config: Optional[Dict] = None
- ) -> Dict:
+ ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
"""Get context for a notebook."""
result = api_client.get_notebook_context(
notebook_id=notebook_id,
diff --git a/api/embedding_service.py b/api/embedding_service.py
index 9a394f6..b3d4d8e 100644
--- a/api/embedding_service.py
+++ b/api/embedding_service.py
@@ -2,7 +2,7 @@
Embedding service layer using API.
"""
-from typing import Dict
+from typing import Any, Dict, List, Union
from loguru import logger
@@ -11,11 +11,11 @@ from api.client import api_client
class EmbeddingService:
"""Service layer for embedding operations using API."""
-
+
def __init__(self):
logger.info("Using API for embedding operations")
-
- def embed_content(self, item_id: str, item_type: str) -> Dict[str, str]:
+
+ def embed_content(self, item_id: str, item_type: str) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
"""Embed content for vector search."""
result = api_client.embed_content(item_id=item_id, item_type=item_type)
return result
diff --git a/api/episode_profiles_service.py b/api/episode_profiles_service.py
index 196141d..420690e 100644
--- a/api/episode_profiles_service.py
+++ b/api/episode_profiles_service.py
@@ -39,7 +39,8 @@ class EpisodeProfilesService:
def get_episode_profile(self, profile_name: str) -> EpisodeProfile:
"""Get a specific episode profile by name."""
- profile_data = api_client.get_episode_profile(profile_name)
+ profile_response = api_client.get_episode_profile(profile_name)
+ profile_data = profile_response if isinstance(profile_response, dict) else profile_response[0]
profile = EpisodeProfile(
name=profile_data["name"],
description=profile_data.get("description", ""),
@@ -67,7 +68,7 @@ class EpisodeProfilesService:
num_segments: int = 5,
) -> EpisodeProfile:
"""Create a new episode profile."""
- profile_data = api_client.create_episode_profile(
+ profile_response = api_client.create_episode_profile(
name=name,
description=description,
speaker_config=speaker_config,
@@ -78,6 +79,7 @@ class EpisodeProfilesService:
default_briefing=default_briefing,
num_segments=num_segments,
)
+ profile_data = profile_response if isinstance(profile_response, dict) else profile_response[0]
profile = EpisodeProfile(
name=profile_data["name"],
description=profile_data.get("description", ""),
diff --git a/api/insights_service.py b/api/insights_service.py
index 78c9d7e..b435519 100644
--- a/api/insights_service.py
+++ b/api/insights_service.py
@@ -34,7 +34,8 @@ class InsightsService:
def get_insight(self, insight_id: str) -> SourceInsight:
"""Get a specific insight."""
- insight_data = api_client.get_insight(insight_id)
+ insight_response = api_client.get_insight(insight_id)
+ insight_data = insight_response if isinstance(insight_response, dict) else insight_response[0]
insight = SourceInsight(
insight_type=insight_data["insight_type"],
content=insight_data["content"],
@@ -42,8 +43,7 @@ class InsightsService:
insight.id = insight_data["id"]
insight.created = insight_data["created"]
insight.updated = insight_data["updated"]
- # Store source_id as an attribute for easy access
- insight._source_id = insight_data["source_id"]
+ # Note: source_id from API response is not stored; use await insight.get_source() if needed
return insight
def delete_insight(self, insight_id: str) -> bool:
@@ -53,7 +53,8 @@ class InsightsService:
def save_insight_as_note(self, insight_id: str, notebook_id: Optional[str] = None) -> Note:
"""Convert an insight to a note."""
- note_data = api_client.save_insight_as_note(insight_id, notebook_id)
+ note_response = api_client.save_insight_as_note(insight_id, notebook_id)
+ note_data = note_response if isinstance(note_response, dict) else note_response[0]
note = Note(
title=note_data["title"],
content=note_data["content"],
@@ -66,7 +67,8 @@ class InsightsService:
def create_source_insight(self, source_id: str, transformation_id: str, model_id: Optional[str] = None) -> SourceInsight:
"""Create a new insight for a source by running a transformation."""
- insight_data = api_client.create_source_insight(source_id, transformation_id, model_id)
+ insight_response = api_client.create_source_insight(source_id, transformation_id, model_id)
+ insight_data = insight_response if isinstance(insight_response, dict) else insight_response[0]
insight = SourceInsight(
insight_type=insight_data["insight_type"],
content=insight_data["content"],
@@ -74,7 +76,7 @@ class InsightsService:
insight.id = insight_data["id"]
insight.created = insight_data["created"]
insight.updated = insight_data["updated"]
- insight._source_id = insight_data["source_id"]
+ # Note: source_id from API response is not stored; use await insight.get_source() if needed
return insight
diff --git a/api/main.py b/api/main.py
index 4db440a..727705e 100644
--- a/api/main.py
+++ b/api/main.py
@@ -1,11 +1,17 @@
+from contextlib import asynccontextmanager
+
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
+from loguru import logger
from api.auth import PasswordAuthMiddleware
-from api.routers import commands as commands_router
from api.routers import (
+ auth,
+ chat,
+ config,
context,
embedding,
+ embedding_rebuild,
episode_profiles,
insights,
models,
@@ -14,30 +20,70 @@ from api.routers import (
podcasts,
search,
settings,
+ source_chat,
sources,
speaker_profiles,
transformations,
)
+from api.routers import commands as commands_router
+from open_notebook.database.async_migrate import AsyncMigrationManager
# Import commands to register them in the API process
try:
- from loguru import logger
-
- import commands.podcast_commands
logger.info("Commands imported in API process")
except Exception as e:
- from loguru import logger
-
logger.error(f"Failed to import commands in API process: {e}")
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+ """
+ Lifespan event handler for the FastAPI application.
+ Runs database migrations automatically on startup.
+ """
+ # Startup: Run database migrations
+ logger.info("Starting API initialization...")
+
+ try:
+ migration_manager = AsyncMigrationManager()
+ current_version = await migration_manager.get_current_version()
+ logger.info(f"Current database version: {current_version}")
+
+ if await migration_manager.needs_migration():
+ logger.warning("Database migrations are pending. Running migrations...")
+ await migration_manager.run_migration_up()
+ new_version = await migration_manager.get_current_version()
+ logger.success(f"Migrations completed successfully. Database is now at version {new_version}")
+ else:
+ logger.info("Database is already at the latest version. No migrations needed.")
+ except Exception as e:
+ logger.error(f"CRITICAL: Database migration failed: {str(e)}")
+ logger.exception(e)
+ # Fail fast - don't start the API with an outdated database schema
+ raise RuntimeError(f"Failed to run database migrations: {str(e)}") from e
+
+ logger.success("API initialization completed successfully")
+
+ # Yield control to the application
+ yield
+
+ # Shutdown: cleanup if needed
+ logger.info("API shutdown complete")
+
+
app = FastAPI(
title="Open Notebook API",
description="API for Open Notebook - Research Assistant",
version="0.2.2",
+ lifespan=lifespan,
)
-# Add CORS middleware
+# Add password authentication middleware first
+# Exclude /api/auth/status and /api/config from authentication
+app.add_middleware(PasswordAuthMiddleware, excluded_paths=["/", "/health", "/docs", "/openapi.json", "/redoc", "/api/auth/status", "/api/config"])
+
+# Add CORS middleware last (so it processes first)
app.add_middleware(
CORSMiddleware,
allow_origins=["*"], # In production, replace with specific origins
@@ -46,16 +92,16 @@ app.add_middleware(
allow_headers=["*"],
)
-# Add password authentication middleware
-app.add_middleware(PasswordAuthMiddleware)
-
# Include routers
+app.include_router(auth.router, prefix="/api", tags=["auth"])
+app.include_router(config.router, prefix="/api", tags=["config"])
app.include_router(notebooks.router, prefix="/api", tags=["notebooks"])
app.include_router(search.router, prefix="/api", tags=["search"])
app.include_router(models.router, prefix="/api", tags=["models"])
app.include_router(transformations.router, prefix="/api", tags=["transformations"])
app.include_router(notes.router, prefix="/api", tags=["notes"])
app.include_router(embedding.router, prefix="/api", tags=["embedding"])
+app.include_router(embedding_rebuild.router, prefix="/api/embeddings", tags=["embeddings"])
app.include_router(settings.router, prefix="/api", tags=["settings"])
app.include_router(context.router, prefix="/api", tags=["context"])
app.include_router(sources.router, prefix="/api", tags=["sources"])
@@ -64,6 +110,8 @@ app.include_router(commands_router.router, prefix="/api", tags=["commands"])
app.include_router(podcasts.router, prefix="/api", tags=["podcasts"])
app.include_router(episode_profiles.router, prefix="/api", tags=["episode-profiles"])
app.include_router(speaker_profiles.router, prefix="/api", tags=["speaker-profiles"])
+app.include_router(chat.router, prefix="/api", tags=["chat"])
+app.include_router(source_chat.router, prefix="/api", tags=["source-chat"])
@app.get("/")
diff --git a/api/models.py b/api/models.py
index a648ebf..4bbea3b 100644
--- a/api/models.py
+++ b/api/models.py
@@ -1,5 +1,6 @@
from typing import Any, Dict, List, Literal, Optional
-from pydantic import BaseModel, Field, ConfigDict
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
# Notebook models
@@ -11,7 +12,9 @@ class NotebookCreate(BaseModel):
class NotebookUpdate(BaseModel):
name: Optional[str] = Field(None, description="Name of the notebook")
description: Optional[str] = Field(None, description="Description of the notebook")
- archived: Optional[bool] = Field(None, description="Whether the notebook is archived")
+ archived: Optional[bool] = Field(
+ None, description="Whether the notebook is archived"
+ )
class NotebookResponse(BaseModel):
@@ -30,7 +33,9 @@ class SearchRequest(BaseModel):
limit: int = Field(100, description="Maximum number of results", le=1000)
search_sources: bool = Field(True, description="Include sources in search")
search_notes: bool = Field(True, description="Include notes in search")
- minimum_score: float = Field(0.2, description="Minimum score for vector search", ge=0, le=1)
+ minimum_score: float = Field(
+ 0.2, description="Minimum score for vector search", ge=0, le=1
+ )
class SearchResponse(BaseModel):
@@ -53,9 +58,14 @@ class AskResponse(BaseModel):
# Models API models
class ModelCreate(BaseModel):
- name: str = Field(..., description="Model name (e.g., gpt-4o-mini, claude, gemini)")
- provider: str = Field(..., description="Provider name (e.g., openai, anthropic, gemini)")
- type: str = Field(..., description="Model type (language, embedding, text_to_speech, speech_to_text)")
+ name: str = Field(..., description="Model name (e.g., gpt-5-mini, claude, gemini)")
+ provider: str = Field(
+ ..., description="Provider name (e.g., openai, anthropic, gemini)"
+ )
+ type: str = Field(
+ ...,
+ description="Model type (language, embedding, text_to_speech, speech_to_text)",
+ )
class ModelResponse(BaseModel):
@@ -77,21 +87,39 @@ class DefaultModelsResponse(BaseModel):
default_tools_model: Optional[str] = None
+class ProviderAvailabilityResponse(BaseModel):
+ available: List[str] = Field(..., description="List of available providers")
+ unavailable: List[str] = Field(..., description="List of unavailable providers")
+ supported_types: Dict[str, List[str]] = Field(
+ ..., description="Provider to supported model types mapping"
+ )
+
+
# Transformations API models
class TransformationCreate(BaseModel):
name: str = Field(..., description="Transformation name")
title: str = Field(..., description="Display title for the transformation")
- description: str = Field(..., description="Description of what this transformation does")
+ description: str = Field(
+ ..., description="Description of what this transformation does"
+ )
prompt: str = Field(..., description="The transformation prompt")
- apply_default: bool = Field(False, description="Whether to apply this transformation by default")
+ apply_default: bool = Field(
+ False, description="Whether to apply this transformation by default"
+ )
class TransformationUpdate(BaseModel):
name: Optional[str] = Field(None, description="Transformation name")
- title: Optional[str] = Field(None, description="Display title for the transformation")
- description: Optional[str] = Field(None, description="Description of what this transformation does")
+ title: Optional[str] = Field(
+ None, description="Display title for the transformation"
+ )
+ description: Optional[str] = Field(
+ None, description="Description of what this transformation does"
+ )
prompt: Optional[str] = Field(None, description="The transformation prompt")
- apply_default: Optional[bool] = Field(None, description="Whether to apply this transformation by default")
+ apply_default: Optional[bool] = Field(
+ None, description="Whether to apply this transformation by default"
+ )
class TransformationResponse(BaseModel):
@@ -107,26 +135,43 @@ class TransformationResponse(BaseModel):
class TransformationExecuteRequest(BaseModel):
model_config = ConfigDict(protected_namespaces=())
-
- transformation_id: str = Field(..., description="ID of the transformation to execute")
+
+ transformation_id: str = Field(
+ ..., description="ID of the transformation to execute"
+ )
input_text: str = Field(..., description="Text to transform")
model_id: str = Field(..., description="Model ID to use for the transformation")
class TransformationExecuteResponse(BaseModel):
model_config = ConfigDict(protected_namespaces=())
-
+
output: str = Field(..., description="Transformed text")
transformation_id: str = Field(..., description="ID of the transformation used")
model_id: str = Field(..., description="Model ID used")
+# Default Prompt API models
+class DefaultPromptResponse(BaseModel):
+ transformation_instructions: str = Field(
+ ..., description="Default transformation instructions"
+ )
+
+
+class DefaultPromptUpdate(BaseModel):
+ transformation_instructions: str = Field(
+ ..., description="Default transformation instructions"
+ )
+
+
# Notes API models
class NoteCreate(BaseModel):
title: Optional[str] = Field(None, description="Note title")
content: str = Field(..., description="Note content")
note_type: Optional[str] = Field("human", description="Type of note (human, ai)")
- notebook_id: Optional[str] = Field(None, description="Notebook ID to add the note to")
+ notebook_id: Optional[str] = Field(
+ None, description="Notebook ID to add the note to"
+ )
class NoteUpdate(BaseModel):
@@ -148,6 +193,9 @@ class NoteResponse(BaseModel):
class EmbedRequest(BaseModel):
item_id: str = Field(..., description="ID of the item to embed")
item_type: str = Field(..., description="Type of item (source, note)")
+ async_processing: bool = Field(
+ False, description="Process asynchronously in background"
+ )
class EmbedResponse(BaseModel):
@@ -155,6 +203,49 @@ class EmbedResponse(BaseModel):
message: str = Field(..., description="Result message")
item_id: str = Field(..., description="ID of the item that was embedded")
item_type: str = Field(..., description="Type of item that was embedded")
+ command_id: Optional[str] = Field(
+ None, description="Command ID for async processing"
+ )
+
+
+# Rebuild request/response models
+class RebuildRequest(BaseModel):
+ mode: Literal["existing", "all"] = Field(
+ ...,
+ description="Rebuild mode: 'existing' only re-embeds items with embeddings, 'all' embeds everything",
+ )
+ include_sources: bool = Field(True, description="Include sources in rebuild")
+ include_notes: bool = Field(True, description="Include notes in rebuild")
+ include_insights: bool = Field(True, description="Include insights in rebuild")
+
+
+class RebuildResponse(BaseModel):
+ command_id: str = Field(..., description="Command ID to track progress")
+ total_items: int = Field(..., description="Estimated number of items to process")
+ message: str = Field(..., description="Status message")
+
+
+class RebuildProgress(BaseModel):
+ processed: int = Field(..., description="Number of items processed")
+ total: int = Field(..., description="Total items to process")
+ percentage: float = Field(..., description="Progress percentage")
+
+
+class RebuildStats(BaseModel):
+ sources: int = Field(0, description="Sources processed")
+ notes: int = Field(0, description="Notes processed")
+ insights: int = Field(0, description="Insights processed")
+ failed: int = Field(0, description="Failed items")
+
+
+class RebuildStatusResponse(BaseModel):
+ command_id: str = Field(..., description="Command ID")
+ status: str = Field(..., description="Status: queued, running, completed, failed")
+ progress: Optional[RebuildProgress] = None
+ stats: Optional[RebuildStats] = None
+ started_at: Optional[str] = None
+ completed_at: Optional[str] = None
+ error_message: Optional[str] = None
# Settings API models
@@ -181,15 +272,50 @@ class AssetModel(BaseModel):
class SourceCreate(BaseModel):
- notebook_id: str = Field(..., description="Notebook ID to add the source to")
+ # Backward compatibility: support old single notebook_id
+ notebook_id: Optional[str] = Field(
+ None, description="Notebook ID to add the source to (deprecated, use notebooks)"
+ )
+ # New multi-notebook support
+ notebooks: Optional[List[str]] = Field(
+ None, description="List of notebook IDs to add the source to"
+ )
+ # Required fields
type: str = Field(..., description="Source type: link, upload, or text")
url: Optional[str] = Field(None, description="URL for link type")
file_path: Optional[str] = Field(None, description="File path for upload type")
content: Optional[str] = Field(None, description="Text content for text type")
title: Optional[str] = Field(None, description="Source title")
- transformations: Optional[List[str]] = Field(default_factory=list, description="Transformation IDs to apply")
+ transformations: Optional[List[str]] = Field(
+ default_factory=list, description="Transformation IDs to apply"
+ )
embed: bool = Field(False, description="Whether to embed content for vector search")
- delete_source: bool = Field(False, description="Whether to delete uploaded file after processing")
+ delete_source: bool = Field(
+ False, description="Whether to delete uploaded file after processing"
+ )
+ # New async processing support
+ async_processing: bool = Field(
+ False, description="Whether to process source asynchronously"
+ )
+
+ @model_validator(mode="after")
+ def validate_notebook_fields(self):
+ # Ensure only one of notebook_id or notebooks is provided
+ if self.notebook_id is not None and self.notebooks is not None:
+ raise ValueError(
+ "Cannot specify both 'notebook_id' and 'notebooks'. Use 'notebooks' for multi-notebook support."
+ )
+
+ # Convert single notebook_id to notebooks array for internal processing
+ if self.notebook_id is not None:
+ self.notebooks = [self.notebook_id]
+ # Keep notebook_id for backward compatibility in response
+
+ # Set empty array if no notebooks specified (allow sources without notebooks)
+ if self.notebooks is None:
+ self.notebooks = []
+
+ return self
class SourceUpdate(BaseModel):
@@ -203,9 +329,15 @@ class SourceResponse(BaseModel):
topics: Optional[List[str]]
asset: Optional[AssetModel]
full_text: Optional[str]
+ embedded: bool
embedded_chunks: int
+ file_available: Optional[bool] = None
created: str
updated: str
+ # New fields for async processing
+ command_id: Optional[str] = None
+ status: Optional[str] = None
+ processing_info: Optional[Dict] = None
class SourceListResponse(BaseModel):
@@ -213,21 +345,33 @@ class SourceListResponse(BaseModel):
title: Optional[str]
topics: Optional[List[str]]
asset: Optional[AssetModel]
- embedded_chunks: int
+ embedded: bool # Boolean flag indicating if source has embeddings
+ embedded_chunks: int # Number of embedded chunks
insights_count: int
created: str
updated: str
+ file_available: Optional[bool] = None
+ # Status fields for async processing
+ command_id: Optional[str] = None
+ status: Optional[str] = None
+ processing_info: Optional[Dict[str, Any]] = None
# Context API models
class ContextConfig(BaseModel):
- sources: Dict[str, str] = Field(default_factory=dict, description="Source inclusion config {source_id: level}")
- notes: Dict[str, str] = Field(default_factory=dict, description="Note inclusion config {note_id: level}")
+ sources: Dict[str, str] = Field(
+ default_factory=dict, description="Source inclusion config {source_id: level}"
+ )
+ notes: Dict[str, str] = Field(
+ default_factory=dict, description="Note inclusion config {note_id: level}"
+ )
class ContextRequest(BaseModel):
notebook_id: str = Field(..., description="Notebook ID to get context for")
- context_config: Optional[ContextConfig] = Field(None, description="Context configuration")
+ context_config: Optional[ContextConfig] = Field(
+ None, description="Context configuration"
+ )
class ContextResponse(BaseModel):
@@ -253,12 +397,24 @@ class SaveAsNoteRequest(BaseModel):
class CreateSourceInsightRequest(BaseModel):
model_config = ConfigDict(protected_namespaces=())
-
+
transformation_id: str = Field(..., description="ID of transformation to apply")
- model_id: Optional[str] = Field(None, description="Model ID (uses default if not provided)")
+ model_id: Optional[str] = Field(
+ None, description="Model ID (uses default if not provided)"
+ )
+
+
+# Source status response
+class SourceStatusResponse(BaseModel):
+ status: Optional[str] = Field(None, description="Processing status")
+ message: str = Field(..., description="Descriptive message about the status")
+ processing_info: Optional[Dict[str, Any]] = Field(
+ None, description="Detailed processing information"
+ )
+ command_id: Optional[str] = Field(None, description="Command ID if available")
# Error response
class ErrorResponse(BaseModel):
error: str
- message: str
\ No newline at end of file
+ message: str
diff --git a/api/models_service.py b/api/models_service.py
index a6f5dcf..8196c61 100644
--- a/api/models_service.py
+++ b/api/models_service.py
@@ -2,7 +2,7 @@
Models service layer using API.
"""
-from typing import Dict, List, Optional
+from typing import List, Optional
from loguru import logger
@@ -35,7 +35,8 @@ class ModelsService:
def create_model(self, name: str, provider: str, model_type: str) -> Model:
"""Create a new model."""
- model_data = api_client.create_model(name, provider, model_type)
+ response = api_client.create_model(name, provider, model_type)
+ model_data = response if isinstance(response, dict) else response[0]
model = Model(
name=model_data["name"],
provider=model_data["provider"],
@@ -53,9 +54,10 @@ class ModelsService:
def get_default_models(self) -> DefaultModels:
"""Get default model assignments."""
- defaults_data = api_client.get_default_models()
+ response = api_client.get_default_models()
+ defaults_data = response if isinstance(response, dict) else response[0]
defaults = DefaultModels()
-
+
# Set the values from API response
defaults.default_chat_model = defaults_data.get("default_chat_model")
defaults.default_transformation_model = defaults_data.get("default_transformation_model")
@@ -64,7 +66,7 @@ class ModelsService:
defaults.default_speech_to_text_model = defaults_data.get("default_speech_to_text_model")
defaults.default_embedding_model = defaults_data.get("default_embedding_model")
defaults.default_tools_model = defaults_data.get("default_tools_model")
-
+
return defaults
def update_default_models(self, defaults: DefaultModels) -> DefaultModels:
@@ -78,9 +80,10 @@ class ModelsService:
"default_embedding_model": defaults.default_embedding_model,
"default_tools_model": defaults.default_tools_model,
}
-
- defaults_data = api_client.update_default_models(**updates)
-
+
+ response = api_client.update_default_models(**updates)
+ defaults_data = response if isinstance(response, dict) else response[0]
+
# Update the defaults object with the response
defaults.default_chat_model = defaults_data.get("default_chat_model")
defaults.default_transformation_model = defaults_data.get("default_transformation_model")
@@ -89,7 +92,7 @@ class ModelsService:
defaults.default_speech_to_text_model = defaults_data.get("default_speech_to_text_model")
defaults.default_embedding_model = defaults_data.get("default_embedding_model")
defaults.default_tools_model = defaults_data.get("default_tools_model")
-
+
return defaults
diff --git a/api/notebook_service.py b/api/notebook_service.py
index f54cf96..340f35e 100644
--- a/api/notebook_service.py
+++ b/api/notebook_service.py
@@ -35,7 +35,8 @@ class NotebookService:
def get_notebook(self, notebook_id: str) -> Optional[Notebook]:
"""Get a specific notebook."""
- nb_data = api_client.get_notebook(notebook_id)
+ response = api_client.get_notebook(notebook_id)
+ nb_data = response if isinstance(response, dict) else response[0]
nb = Notebook(
name=nb_data["name"],
description=nb_data["description"],
@@ -45,10 +46,11 @@ class NotebookService:
nb.created = nb_data["created"]
nb.updated = nb_data["updated"]
return nb
-
+
def create_notebook(self, name: str, description: str = "") -> Notebook:
"""Create a new notebook."""
- nb_data = api_client.create_notebook(name, description)
+ response = api_client.create_notebook(name, description)
+ nb_data = response if isinstance(response, dict) else response[0]
nb = Notebook(
name=nb_data["name"],
description=nb_data["description"],
@@ -66,7 +68,8 @@ class NotebookService:
"description": notebook.description,
"archived": notebook.archived,
}
- nb_data = api_client.update_notebook(notebook.id, **updates)
+ response = api_client.update_notebook(notebook.id or "", **updates)
+ nb_data = response if isinstance(response, dict) else response[0]
# Update the notebook object with the response
notebook.name = nb_data["name"]
notebook.description = nb_data["description"]
@@ -76,7 +79,7 @@ class NotebookService:
def delete_notebook(self, notebook: Notebook) -> bool:
"""Delete a notebook."""
- api_client.delete_notebook(notebook.id)
+ api_client.delete_notebook(notebook.id or "")
return True
diff --git a/api/notes_service.py b/api/notes_service.py
index 0e7344b..d47a37b 100644
--- a/api/notes_service.py
+++ b/api/notes_service.py
@@ -2,7 +2,7 @@
Notes service layer using API.
"""
-from typing import Dict, List, Optional
+from typing import List, Optional
from loguru import logger
@@ -35,7 +35,8 @@ class NotesService:
def get_note(self, note_id: str) -> Note:
"""Get a specific note."""
- note_data = api_client.get_note(note_id)
+ note_response = api_client.get_note(note_id)
+ note_data = note_response if isinstance(note_response, dict) else note_response[0]
note = Note(
title=note_data["title"],
content=note_data["content"],
@@ -54,12 +55,13 @@ class NotesService:
notebook_id: Optional[str] = None
) -> Note:
"""Create a new note."""
- note_data = api_client.create_note(
+ note_response = api_client.create_note(
content=content,
title=title,
note_type=note_type,
notebook_id=notebook_id
)
+ note_data = note_response if isinstance(note_response, dict) else note_response[0]
note = Note(
title=note_data["title"],
content=note_data["content"],
@@ -77,14 +79,15 @@ class NotesService:
"content": note.content,
"note_type": note.note_type,
}
- note_data = api_client.update_note(note.id, **updates)
-
+ note_response = api_client.update_note(note.id or "", **updates)
+ note_data = note_response if isinstance(note_response, dict) else note_response[0]
+
# Update the note object with the response
note.title = note_data["title"]
note.content = note_data["content"]
note.note_type = note_data["note_type"]
note.updated = note_data["updated"]
-
+
return note
def delete_note(self, note_id: str) -> bool:
diff --git a/api/podcast_api_service.py b/api/podcast_api_service.py
index 29a9f12..edf4d84 100644
--- a/api/podcast_api_service.py
+++ b/api/podcast_api_service.py
@@ -3,7 +3,7 @@ Podcast service layer using API client.
This replaces direct httpx calls in the Streamlit pages.
"""
-from typing import Dict, List
+from typing import Any, Dict, List
from loguru import logger
@@ -17,9 +17,10 @@ class PodcastAPIService:
logger.info("Using API client for podcast operations")
# Episode methods
- def get_episodes(self) -> List[Dict]:
+ def get_episodes(self) -> List[Dict[Any, Any]]:
"""Get all podcast episodes."""
- return api_client._make_request("GET", "/api/podcasts/episodes")
+ result = api_client._make_request("GET", "/api/podcasts/episodes")
+ return result if isinstance(result, list) else [result]
def delete_episode(self, episode_id: str) -> bool:
"""Delete a podcast episode."""
@@ -74,9 +75,10 @@ class PodcastAPIService:
return False
# Speaker Profile methods
- def get_speaker_profiles(self) -> List[Dict]:
+ def get_speaker_profiles(self) -> List[Dict[Any, Any]]:
"""Get all speaker profiles."""
- return api_client._make_request("GET", "/api/speaker-profiles")
+ result = api_client._make_request("GET", "/api/speaker-profiles")
+ return result if isinstance(result, list) else [result]
def create_speaker_profile(self, profile_data: Dict) -> bool:
"""Create a new speaker profile."""
diff --git a/api/podcast_service.py b/api/podcast_service.py
index 3041fe1..8bee41e 100644
--- a/api/podcast_service.py
+++ b/api/podcast_service.py
@@ -96,7 +96,9 @@ class PodcastService:
job_id = submit_command("open_notebook", "generate_podcast", command_args)
# Convert RecordID to string if needed
- job_id_str = str(job_id) if job_id else None
+ if not job_id:
+ raise ValueError("Failed to get job_id from submit_command")
+ job_id_str = str(job_id)
logger.info(
f"Submitted podcast generation job: {job_id_str} for episode '{episode_name}'"
)
diff --git a/api/routers/auth.py b/api/routers/auth.py
new file mode 100644
index 0000000..5c35c38
--- /dev/null
+++ b/api/routers/auth.py
@@ -0,0 +1,24 @@
+"""
+Authentication router for Open Notebook API.
+Provides endpoints to check authentication status.
+"""
+
+import os
+
+from fastapi import APIRouter
+
+router = APIRouter(prefix="/auth", tags=["auth"])
+
+
+@router.get("/status")
+async def get_auth_status():
+ """
+ Check if authentication is enabled.
+ Returns whether a password is required to access the API.
+ """
+ auth_enabled = bool(os.environ.get("OPEN_NOTEBOOK_PASSWORD"))
+
+ return {
+ "auth_enabled": auth_enabled,
+ "message": "Authentication is required" if auth_enabled else "Authentication is disabled"
+ }
diff --git a/api/routers/chat.py b/api/routers/chat.py
new file mode 100644
index 0000000..61e1468
--- /dev/null
+++ b/api/routers/chat.py
@@ -0,0 +1,493 @@
+import asyncio
+from typing import Any, Dict, List, Optional
+
+from fastapi import APIRouter, HTTPException, Query
+from langchain_core.runnables import RunnableConfig
+from loguru import logger
+from pydantic import BaseModel, Field
+
+from open_notebook.database.repository import ensure_record_id, repo_query
+from open_notebook.domain.notebook import ChatSession, Note, Notebook, Source
+from open_notebook.exceptions import (
+ NotFoundError,
+)
+from open_notebook.graphs.chat import graph as chat_graph
+
+router = APIRouter()
+
+# Request/Response models
+class CreateSessionRequest(BaseModel):
+ notebook_id: str = Field(..., description="Notebook ID to create session for")
+ title: Optional[str] = Field(None, description="Optional session title")
+ model_override: Optional[str] = Field(
+ None, description="Optional model override for this session"
+ )
+
+
+class UpdateSessionRequest(BaseModel):
+ title: Optional[str] = Field(None, description="New session title")
+ model_override: Optional[str] = Field(
+ None, description="Model override for this session"
+ )
+
+
+class ChatMessage(BaseModel):
+ id: str = Field(..., description="Message ID")
+ type: str = Field(..., description="Message type (human|ai)")
+ content: str = Field(..., description="Message content")
+ timestamp: Optional[str] = Field(None, description="Message timestamp")
+
+
+class ChatSessionResponse(BaseModel):
+ id: str = Field(..., description="Session ID")
+ title: str = Field(..., description="Session title")
+ notebook_id: Optional[str] = Field(None, description="Notebook ID")
+ created: str = Field(..., description="Creation timestamp")
+ updated: str = Field(..., description="Last update timestamp")
+ message_count: Optional[int] = Field(
+ None, description="Number of messages in session"
+ )
+ model_override: Optional[str] = Field(
+ None, description="Model override for this session"
+ )
+
+
+class ChatSessionWithMessagesResponse(ChatSessionResponse):
+ messages: List[ChatMessage] = Field(
+ default_factory=list, description="Session messages"
+ )
+
+
+class ExecuteChatRequest(BaseModel):
+ session_id: str = Field(..., description="Chat session ID")
+ message: str = Field(..., description="User message content")
+ context: Dict[str, Any] = Field(
+ ..., description="Chat context with sources and notes"
+ )
+ model_override: Optional[str] = Field(
+ None, description="Optional model override for this message"
+ )
+
+
+class ExecuteChatResponse(BaseModel):
+ session_id: str = Field(..., description="Session ID")
+ messages: List[ChatMessage] = Field(..., description="Updated message list")
+
+
+class BuildContextRequest(BaseModel):
+ notebook_id: str = Field(..., description="Notebook ID")
+ context_config: Dict[str, Any] = Field(..., description="Context configuration")
+
+
+class BuildContextResponse(BaseModel):
+ context: Dict[str, Any] = Field(..., description="Built context data")
+ token_count: int = Field(..., description="Estimated token count")
+ char_count: int = Field(..., description="Character count")
+
+
+class SuccessResponse(BaseModel):
+ success: bool = Field(True, description="Operation success status")
+ message: str = Field(..., description="Success message")
+
+
+@router.get("/chat/sessions", response_model=List[ChatSessionResponse])
+async def get_sessions(notebook_id: str = Query(..., description="Notebook ID")):
+ """Get all chat sessions for a notebook."""
+ try:
+ # Get notebook to verify it exists
+ notebook = await Notebook.get(notebook_id)
+ if not notebook:
+ raise HTTPException(status_code=404, detail="Notebook not found")
+
+ # Get sessions for this notebook
+ sessions = await notebook.get_chat_sessions()
+
+ return [
+ ChatSessionResponse(
+ id=session.id or "",
+ title=session.title or "Untitled Session",
+ notebook_id=notebook_id,
+ created=str(session.created),
+ updated=str(session.updated),
+ message_count=0, # TODO: Add message count if needed
+ model_override=getattr(session, "model_override", None),
+ )
+ for session in sessions
+ ]
+ except NotFoundError:
+ raise HTTPException(status_code=404, detail="Notebook not found")
+ except Exception as e:
+ logger.error(f"Error fetching chat sessions: {str(e)}")
+ raise HTTPException(
+ status_code=500, detail=f"Error fetching chat sessions: {str(e)}"
+ )
+
+
+@router.post("/chat/sessions", response_model=ChatSessionResponse)
+async def create_session(request: CreateSessionRequest):
+ """Create a new chat session."""
+ try:
+ # Verify notebook exists
+ notebook = await Notebook.get(request.notebook_id)
+ if not notebook:
+ raise HTTPException(status_code=404, detail="Notebook not found")
+
+ # Create new session
+ session = ChatSession(
+ title=request.title or f"Chat Session {asyncio.get_event_loop().time():.0f}",
+ model_override=request.model_override,
+ )
+ await session.save()
+
+ # Relate session to notebook
+ await session.relate_to_notebook(request.notebook_id)
+
+ return ChatSessionResponse(
+ id=session.id or "",
+ title=session.title or "",
+ notebook_id=request.notebook_id,
+ created=str(session.created),
+ updated=str(session.updated),
+ message_count=0,
+ model_override=session.model_override,
+ )
+ except NotFoundError:
+ raise HTTPException(status_code=404, detail="Notebook not found")
+ except Exception as e:
+ logger.error(f"Error creating chat session: {str(e)}")
+ raise HTTPException(
+ status_code=500, detail=f"Error creating chat session: {str(e)}"
+ )
+
+
+@router.get(
+ "/chat/sessions/{session_id}", response_model=ChatSessionWithMessagesResponse
+)
+async def get_session(session_id: str):
+ """Get a specific session with its messages."""
+ try:
+ # Get session
+ # Ensure session_id has proper table prefix
+ full_session_id = (
+ session_id
+ if session_id.startswith("chat_session:")
+ else f"chat_session:{session_id}"
+ )
+ session = await ChatSession.get(full_session_id)
+ if not session:
+ raise HTTPException(status_code=404, detail="Session not found")
+
+ # Get session state from LangGraph to retrieve messages
+ thread_state = chat_graph.get_state(
+ config=RunnableConfig(configurable={"thread_id": session_id})
+ )
+
+ # Extract messages from state
+ messages: list[ChatMessage] = []
+ if thread_state and thread_state.values and "messages" in thread_state.values:
+ for msg in thread_state.values["messages"]:
+ messages.append(
+ ChatMessage(
+ id=getattr(msg, "id", f"msg_{len(messages)}"),
+ type=msg.type if hasattr(msg, "type") else "unknown",
+ content=msg.content if hasattr(msg, "content") else str(msg),
+ timestamp=None, # LangChain messages don't have timestamps by default
+ )
+ )
+
+ # Find notebook_id (we need to query the relationship)
+ # Ensure session_id has proper table prefix
+ full_session_id = (
+ session_id
+ if session_id.startswith("chat_session:")
+ else f"chat_session:{session_id}"
+ )
+
+ notebook_query = await repo_query(
+ "SELECT out FROM refers_to WHERE in = $session_id",
+ {"session_id": ensure_record_id(full_session_id)},
+ )
+
+ notebook_id = notebook_query[0]["out"] if notebook_query else None
+
+ if not notebook_id:
+ # This might be an old session created before API migration
+ logger.warning(
+ f"No notebook relationship found for session {session_id} - may be an orphaned session"
+ )
+
+ return ChatSessionWithMessagesResponse(
+ id=session.id or "",
+ title=session.title or "Untitled Session",
+ notebook_id=notebook_id,
+ created=str(session.created),
+ updated=str(session.updated),
+ message_count=len(messages),
+ messages=messages,
+ model_override=getattr(session, "model_override", None),
+ )
+ except NotFoundError:
+ raise HTTPException(status_code=404, detail="Session not found")
+ except Exception as e:
+ logger.error(f"Error fetching session: {str(e)}")
+ raise HTTPException(status_code=500, detail=f"Error fetching session: {str(e)}")
+
+
+@router.put("/chat/sessions/{session_id}", response_model=ChatSessionResponse)
+async def update_session(session_id: str, request: UpdateSessionRequest):
+ """Update session title."""
+ try:
+ # Ensure session_id has proper table prefix
+ full_session_id = (
+ session_id
+ if session_id.startswith("chat_session:")
+ else f"chat_session:{session_id}"
+ )
+ session = await ChatSession.get(full_session_id)
+ if not session:
+ raise HTTPException(status_code=404, detail="Session not found")
+
+ update_data = request.model_dump(exclude_unset=True)
+
+ if "title" in update_data:
+ session.title = update_data["title"]
+
+ if "model_override" in update_data:
+ session.model_override = update_data["model_override"]
+
+ await session.save()
+
+ # Find notebook_id
+ # Ensure session_id has proper table prefix
+ full_session_id = (
+ session_id
+ if session_id.startswith("chat_session:")
+ else f"chat_session:{session_id}"
+ )
+ notebook_query = await repo_query(
+ "SELECT out FROM refers_to WHERE in = $session_id",
+ {"session_id": ensure_record_id(full_session_id)},
+ )
+ notebook_id = notebook_query[0]["out"] if notebook_query else None
+
+ return ChatSessionResponse(
+ id=session.id or "",
+ title=session.title or "",
+ notebook_id=notebook_id,
+ created=str(session.created),
+ updated=str(session.updated),
+ message_count=0,
+ model_override=session.model_override,
+ )
+ except NotFoundError:
+ raise HTTPException(status_code=404, detail="Session not found")
+ except Exception as e:
+ logger.error(f"Error updating session: {str(e)}")
+ raise HTTPException(status_code=500, detail=f"Error updating session: {str(e)}")
+
+
+@router.delete("/chat/sessions/{session_id}", response_model=SuccessResponse)
+async def delete_session(session_id: str):
+ """Delete a chat session."""
+ try:
+ # Ensure session_id has proper table prefix
+ full_session_id = (
+ session_id
+ if session_id.startswith("chat_session:")
+ else f"chat_session:{session_id}"
+ )
+ session = await ChatSession.get(full_session_id)
+ if not session:
+ raise HTTPException(status_code=404, detail="Session not found")
+
+ await session.delete()
+
+ return SuccessResponse(success=True, message="Session deleted successfully")
+ except NotFoundError:
+ raise HTTPException(status_code=404, detail="Session not found")
+ except Exception as e:
+ logger.error(f"Error deleting session: {str(e)}")
+ raise HTTPException(status_code=500, detail=f"Error deleting session: {str(e)}")
+
+
+@router.post("/chat/execute", response_model=ExecuteChatResponse)
+async def execute_chat(request: ExecuteChatRequest):
+ """Execute a chat request and get AI response."""
+ try:
+ # Verify session exists
+ # Ensure session_id has proper table prefix
+ full_session_id = (
+ request.session_id
+ if request.session_id.startswith("chat_session:")
+ else f"chat_session:{request.session_id}"
+ )
+ session = await ChatSession.get(full_session_id)
+ if not session:
+ raise HTTPException(status_code=404, detail="Session not found")
+
+ # Determine model override (per-request override takes precedence over session-level)
+ model_override = (
+ request.model_override
+ if request.model_override is not None
+ else getattr(session, "model_override", None)
+ )
+
+ # Get current state
+ current_state = chat_graph.get_state(
+ config=RunnableConfig(
+ configurable={"thread_id": request.session_id}
+ )
+ )
+
+ # Prepare state for execution
+ state_values = current_state.values if current_state else {}
+ state_values["messages"] = state_values.get("messages", [])
+ state_values["context"] = request.context
+ state_values["model_override"] = model_override
+
+ # Add user message to state
+ from langchain_core.messages import HumanMessage
+
+ user_message = HumanMessage(content=request.message)
+ state_values["messages"].append(user_message)
+
+ # Execute chat graph
+ result = chat_graph.invoke(
+ input=state_values, # type: ignore[arg-type]
+ config=RunnableConfig(
+ configurable={
+ "thread_id": request.session_id,
+ "model_id": model_override,
+ }
+ ),
+ )
+
+ # Update session timestamp
+ await session.save()
+
+ # Convert messages to response format
+ messages: list[ChatMessage] = []
+ for msg in result.get("messages", []):
+ messages.append(
+ ChatMessage(
+ id=getattr(msg, "id", f"msg_{len(messages)}"),
+ type=msg.type if hasattr(msg, "type") else "unknown",
+ content=msg.content if hasattr(msg, "content") else str(msg),
+ timestamp=None,
+ )
+ )
+
+ return ExecuteChatResponse(session_id=request.session_id, messages=messages)
+ except NotFoundError:
+ raise HTTPException(status_code=404, detail="Session not found")
+ except Exception as e:
+ logger.error(f"Error executing chat: {str(e)}")
+ raise HTTPException(status_code=500, detail=f"Error executing chat: {str(e)}")
+
+
+@router.post("/chat/context", response_model=BuildContextResponse)
+async def build_context(request: BuildContextRequest):
+ """Build context for a notebook based on context configuration."""
+ try:
+ # Verify notebook exists
+ notebook = await Notebook.get(request.notebook_id)
+ if not notebook:
+ raise HTTPException(status_code=404, detail="Notebook not found")
+
+ context_data: dict[str, list[dict[str, str]]] = {"sources": [], "notes": []}
+ total_content = ""
+
+ # Process context configuration if provided
+ if request.context_config:
+ # Process sources
+ for source_id, status in request.context_config.get("sources", {}).items():
+ if "not in" in status:
+ continue
+
+ try:
+ # Add table prefix if not present
+ full_source_id = (
+ source_id
+ if source_id.startswith("source:")
+ else f"source:{source_id}"
+ )
+
+ try:
+ source = await Source.get(full_source_id)
+ except Exception:
+ continue
+
+ if "insights" in status:
+ source_context = await source.get_context(context_size="short")
+ context_data["sources"].append(source_context)
+ total_content += str(source_context)
+ elif "full content" in status:
+ source_context = await source.get_context(context_size="long")
+ context_data["sources"].append(source_context)
+ total_content += str(source_context)
+ except Exception as e:
+ logger.warning(f"Error processing source {source_id}: {str(e)}")
+ continue
+
+ # Process notes
+ for note_id, status in request.context_config.get("notes", {}).items():
+ if "not in" in status:
+ continue
+
+ try:
+ # Add table prefix if not present
+ full_note_id = (
+ note_id if note_id.startswith("note:") else f"note:{note_id}"
+ )
+ note = await Note.get(full_note_id)
+ if not note:
+ continue
+
+ if "full content" in status:
+ note_context = note.get_context(context_size="long")
+ context_data["notes"].append(note_context)
+ total_content += str(note_context)
+ except Exception as e:
+ logger.warning(f"Error processing note {note_id}: {str(e)}")
+ continue
+ else:
+ # Default behavior - include all sources and notes with short context
+ sources = await notebook.get_sources()
+ for source in sources:
+ try:
+ source_context = await source.get_context(context_size="short")
+ context_data["sources"].append(source_context)
+ total_content += str(source_context)
+ except Exception as e:
+ logger.warning(f"Error processing source {source.id}: {str(e)}")
+ continue
+
+ notes = await notebook.get_notes()
+ for note in notes:
+ try:
+ note_context = note.get_context(context_size="short")
+ context_data["notes"].append(note_context)
+ total_content += str(note_context)
+ except Exception as e:
+ logger.warning(f"Error processing note {note.id}: {str(e)}")
+ continue
+
+ # Calculate character and token counts
+ char_count = len(total_content)
+ # Use token count utility if available
+ try:
+ from open_notebook.utils import token_count
+
+ estimated_tokens = token_count(total_content) if total_content else 0
+ except ImportError:
+ # Fallback to simple estimation
+ estimated_tokens = char_count // 4
+
+ return BuildContextResponse(
+ context=context_data, token_count=estimated_tokens, char_count=char_count
+ )
+ except HTTPException:
+ raise
+ except Exception as e:
+ logger.error(f"Error building context: {str(e)}")
+ raise HTTPException(status_code=500, detail=f"Error building context: {str(e)}")
diff --git a/api/routers/commands.py b/api/routers/commands.py
index c386c57..264e0d3 100644
--- a/api/routers/commands.py
+++ b/api/routers/commands.py
@@ -1,11 +1,11 @@
-from typing import List, Optional, Dict, Any
+from typing import Any, Dict, List, Optional
+
from fastapi import APIRouter, HTTPException, Query
-from pydantic import BaseModel, Field
from loguru import logger
+from pydantic import BaseModel, Field
+from surreal_commands import registry
from api.command_service import CommandService
-from api.models import ErrorResponse
-from surreal_commands import registry
router = APIRouter()
@@ -136,7 +136,7 @@ async def debug_registry():
# Get the basic command structure
try:
- commands_dict = {}
+ commands_dict: dict[str, list[str]] = {}
for item in all_items:
if item.app_id not in commands_dict:
commands_dict[item.app_id] = []
diff --git a/api/routers/config.py b/api/routers/config.py
new file mode 100644
index 0000000..b999565
--- /dev/null
+++ b/api/routers/config.py
@@ -0,0 +1,176 @@
+import asyncio
+import os
+import time
+import tomllib
+from pathlib import Path
+from typing import Optional
+
+from fastapi import APIRouter, Request
+from loguru import logger
+
+from open_notebook.database.repository import repo_query
+from open_notebook.utils.version_utils import (
+ compare_versions,
+ get_version_from_github,
+)
+
+router = APIRouter()
+
+# In-memory cache for version check results
+_version_cache: dict = {
+ "latest_version": None,
+ "has_update": False,
+ "timestamp": 0,
+ "check_failed": False,
+}
+
+
+def get_version() -> str:
+ """Read version from pyproject.toml"""
+ try:
+ pyproject_path = Path(__file__).parent.parent.parent / "pyproject.toml"
+ with open(pyproject_path, "rb") as f:
+ pyproject = tomllib.load(f)
+ return pyproject.get("project", {}).get("version", "unknown")
+ except Exception as e:
+ logger.warning(f"Could not read version from pyproject.toml: {e}")
+ return "unknown"
+
+
+def get_latest_version_cached(current_version: str) -> tuple[Optional[str], bool]:
+ """
+ Check for the latest version from GitHub with caching.
+
+ Returns:
+ tuple: (latest_version, has_update)
+ - latest_version: str or None if check failed
+ - has_update: bool indicating if update is available
+ """
+ global _version_cache
+
+ # Use cache if available (lives for entire API process lifetime)
+ if _version_cache["timestamp"] > 0:
+ logger.debug("Using cached version check result")
+ return _version_cache["latest_version"], _version_cache["has_update"]
+
+ # Perform version check with strict error handling
+ try:
+ logger.info("Checking for latest version from GitHub...")
+
+ # Fetch latest version from GitHub with 10-second timeout
+ latest_version = get_version_from_github(
+ "https://github.com/lfnovo/open-notebook",
+ "main"
+ )
+
+ logger.info(f"Latest version from GitHub: {latest_version}, Current version: {current_version}")
+
+ # Compare versions
+ has_update = compare_versions(current_version, latest_version) < 0
+
+ # Cache the result
+ _version_cache["latest_version"] = latest_version
+ _version_cache["has_update"] = has_update
+ _version_cache["timestamp"] = time.time()
+ _version_cache["check_failed"] = False
+
+ logger.info(f"Version check complete. Update available: {has_update}")
+
+ return latest_version, has_update
+
+ except Exception as e:
+ logger.warning(f"Version check failed: {e}")
+
+ # Cache the failure to avoid repeated attempts
+ _version_cache["latest_version"] = None
+ _version_cache["has_update"] = False
+ _version_cache["timestamp"] = time.time()
+ _version_cache["check_failed"] = True
+
+ return None, False
+
+
+async def check_database_health() -> dict:
+ """
+ Check if database is reachable using a lightweight query.
+
+ Returns:
+ dict with 'status' ("online" | "offline") and optional 'error'
+ """
+ try:
+ # 2-second timeout for database health check
+ result = await asyncio.wait_for(
+ repo_query("RETURN 1"),
+ timeout=2.0
+ )
+ if result:
+ return {"status": "online"}
+ return {"status": "offline", "error": "Empty result"}
+ except asyncio.TimeoutError:
+ logger.warning("Database health check timed out after 2 seconds")
+ return {"status": "offline", "error": "Health check timeout"}
+ except Exception as e:
+ logger.warning(f"Database health check failed: {e}")
+ return {"status": "offline", "error": str(e)}
+
+
+@router.get("/config")
+async def get_config(request: Request):
+ """
+ Get frontend configuration.
+ This endpoint provides runtime configuration to the frontend,
+ allowing the same Docker image to work in different environments.
+
+ Auto-detection logic:
+ 1. If API_URL env var is set, use it (explicit override)
+ 2. Otherwise, detect from incoming HTTP request (zero-config)
+
+ Also checks for version updates from GitHub (with caching and error handling).
+ """
+ # Check if API_URL is explicitly set
+ env_api_url = os.getenv("API_URL")
+
+ if env_api_url:
+ logger.debug(f"Using API_URL from environment: {env_api_url}")
+ api_url = env_api_url
+ else:
+ # Auto-detect from request
+ # Get the protocol (http or https)
+ # Check X-Forwarded-Proto first (for reverse proxies), then fallback to request scheme
+ proto = request.headers.get("x-forwarded-proto", request.url.scheme)
+
+ # Get the host (includes port if non-standard)
+ host = request.headers.get("host", f"{request.client.host}:5055")
+
+ # Construct the API URL
+ api_url = f"{proto}://{host}"
+ logger.info(f"Auto-detected API URL from request: {api_url} (proto={proto}, host={host})")
+
+ # Get current version
+ current_version = get_version()
+
+ # Check for updates (with caching and error handling)
+ # This MUST NOT break the endpoint - wrapped in try-except as extra safety
+ latest_version = None
+ has_update = False
+
+ try:
+ latest_version, has_update = get_latest_version_cached(current_version)
+ except Exception as e:
+ # Extra safety: ensure version check never breaks the config endpoint
+ logger.error(f"Unexpected error during version check: {e}")
+
+ # Check database health
+ db_health = await check_database_health()
+ db_status = db_health["status"]
+
+ if db_status == "offline":
+ logger.warning(f"Database offline: {db_health.get('error', 'Unknown error')}")
+
+ return {
+ "apiUrl": api_url,
+ "version": current_version,
+ "latestVersion": latest_version,
+ "hasUpdate": has_update,
+ "dbStatus": db_status,
+ }
diff --git a/api/routers/context.py b/api/routers/context.py
index 29e56e2..70cd70f 100644
--- a/api/routers/context.py
+++ b/api/routers/context.py
@@ -1,12 +1,10 @@
-from typing import Dict, List, Union
from fastapi import APIRouter, HTTPException
from loguru import logger
from api.models import ContextRequest, ContextResponse
-from open_notebook.domain.base import ObjectModel
from open_notebook.domain.notebook import Note, Notebook, Source
-from open_notebook.exceptions import DatabaseOperationError, InvalidInputError
+from open_notebook.exceptions import InvalidInputError
from open_notebook.utils import token_count
router = APIRouter()
@@ -21,7 +19,7 @@ async def get_notebook_context(notebook_id: str, context_request: ContextRequest
if not notebook:
raise HTTPException(status_code=404, detail="Notebook not found")
- context_data = {"note": [], "source": []}
+ context_data: dict[str, list[dict[str, str]]] = {"note": [], "source": []}
total_content = ""
# Process context configuration if provided
@@ -41,7 +39,7 @@ async def get_notebook_context(notebook_id: str, context_request: ContextRequest
try:
source = await Source.get(full_source_id)
- except Exception as e:
+ except Exception:
continue
if "insights" in status:
diff --git a/api/routers/embedding.py b/api/routers/embedding.py
index 017d6ac..ecee428 100644
--- a/api/routers/embedding.py
+++ b/api/routers/embedding.py
@@ -1,6 +1,7 @@
from fastapi import APIRouter, HTTPException
from loguru import logger
+from api.command_service import CommandService
from api.models import EmbedRequest, EmbedResponse
from open_notebook.domain.models import model_manager
from open_notebook.domain.notebook import Note, Source
@@ -28,35 +29,63 @@ async def embed_content(embed_request: EmbedRequest):
status_code=400, detail="Item type must be either 'source' or 'note'"
)
- # Get the item and embed it
- if item_type == "source":
- source_item = await Source.get(item_id)
- if not source_item:
- raise HTTPException(status_code=404, detail="Source not found")
+ # Branch based on processing mode
+ if embed_request.async_processing:
+ # ASYNC PATH: Submit command for background processing
+ logger.info(f"Using async processing for {item_type} {item_id}")
- # Check if already embedded
- if await source_item.get_embedded_chunks() > 0:
- return EmbedResponse(
- success=True,
- message="Source is already embedded",
- item_id=item_id,
- item_type=item_type,
+ try:
+ # Import commands to ensure they're registered
+ import commands.embedding_commands # noqa: F401
+
+ # Submit command
+ command_id = await CommandService.submit_command_job(
+ "open_notebook", # app name
+ "embed_single_item", # command name
+ {"item_id": item_id, "item_type": item_type},
)
- # Perform embedding
- await source_item.vectorize()
- message = "Source embedded successfully"
+ logger.info(f"Submitted async embedding command: {command_id}")
- elif item_type == "note":
- note_item = await Note.get(item_id)
- if not note_item:
- raise HTTPException(status_code=404, detail="Note not found")
+ return EmbedResponse(
+ success=True,
+ message="Embedding queued for background processing",
+ item_id=item_id,
+ item_type=item_type,
+ command_id=command_id,
+ )
- await note_item.vectorize()
+ except Exception as e:
+ logger.error(f"Failed to submit async embedding command: {e}")
+ raise HTTPException(
+ status_code=500, detail=f"Failed to queue embedding: {str(e)}"
+ )
- return EmbedResponse(
- success=True, message=message, item_id=item_id, item_type=item_type
- )
+ else:
+ # SYNC PATH: Execute synchronously (existing behavior)
+ logger.info(f"Using sync processing for {item_type} {item_id}")
+
+ # Get the item and embed it
+ if item_type == "source":
+ source_item = await Source.get(item_id)
+ if not source_item:
+ raise HTTPException(status_code=404, detail="Source not found")
+
+ # Perform embedding (vectorize is now idempotent - safe to call multiple times)
+ await source_item.vectorize()
+ message = "Source embedded successfully"
+
+ elif item_type == "note":
+ note_item = await Note.get(item_id)
+ if not note_item:
+ raise HTTPException(status_code=404, detail="Note not found")
+
+ await note_item.save() # Auto-embeds via ObjectModel.save()
+ message = "Note embedded successfully"
+
+ return EmbedResponse(
+ success=True, message=message, item_id=item_id, item_type=item_type, command_id=None
+ )
except HTTPException:
raise
diff --git a/api/routers/embedding_rebuild.py b/api/routers/embedding_rebuild.py
new file mode 100644
index 0000000..8a0f9a1
--- /dev/null
+++ b/api/routers/embedding_rebuild.py
@@ -0,0 +1,190 @@
+from fastapi import APIRouter, HTTPException
+from loguru import logger
+from surreal_commands import get_command_status
+
+from api.command_service import CommandService
+from api.models import (
+ RebuildProgress,
+ RebuildRequest,
+ RebuildResponse,
+ RebuildStats,
+ RebuildStatusResponse,
+)
+from open_notebook.database.repository import repo_query
+
+router = APIRouter()
+
+
+@router.post("/rebuild", response_model=RebuildResponse)
+async def start_rebuild(request: RebuildRequest):
+ """
+ Start a background job to rebuild embeddings.
+
+ - **mode**: "existing" (re-embed items with embeddings) or "all" (embed everything)
+ - **include_sources**: Include sources in rebuild (default: true)
+ - **include_notes**: Include notes in rebuild (default: true)
+ - **include_insights**: Include insights in rebuild (default: true)
+
+ Returns command ID to track progress and estimated item count.
+ """
+ try:
+ logger.info(f"Starting rebuild request: mode={request.mode}")
+
+ # Import commands to ensure they're registered
+ import commands.embedding_commands # noqa: F401
+
+ # Estimate total items (quick count query)
+ # This is a rough estimate before the command runs
+ total_estimate = 0
+
+ if request.include_sources:
+ if request.mode == "existing":
+ # Count sources with embeddings
+ result = await repo_query(
+ """
+ SELECT VALUE count(array::distinct(
+ SELECT VALUE source.id
+ FROM source_embedding
+ WHERE embedding != none AND array::len(embedding) > 0
+ )) as count FROM {}
+ """
+ )
+ else:
+ # Count all sources with content
+ result = await repo_query(
+ "SELECT VALUE count() as count FROM source WHERE full_text != none GROUP ALL"
+ )
+
+ if result and isinstance(result[0], dict):
+ total_estimate += result[0].get("count", 0)
+ elif result:
+ total_estimate += result[0] if isinstance(result[0], int) else 0
+
+ if request.include_notes:
+ if request.mode == "existing":
+ result = await repo_query(
+ "SELECT VALUE count() as count FROM note WHERE embedding != none AND array::len(embedding) > 0 GROUP ALL"
+ )
+ else:
+ result = await repo_query(
+ "SELECT VALUE count() as count FROM note WHERE content != none GROUP ALL"
+ )
+
+ if result and isinstance(result[0], dict):
+ total_estimate += result[0].get("count", 0)
+ elif result:
+ total_estimate += result[0] if isinstance(result[0], int) else 0
+
+ if request.include_insights:
+ if request.mode == "existing":
+ result = await repo_query(
+ "SELECT VALUE count() as count FROM source_insight WHERE embedding != none AND array::len(embedding) > 0 GROUP ALL"
+ )
+ else:
+ result = await repo_query(
+ "SELECT VALUE count() as count FROM source_insight GROUP ALL"
+ )
+
+ if result and isinstance(result[0], dict):
+ total_estimate += result[0].get("count", 0)
+ elif result:
+ total_estimate += result[0] if isinstance(result[0], int) else 0
+
+ logger.info(f"Estimated {total_estimate} items to process")
+
+ # Submit command
+ command_id = await CommandService.submit_command_job(
+ "open_notebook",
+ "rebuild_embeddings",
+ {
+ "mode": request.mode,
+ "include_sources": request.include_sources,
+ "include_notes": request.include_notes,
+ "include_insights": request.include_insights,
+ },
+ )
+
+ logger.info(f"Submitted rebuild command: {command_id}")
+
+ return RebuildResponse(
+ command_id=command_id,
+ total_items=total_estimate,
+ message=f"Rebuild operation started. Estimated {total_estimate} items to process.",
+ )
+
+ except Exception as e:
+ logger.error(f"Failed to start rebuild: {e}")
+ logger.exception(e)
+ raise HTTPException(
+ status_code=500, detail=f"Failed to start rebuild operation: {str(e)}"
+ )
+
+
+@router.get("/rebuild/{command_id}/status", response_model=RebuildStatusResponse)
+async def get_rebuild_status(command_id: str):
+ """
+ Get the status of a rebuild operation.
+
+ Returns:
+ - **status**: queued, running, completed, failed
+ - **progress**: processed count, total count, percentage
+ - **stats**: breakdown by type (sources, notes, insights, failed)
+ - **timestamps**: started_at, completed_at
+ """
+ try:
+ # Get command status from surreal_commands
+ status = await get_command_status(command_id)
+
+ if not status:
+ raise HTTPException(status_code=404, detail="Rebuild command not found")
+
+ # Build response based on status
+ response = RebuildStatusResponse(
+ command_id=command_id,
+ status=status.status,
+ )
+
+ # Extract metadata from command result
+ if status.result and isinstance(status.result, dict):
+ result = status.result
+
+ # Build progress info
+ if "total_items" in result and "processed_items" in result:
+ total = result["total_items"]
+ processed = result["processed_items"]
+ response.progress = RebuildProgress(
+ processed=processed,
+ total=total,
+ percentage=round((processed / total * 100) if total > 0 else 0, 2),
+ )
+
+ # Build stats
+ response.stats = RebuildStats(
+ sources=result.get("sources_processed", 0),
+ notes=result.get("notes_processed", 0),
+ insights=result.get("insights_processed", 0),
+ failed=result.get("failed_items", 0),
+ )
+
+ # Add timestamps
+ if hasattr(status, "created") and status.created:
+ response.started_at = str(status.created)
+ if hasattr(status, "updated") and status.updated:
+ response.completed_at = str(status.updated)
+
+ # Add error message if failed
+ if status.status == "failed" and status.result and isinstance(status.result, dict):
+ response.error_message = status.result.get(
+ "error_message", "Unknown error"
+ )
+
+ return response
+
+ except HTTPException:
+ raise
+ except Exception as e:
+ logger.error(f"Failed to get rebuild status: {e}")
+ logger.exception(e)
+ raise HTTPException(
+ status_code=500, detail=f"Failed to get rebuild status: {str(e)}"
+ )
diff --git a/api/routers/episode_profiles.py b/api/routers/episode_profiles.py
index 45a3af5..076723a 100644
--- a/api/routers/episode_profiles.py
+++ b/api/routers/episode_profiles.py
@@ -1,11 +1,11 @@
from typing import List
+
from fastapi import APIRouter, HTTPException
-from pydantic import BaseModel, Field
from loguru import logger
+from pydantic import BaseModel, Field
from open_notebook.domain.podcast import EpisodeProfile
-
router = APIRouter()
diff --git a/api/routers/insights.py b/api/routers/insights.py
index 890ff5d..b651e70 100644
--- a/api/routers/insights.py
+++ b/api/routers/insights.py
@@ -1,11 +1,10 @@
-from typing import Optional
from fastapi import APIRouter, HTTPException
from loguru import logger
from api.models import NoteResponse, SaveAsNoteRequest, SourceInsightResponse
-from open_notebook.domain.notebook import Note, SourceInsight
-from open_notebook.exceptions import DatabaseOperationError, InvalidInputError
+from open_notebook.domain.notebook import SourceInsight
+from open_notebook.exceptions import InvalidInputError
router = APIRouter()
@@ -22,8 +21,8 @@ async def get_insight(insight_id: str):
source = await insight.get_source()
return SourceInsightResponse(
- id=insight.id,
- source_id=source.id,
+ id=insight.id or "",
+ source_id=source.id or "",
insight_type=insight.insight_type,
content=insight.content,
created=str(insight.created),
@@ -66,7 +65,7 @@ async def save_insight_as_note(insight_id: str, request: SaveAsNoteRequest):
note = await insight.save_as_note(request.notebook_id)
return NoteResponse(
- id=note.id,
+ id=note.id or "",
title=note.title,
content=note.content,
note_type=note.note_type,
diff --git a/api/routers/models.py b/api/routers/models.py
index d82e046..9a5f725 100644
--- a/api/routers/models.py
+++ b/api/routers/models.py
@@ -1,11 +1,18 @@
+import os
from typing import List, Optional
+from esperanto import AIFactory
from fastapi import APIRouter, HTTPException, Query
from loguru import logger
-from api.models import DefaultModelsResponse, ModelCreate, ModelResponse
+from api.models import (
+ DefaultModelsResponse,
+ ModelCreate,
+ ModelResponse,
+ ProviderAvailabilityResponse,
+)
from open_notebook.domain.models import DefaultModels, Model
-from open_notebook.exceptions import DatabaseOperationError, InvalidInputError
+from open_notebook.exceptions import InvalidInputError
router = APIRouter()
@@ -57,7 +64,7 @@ async def create_model(model_data: ModelCreate):
await new_model.save()
return ModelResponse(
- id=new_model.id,
+ id=new_model.id or "",
name=new_model.name,
provider=new_model.provider,
type=new_model.type,
@@ -94,15 +101,15 @@ async def get_default_models():
"""Get default model assignments."""
try:
defaults = await DefaultModels.get_instance()
-
+
return DefaultModelsResponse(
- default_chat_model=defaults.default_chat_model,
- default_transformation_model=defaults.default_transformation_model,
- large_context_model=defaults.large_context_model,
- default_text_to_speech_model=defaults.default_text_to_speech_model,
- default_speech_to_text_model=defaults.default_speech_to_text_model,
- default_embedding_model=defaults.default_embedding_model,
- default_tools_model=defaults.default_tools_model,
+ default_chat_model=defaults.default_chat_model, # type: ignore[attr-defined]
+ default_transformation_model=defaults.default_transformation_model, # type: ignore[attr-defined]
+ large_context_model=defaults.large_context_model, # type: ignore[attr-defined]
+ default_text_to_speech_model=defaults.default_text_to_speech_model, # type: ignore[attr-defined]
+ default_speech_to_text_model=defaults.default_speech_to_text_model, # type: ignore[attr-defined]
+ default_embedding_model=defaults.default_embedding_model, # type: ignore[attr-defined]
+ default_tools_model=defaults.default_tools_model, # type: ignore[attr-defined]
)
except Exception as e:
logger.error(f"Error fetching default models: {str(e)}")
@@ -117,19 +124,19 @@ async def update_default_models(defaults_data: DefaultModelsResponse):
# Update only provided fields
if defaults_data.default_chat_model is not None:
- defaults.default_chat_model = defaults_data.default_chat_model
+ defaults.default_chat_model = defaults_data.default_chat_model # type: ignore[attr-defined]
if defaults_data.default_transformation_model is not None:
- defaults.default_transformation_model = defaults_data.default_transformation_model
+ defaults.default_transformation_model = defaults_data.default_transformation_model # type: ignore[attr-defined]
if defaults_data.large_context_model is not None:
- defaults.large_context_model = defaults_data.large_context_model
+ defaults.large_context_model = defaults_data.large_context_model # type: ignore[attr-defined]
if defaults_data.default_text_to_speech_model is not None:
- defaults.default_text_to_speech_model = defaults_data.default_text_to_speech_model
+ defaults.default_text_to_speech_model = defaults_data.default_text_to_speech_model # type: ignore[attr-defined]
if defaults_data.default_speech_to_text_model is not None:
- defaults.default_speech_to_text_model = defaults_data.default_speech_to_text_model
+ defaults.default_speech_to_text_model = defaults_data.default_speech_to_text_model # type: ignore[attr-defined]
if defaults_data.default_embedding_model is not None:
- defaults.default_embedding_model = defaults_data.default_embedding_model
+ defaults.default_embedding_model = defaults_data.default_embedding_model # type: ignore[attr-defined]
if defaults_data.default_tools_model is not None:
- defaults.default_tools_model = defaults_data.default_tools_model
+ defaults.default_tools_model = defaults_data.default_tools_model # type: ignore[attr-defined]
await defaults.update()
@@ -138,16 +145,74 @@ async def update_default_models(defaults_data: DefaultModelsResponse):
await model_manager.refresh_defaults()
return DefaultModelsResponse(
- default_chat_model=defaults.default_chat_model,
- default_transformation_model=defaults.default_transformation_model,
- large_context_model=defaults.large_context_model,
- default_text_to_speech_model=defaults.default_text_to_speech_model,
- default_speech_to_text_model=defaults.default_speech_to_text_model,
- default_embedding_model=defaults.default_embedding_model,
- default_tools_model=defaults.default_tools_model,
+ default_chat_model=defaults.default_chat_model, # type: ignore[attr-defined]
+ default_transformation_model=defaults.default_transformation_model, # type: ignore[attr-defined]
+ large_context_model=defaults.large_context_model, # type: ignore[attr-defined]
+ default_text_to_speech_model=defaults.default_text_to_speech_model, # type: ignore[attr-defined]
+ default_speech_to_text_model=defaults.default_speech_to_text_model, # type: ignore[attr-defined]
+ default_embedding_model=defaults.default_embedding_model, # type: ignore[attr-defined]
+ default_tools_model=defaults.default_tools_model, # type: ignore[attr-defined]
)
except HTTPException:
raise
except Exception as e:
logger.error(f"Error updating default models: {str(e)}")
- raise HTTPException(status_code=500, detail=f"Error updating default models: {str(e)}")
\ No newline at end of file
+ raise HTTPException(status_code=500, detail=f"Error updating default models: {str(e)}")
+
+
+@router.get("/models/providers", response_model=ProviderAvailabilityResponse)
+async def get_provider_availability():
+ """Get provider availability based on environment variables."""
+ try:
+ # Check which providers have API keys configured
+ provider_status = {
+ "ollama": os.environ.get("OLLAMA_API_BASE") is not None,
+ "openai": os.environ.get("OPENAI_API_KEY") is not None,
+ "groq": os.environ.get("GROQ_API_KEY") is not None,
+ "xai": os.environ.get("XAI_API_KEY") is not None,
+ "vertex": (
+ os.environ.get("VERTEX_PROJECT") is not None
+ and os.environ.get("VERTEX_LOCATION") is not None
+ and os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") is not None
+ ),
+ "google": (
+ os.environ.get("GOOGLE_API_KEY") is not None
+ or os.environ.get("GEMINI_API_KEY") is not None
+ ),
+ "openrouter": os.environ.get("OPENROUTER_API_KEY") is not None,
+ "anthropic": os.environ.get("ANTHROPIC_API_KEY") is not None,
+ "elevenlabs": os.environ.get("ELEVENLABS_API_KEY") is not None,
+ "voyage": os.environ.get("VOYAGE_API_KEY") is not None,
+ "azure": (
+ os.environ.get("AZURE_OPENAI_API_KEY") is not None
+ and os.environ.get("AZURE_OPENAI_ENDPOINT") is not None
+ and os.environ.get("AZURE_OPENAI_DEPLOYMENT_NAME") is not None
+ and os.environ.get("AZURE_OPENAI_API_VERSION") is not None
+ ),
+ "mistral": os.environ.get("MISTRAL_API_KEY") is not None,
+ "deepseek": os.environ.get("DEEPSEEK_API_KEY") is not None,
+ "openai-compatible": os.environ.get("OPENAI_COMPATIBLE_BASE_URL") is not None,
+ }
+
+ available_providers = [k for k, v in provider_status.items() if v]
+ unavailable_providers = [k for k, v in provider_status.items() if not v]
+
+ # Get supported model types from Esperanto
+ esperanto_available = AIFactory.get_available_providers()
+
+ # Build supported types mapping only for available providers
+ supported_types: dict[str, list[str]] = {}
+ for provider in available_providers:
+ supported_types[provider] = []
+ for model_type, providers in esperanto_available.items():
+ if provider in providers:
+ supported_types[provider].append(model_type)
+
+ return ProviderAvailabilityResponse(
+ available=available_providers,
+ unavailable=unavailable_providers,
+ supported_types=supported_types
+ )
+ except Exception as e:
+ logger.error(f"Error checking provider availability: {str(e)}")
+ raise HTTPException(status_code=500, detail=f"Error checking provider availability: {str(e)}")
\ No newline at end of file
diff --git a/api/routers/notebooks.py b/api/routers/notebooks.py
index 3681b59..6ac8ab9 100644
--- a/api/routers/notebooks.py
+++ b/api/routers/notebooks.py
@@ -3,9 +3,10 @@ from typing import List, Optional
from fastapi import APIRouter, HTTPException, Query
from loguru import logger
-from api.models import ErrorResponse, NotebookCreate, NotebookResponse, NotebookUpdate
+from api.models import NotebookCreate, NotebookResponse, NotebookUpdate
+from open_notebook.database.repository import ensure_record_id, repo_query
from open_notebook.domain.notebook import Notebook
-from open_notebook.exceptions import DatabaseOperationError, InvalidInputError
+from open_notebook.exceptions import InvalidInputError
router = APIRouter()
@@ -18,14 +19,14 @@ async def get_notebooks(
"""Get all notebooks with optional filtering and ordering."""
try:
notebooks = await Notebook.get_all(order_by=order_by)
-
+
# Filter by archived status if specified
if archived is not None:
notebooks = [nb for nb in notebooks if nb.archived == archived]
-
+
return [
NotebookResponse(
- id=nb.id,
+ id=nb.id or "",
name=nb.name,
description=nb.description,
archived=nb.archived or False,
@@ -36,7 +37,9 @@ async def get_notebooks(
]
except Exception as e:
logger.error(f"Error fetching notebooks: {str(e)}")
- raise HTTPException(status_code=500, detail=f"Error fetching notebooks: {str(e)}")
+ raise HTTPException(
+ status_code=500, detail=f"Error fetching notebooks: {str(e)}"
+ )
@router.post("/notebooks", response_model=NotebookResponse)
@@ -48,9 +51,9 @@ async def create_notebook(notebook: NotebookCreate):
description=notebook.description,
)
await new_notebook.save()
-
+
return NotebookResponse(
- id=new_notebook.id,
+ id=new_notebook.id or "",
name=new_notebook.name,
description=new_notebook.description,
archived=new_notebook.archived or False,
@@ -61,7 +64,9 @@ async def create_notebook(notebook: NotebookCreate):
raise HTTPException(status_code=400, detail=str(e))
except Exception as e:
logger.error(f"Error creating notebook: {str(e)}")
- raise HTTPException(status_code=500, detail=f"Error creating notebook: {str(e)}")
+ raise HTTPException(
+ status_code=500, detail=f"Error creating notebook: {str(e)}"
+ )
@router.get("/notebooks/{notebook_id}", response_model=NotebookResponse)
@@ -71,9 +76,9 @@ async def get_notebook(notebook_id: str):
notebook = await Notebook.get(notebook_id)
if not notebook:
raise HTTPException(status_code=404, detail="Notebook not found")
-
+
return NotebookResponse(
- id=notebook.id,
+ id=notebook.id or "",
name=notebook.name,
description=notebook.description,
archived=notebook.archived or False,
@@ -84,7 +89,9 @@ async def get_notebook(notebook_id: str):
raise
except Exception as e:
logger.error(f"Error fetching notebook {notebook_id}: {str(e)}")
- raise HTTPException(status_code=500, detail=f"Error fetching notebook: {str(e)}")
+ raise HTTPException(
+ status_code=500, detail=f"Error fetching notebook: {str(e)}"
+ )
@router.put("/notebooks/{notebook_id}", response_model=NotebookResponse)
@@ -94,7 +101,7 @@ async def update_notebook(notebook_id: str, notebook_update: NotebookUpdate):
notebook = await Notebook.get(notebook_id)
if not notebook:
raise HTTPException(status_code=404, detail="Notebook not found")
-
+
# Update only provided fields
if notebook_update.name is not None:
notebook.name = notebook_update.name
@@ -102,11 +109,11 @@ async def update_notebook(notebook_id: str, notebook_update: NotebookUpdate):
notebook.description = notebook_update.description
if notebook_update.archived is not None:
notebook.archived = notebook_update.archived
-
+
await notebook.save()
-
+
return NotebookResponse(
- id=notebook.id,
+ id=notebook.id or "",
name=notebook.name,
description=notebook.description,
archived=notebook.archived or False,
@@ -119,7 +126,39 @@ async def update_notebook(notebook_id: str, notebook_update: NotebookUpdate):
raise HTTPException(status_code=400, detail=str(e))
except Exception as e:
logger.error(f"Error updating notebook {notebook_id}: {str(e)}")
- raise HTTPException(status_code=500, detail=f"Error updating notebook: {str(e)}")
+ raise HTTPException(
+ status_code=500, detail=f"Error updating notebook: {str(e)}"
+ )
+
+
+@router.delete("/notebooks/{notebook_id}/sources/{source_id}")
+async def remove_source_from_notebook(notebook_id: str, source_id: str):
+ """Remove a source from a notebook (delete the reference)."""
+ try:
+ # Check if notebook exists
+ notebook = await Notebook.get(notebook_id)
+ if not notebook:
+ raise HTTPException(status_code=404, detail="Notebook not found")
+
+ # Delete the reference record linking source to notebook
+ await repo_query(
+ "DELETE FROM reference WHERE out = $notebook_id AND in = $source_id",
+ {
+ "notebook_id": ensure_record_id(notebook_id),
+ "source_id": ensure_record_id(source_id),
+ },
+ )
+
+ return {"message": "Source removed from notebook successfully"}
+ except HTTPException:
+ raise
+ except Exception as e:
+ logger.error(
+ f"Error removing source {source_id} from notebook {notebook_id}: {str(e)}"
+ )
+ raise HTTPException(
+ status_code=500, detail=f"Error removing source from notebook: {str(e)}"
+ )
@router.delete("/notebooks/{notebook_id}")
@@ -129,12 +168,14 @@ async def delete_notebook(notebook_id: str):
notebook = await Notebook.get(notebook_id)
if not notebook:
raise HTTPException(status_code=404, detail="Notebook not found")
-
+
await notebook.delete()
-
+
return {"message": "Notebook deleted successfully"}
except HTTPException:
raise
except Exception as e:
logger.error(f"Error deleting notebook {notebook_id}: {str(e)}")
- raise HTTPException(status_code=500, detail=f"Error deleting notebook: {str(e)}")
\ No newline at end of file
+ raise HTTPException(
+ status_code=500, detail=f"Error deleting notebook: {str(e)}"
+ )
diff --git a/api/routers/notes.py b/api/routers/notes.py
index 33f9826..1eed228 100644
--- a/api/routers/notes.py
+++ b/api/routers/notes.py
@@ -1,4 +1,4 @@
-from typing import List, Optional
+from typing import List, Literal, Optional
from fastapi import APIRouter, HTTPException, Query
from loguru import logger
@@ -29,7 +29,7 @@ async def get_notes(
return [
NoteResponse(
- id=note.id,
+ id=note.id or "",
title=note.title,
content=note.content,
note_type=note.note_type,
@@ -54,16 +54,25 @@ async def create_note(note_data: NoteCreate):
if not title and note_data.note_type == "ai" and note_data.content:
from open_notebook.graphs.prompt import graph as prompt_graph
prompt = "Based on the Note below, please provide a Title for this content, with max 15 words"
- result = await prompt_graph.ainvoke({
- "input_text": note_data.content,
- "prompt": prompt
- })
+ result = await prompt_graph.ainvoke(
+ { # type: ignore[arg-type]
+ "input_text": note_data.content,
+ "prompt": prompt
+ }
+ )
title = result.get("output", "Untitled Note")
+ # Validate note_type
+ note_type: Optional[Literal["human", "ai"]] = None
+ if note_data.note_type in ("human", "ai"):
+ note_type = note_data.note_type # type: ignore[assignment]
+ elif note_data.note_type is not None:
+ raise HTTPException(status_code=400, detail="note_type must be 'human' or 'ai'")
+
new_note = Note(
title=title,
content=note_data.content,
- note_type=note_data.note_type,
+ note_type=note_type,
)
await new_note.save()
@@ -76,7 +85,7 @@ async def create_note(note_data: NoteCreate):
await new_note.add_to_notebook(note_data.notebook_id)
return NoteResponse(
- id=new_note.id,
+ id=new_note.id or "",
title=new_note.title,
content=new_note.content,
note_type=new_note.note_type,
@@ -101,7 +110,7 @@ async def get_note(note_id: str):
raise HTTPException(status_code=404, detail="Note not found")
return NoteResponse(
- id=note.id,
+ id=note.id or "",
title=note.title,
content=note.content,
note_type=note.note_type,
@@ -129,12 +138,15 @@ async def update_note(note_id: str, note_update: NoteUpdate):
if note_update.content is not None:
note.content = note_update.content
if note_update.note_type is not None:
- note.note_type = note_update.note_type
-
+ if note_update.note_type in ("human", "ai"):
+ note.note_type = note_update.note_type # type: ignore[assignment]
+ else:
+ raise HTTPException(status_code=400, detail="note_type must be 'human' or 'ai'")
+
await note.save()
-
+
return NoteResponse(
- id=note.id,
+ id=note.id or "",
title=note.title,
content=note.content,
note_type=note.note_type,
diff --git a/api/routers/podcasts.py b/api/routers/podcasts.py
index 2f77611..9f833bc 100644
--- a/api/routers/podcasts.py
+++ b/api/routers/podcasts.py
@@ -1,7 +1,9 @@
-from typing import List, Optional
from pathlib import Path
+from typing import List, Optional
+from urllib.parse import unquote, urlparse
from fastapi import APIRouter, HTTPException
+from fastapi.responses import FileResponse
from loguru import logger
from pydantic import BaseModel
@@ -10,7 +12,6 @@ from api.podcast_service import (
PodcastGenerationResponse,
PodcastService,
)
-from open_notebook.domain.podcast import PodcastEpisode
router = APIRouter()
@@ -22,12 +23,20 @@ class PodcastEpisodeResponse(BaseModel):
speaker_profile: dict
briefing: str
audio_file: Optional[str] = None
+ audio_url: Optional[str] = None
transcript: Optional[dict] = None
outline: Optional[dict] = None
created: Optional[str] = None
job_status: Optional[str] = None
+def _resolve_audio_path(audio_file: str) -> Path:
+ if audio_file.startswith("file://"):
+ parsed = urlparse(audio_file)
+ return Path(unquote(parsed.path))
+ return Path(audio_file)
+
+
@router.post("/podcasts/generate", response_model=PodcastGenerationResponse)
async def generate_podcast(request: PodcastGenerationRequest):
"""
@@ -90,12 +99,18 @@ async def list_podcast_episodes():
if episode.command:
try:
job_status = await episode.get_job_status()
- except:
+ except Exception:
job_status = "unknown"
else:
# No command but has audio file = completed import
job_status = "completed"
+ audio_url = None
+ if episode.audio_file:
+ audio_path = _resolve_audio_path(episode.audio_file)
+ if audio_path.exists():
+ audio_url = f"/api/podcasts/episodes/{episode.id}/audio"
+
response_episodes.append(
PodcastEpisodeResponse(
id=str(episode.id),
@@ -104,6 +119,7 @@ async def list_podcast_episodes():
speaker_profile=episode.speaker_profile,
briefing=episode.briefing,
audio_file=episode.audio_file,
+ audio_url=audio_url,
transcript=episode.transcript,
outline=episode.outline,
created=str(episode.created) if episode.created else None,
@@ -131,12 +147,18 @@ async def get_podcast_episode(episode_id: str):
if episode.command:
try:
job_status = await episode.get_job_status()
- except:
+ except Exception:
job_status = "unknown"
else:
# No command but has audio file = completed import
job_status = "completed" if episode.audio_file else "unknown"
+ audio_url = None
+ if episode.audio_file:
+ audio_path = _resolve_audio_path(episode.audio_file)
+ if audio_path.exists():
+ audio_url = f"/api/podcasts/episodes/{episode.id}/audio"
+
return PodcastEpisodeResponse(
id=str(episode.id),
name=episode.name,
@@ -144,6 +166,7 @@ async def get_podcast_episode(episode_id: str):
speaker_profile=episode.speaker_profile,
briefing=episode.briefing,
audio_file=episode.audio_file,
+ audio_url=audio_url,
transcript=episode.transcript,
outline=episode.outline,
created=str(episode.created) if episode.created else None,
@@ -155,6 +178,31 @@ async def get_podcast_episode(episode_id: str):
raise HTTPException(status_code=404, detail=f"Episode not found: {str(e)}")
+@router.get("/podcasts/episodes/{episode_id}/audio")
+async def stream_podcast_episode_audio(episode_id: str):
+ """Stream the audio file associated with a podcast episode"""
+ try:
+ episode = await PodcastService.get_episode(episode_id)
+ except HTTPException:
+ raise
+ except Exception as e:
+ logger.error(f"Error fetching podcast episode for audio: {str(e)}")
+ raise HTTPException(status_code=404, detail=f"Episode not found: {str(e)}")
+
+ if not episode.audio_file:
+ raise HTTPException(status_code=404, detail="Episode has no audio file")
+
+ audio_path = _resolve_audio_path(episode.audio_file)
+ if not audio_path.exists():
+ raise HTTPException(status_code=404, detail="Audio file not found on disk")
+
+ return FileResponse(
+ audio_path,
+ media_type="audio/mpeg",
+ filename=audio_path.name,
+ )
+
+
@router.delete("/podcasts/episodes/{episode_id}")
async def delete_podcast_episode(episode_id: str):
"""Delete a podcast episode and its associated audio file"""
@@ -164,7 +212,7 @@ async def delete_podcast_episode(episode_id: str):
# Delete the physical audio file if it exists
if episode.audio_file:
- audio_path = Path(episode.audio_file)
+ audio_path = _resolve_audio_path(episode.audio_file)
if audio_path.exists():
try:
audio_path.unlink()
diff --git a/api/routers/search.py b/api/routers/search.py
index fd14362..e6059ab 100644
--- a/api/routers/search.py
+++ b/api/routers/search.py
@@ -1,5 +1,5 @@
-import asyncio
-from typing import AsyncGenerator, Dict
+import json
+from typing import AsyncGenerator
from fastapi import APIRouter, HTTPException
from fastapi.responses import StreamingResponse
@@ -66,7 +66,7 @@ async def stream_ask_response(
final_answer = None
async for chunk in ask_graph.astream(
- input=dict(question=question),
+ input=dict(question=question), # type: ignore[arg-type]
config=dict(
configurable=dict(
strategy_model=strategy_model.id,
@@ -85,25 +85,26 @@ async def stream_ask_response(
for search in chunk["agent"]["strategy"].searches
],
}
- yield f"data: {strategy_data}\n\n"
+ yield f"data: {json.dumps(strategy_data)}\n\n"
elif "provide_answer" in chunk:
for answer in chunk["provide_answer"]["answers"]:
answer_data = {"type": "answer", "content": answer}
- yield f"data: {answer_data}\n\n"
+ yield f"data: {json.dumps(answer_data)}\n\n"
elif "write_final_answer" in chunk:
final_answer = chunk["write_final_answer"]["final_answer"]
final_data = {"type": "final_answer", "content": final_answer}
- yield f"data: {final_data}\n\n"
+ yield f"data: {json.dumps(final_data)}\n\n"
# Send completion signal
- yield f"data: {{'type': 'complete', 'final_answer': '{final_answer}'}}\n\n"
+ completion_data = {"type": "complete", "final_answer": final_answer}
+ yield f"data: {json.dumps(completion_data)}\n\n"
except Exception as e:
logger.error(f"Error in ask streaming: {str(e)}")
error_data = {"type": "error", "message": str(e)}
- yield f"data: {error_data}\n\n"
+ yield f"data: {json.dumps(error_data)}\n\n"
@router.post("/search/ask")
@@ -140,7 +141,7 @@ async def ask_knowledge_base(ask_request: AskRequest):
# For streaming response
return StreamingResponse(
- await stream_ask_response(
+ stream_ask_response(
ask_request.question, strategy_model, answer_model, final_answer_model
),
media_type="text/plain",
@@ -188,7 +189,7 @@ async def ask_knowledge_base_simple(ask_request: AskRequest):
# Run the ask graph and get final result
final_answer = None
async for chunk in ask_graph.astream(
- input=dict(question=ask_request.question),
+ input=dict(question=ask_request.question), # type: ignore[arg-type]
config=dict(
configurable=dict(
strategy_model=strategy_model.id,
diff --git a/api/routers/settings.py b/api/routers/settings.py
index d44562b..c5eabeb 100644
--- a/api/routers/settings.py
+++ b/api/routers/settings.py
@@ -3,7 +3,7 @@ from loguru import logger
from api.models import SettingsResponse, SettingsUpdate
from open_notebook.domain.content_settings import ContentSettings
-from open_notebook.exceptions import DatabaseOperationError, InvalidInputError
+from open_notebook.exceptions import InvalidInputError
router = APIRouter()
@@ -12,8 +12,8 @@ router = APIRouter()
async def get_settings():
"""Get all application settings."""
try:
- settings = await ContentSettings.get_instance()
-
+ settings: ContentSettings = await ContentSettings.get_instance() # type: ignore[assignment]
+
return SettingsResponse(
default_content_processing_engine_doc=settings.default_content_processing_engine_doc,
default_content_processing_engine_url=settings.default_content_processing_engine_url,
@@ -30,22 +30,39 @@ async def get_settings():
async def update_settings(settings_update: SettingsUpdate):
"""Update application settings."""
try:
- settings = await ContentSettings.get_instance()
-
+ settings: ContentSettings = await ContentSettings.get_instance() # type: ignore[assignment]
+
# Update only provided fields
if settings_update.default_content_processing_engine_doc is not None:
- settings.default_content_processing_engine_doc = settings_update.default_content_processing_engine_doc
+ # Cast to proper literal type
+ from typing import Literal, cast
+ settings.default_content_processing_engine_doc = cast(
+ Literal["auto", "docling", "simple"],
+ settings_update.default_content_processing_engine_doc
+ )
if settings_update.default_content_processing_engine_url is not None:
- settings.default_content_processing_engine_url = settings_update.default_content_processing_engine_url
+ from typing import Literal, cast
+ settings.default_content_processing_engine_url = cast(
+ Literal["auto", "firecrawl", "jina", "simple"],
+ settings_update.default_content_processing_engine_url
+ )
if settings_update.default_embedding_option is not None:
- settings.default_embedding_option = settings_update.default_embedding_option
+ from typing import Literal, cast
+ settings.default_embedding_option = cast(
+ Literal["ask", "always", "never"],
+ settings_update.default_embedding_option
+ )
if settings_update.auto_delete_files is not None:
- settings.auto_delete_files = settings_update.auto_delete_files
+ from typing import Literal, cast
+ settings.auto_delete_files = cast(
+ Literal["yes", "no"],
+ settings_update.auto_delete_files
+ )
if settings_update.youtube_preferred_languages is not None:
settings.youtube_preferred_languages = settings_update.youtube_preferred_languages
-
+
await settings.update()
-
+
return SettingsResponse(
default_content_processing_engine_doc=settings.default_content_processing_engine_doc,
default_content_processing_engine_url=settings.default_content_processing_engine_url,
diff --git a/api/routers/source_chat.py b/api/routers/source_chat.py
new file mode 100644
index 0000000..38d31d2
--- /dev/null
+++ b/api/routers/source_chat.py
@@ -0,0 +1,446 @@
+import asyncio
+import json
+from typing import AsyncGenerator, List, Optional
+
+from fastapi import APIRouter, HTTPException, Path
+from fastapi.responses import StreamingResponse
+from langchain_core.messages import HumanMessage
+from langchain_core.runnables import RunnableConfig
+from loguru import logger
+from pydantic import BaseModel, Field
+
+from open_notebook.database.repository import ensure_record_id, repo_query
+from open_notebook.domain.notebook import ChatSession, Source
+from open_notebook.exceptions import (
+ NotFoundError,
+)
+from open_notebook.graphs.source_chat import source_chat_graph as source_chat_graph
+
+router = APIRouter()
+
+# Request/Response models
+class CreateSourceChatSessionRequest(BaseModel):
+ source_id: str = Field(..., description="Source ID to create chat session for")
+ title: Optional[str] = Field(None, description="Optional session title")
+ model_override: Optional[str] = Field(None, description="Optional model override for this session")
+
+class UpdateSourceChatSessionRequest(BaseModel):
+ title: Optional[str] = Field(None, description="New session title")
+ model_override: Optional[str] = Field(None, description="Model override for this session")
+
+class ChatMessage(BaseModel):
+ id: str = Field(..., description="Message ID")
+ type: str = Field(..., description="Message type (human|ai)")
+ content: str = Field(..., description="Message content")
+ timestamp: Optional[str] = Field(None, description="Message timestamp")
+
+class ContextIndicator(BaseModel):
+ sources: List[str] = Field(default_factory=list, description="Source IDs used in context")
+ insights: List[str] = Field(default_factory=list, description="Insight IDs used in context")
+ notes: List[str] = Field(default_factory=list, description="Note IDs used in context")
+
+class SourceChatSessionResponse(BaseModel):
+ id: str = Field(..., description="Session ID")
+ title: str = Field(..., description="Session title")
+ source_id: str = Field(..., description="Source ID")
+ model_override: Optional[str] = Field(None, description="Model override for this session")
+ created: str = Field(..., description="Creation timestamp")
+ updated: str = Field(..., description="Last update timestamp")
+ message_count: Optional[int] = Field(None, description="Number of messages in session")
+
+class SourceChatSessionWithMessagesResponse(SourceChatSessionResponse):
+ messages: List[ChatMessage] = Field(default_factory=list, description="Session messages")
+ context_indicators: Optional[ContextIndicator] = Field(None, description="Context indicators from last response")
+
+class SendMessageRequest(BaseModel):
+ message: str = Field(..., description="User message content")
+ model_override: Optional[str] = Field(None, description="Optional model override for this message")
+
+class SuccessResponse(BaseModel):
+ success: bool = Field(True, description="Operation success status")
+ message: str = Field(..., description="Success message")
+
+
+@router.post("/sources/{source_id}/chat/sessions", response_model=SourceChatSessionResponse)
+async def create_source_chat_session(
+ request: CreateSourceChatSessionRequest,
+ source_id: str = Path(..., description="Source ID")
+):
+ """Create a new chat session for a source."""
+ try:
+ # Verify source exists
+ full_source_id = source_id if source_id.startswith("source:") else f"source:{source_id}"
+ source = await Source.get(full_source_id)
+ if not source:
+ raise HTTPException(status_code=404, detail="Source not found")
+
+ # Create new session with model_override support
+ session = ChatSession(
+ title=request.title or f"Source Chat {asyncio.get_event_loop().time():.0f}",
+ model_override=request.model_override
+ )
+ await session.save()
+
+ # Relate session to source using "refers_to" relation
+ await session.relate("refers_to", full_source_id)
+
+ return SourceChatSessionResponse(
+ id=session.id or "",
+ title=session.title or "Untitled Session",
+ source_id=source_id,
+ model_override=session.model_override,
+ created=str(session.created),
+ updated=str(session.updated),
+ message_count=0
+ )
+ except NotFoundError:
+ raise HTTPException(status_code=404, detail="Source not found")
+ except Exception as e:
+ logger.error(f"Error creating source chat session: {str(e)}")
+ raise HTTPException(status_code=500, detail=f"Error creating source chat session: {str(e)}")
+
+
+@router.get("/sources/{source_id}/chat/sessions", response_model=List[SourceChatSessionResponse])
+async def get_source_chat_sessions(
+ source_id: str = Path(..., description="Source ID")
+):
+ """Get all chat sessions for a source."""
+ try:
+ # Verify source exists
+ full_source_id = source_id if source_id.startswith("source:") else f"source:{source_id}"
+ source = await Source.get(full_source_id)
+ if not source:
+ raise HTTPException(status_code=404, detail="Source not found")
+
+ # Get sessions that refer to this source - first get relations, then sessions
+ relations = await repo_query(
+ "SELECT in FROM refers_to WHERE out = $source_id",
+ {"source_id": ensure_record_id(full_source_id)}
+ )
+
+ sessions = []
+ for relation in relations:
+ session_id = relation.get("in")
+ if session_id:
+ session_result = await repo_query(f"SELECT * FROM {session_id}")
+ if session_result and len(session_result) > 0:
+ session_data = session_result[0]
+ sessions.append(SourceChatSessionResponse(
+ id=session_data.get("id") or "",
+ title=session_data.get("title") or "Untitled Session",
+ source_id=source_id,
+ model_override=session_data.get("model_override"),
+ created=str(session_data.get("created")),
+ updated=str(session_data.get("updated")),
+ message_count=0 # TODO: Add message count if needed
+ ))
+
+ # Sort sessions by created date (newest first)
+ sessions.sort(key=lambda x: x.created, reverse=True)
+ return sessions
+ except NotFoundError:
+ raise HTTPException(status_code=404, detail="Source not found")
+ except Exception as e:
+ logger.error(f"Error fetching source chat sessions: {str(e)}")
+ raise HTTPException(status_code=500, detail=f"Error fetching source chat sessions: {str(e)}")
+
+
+@router.get("/sources/{source_id}/chat/sessions/{session_id}", response_model=SourceChatSessionWithMessagesResponse)
+async def get_source_chat_session(
+ source_id: str = Path(..., description="Source ID"),
+ session_id: str = Path(..., description="Session ID")
+):
+ """Get a specific source chat session with its messages."""
+ try:
+ # Verify source exists
+ full_source_id = source_id if source_id.startswith("source:") else f"source:{source_id}"
+ source = await Source.get(full_source_id)
+ if not source:
+ raise HTTPException(status_code=404, detail="Source not found")
+
+ # Get session
+ full_session_id = session_id if session_id.startswith("chat_session:") else f"chat_session:{session_id}"
+ session = await ChatSession.get(full_session_id)
+ if not session:
+ raise HTTPException(status_code=404, detail="Session not found")
+
+ # Verify session is related to this source
+ relation_query = await repo_query(
+ "SELECT * FROM refers_to WHERE in = $session_id AND out = $source_id",
+ {"session_id": ensure_record_id(full_session_id), "source_id": ensure_record_id(full_source_id)}
+ )
+
+ if not relation_query:
+ raise HTTPException(status_code=404, detail="Session not found for this source")
+
+ # Get session state from LangGraph to retrieve messages
+ thread_state = source_chat_graph.get_state(
+ config=RunnableConfig(configurable={"thread_id": session_id})
+ )
+
+ # Extract messages from state
+ messages: list[ChatMessage] = []
+ context_indicators = None
+
+ if thread_state and thread_state.values:
+ # Extract messages
+ if "messages" in thread_state.values:
+ for msg in thread_state.values["messages"]:
+ messages.append(ChatMessage(
+ id=getattr(msg, 'id', f"msg_{len(messages)}"),
+ type=msg.type if hasattr(msg, 'type') else 'unknown',
+ content=msg.content if hasattr(msg, 'content') else str(msg),
+ timestamp=None # LangChain messages don't have timestamps by default
+ ))
+
+ # Extract context indicators from the last state
+ if "context_indicators" in thread_state.values:
+ context_data = thread_state.values["context_indicators"]
+ context_indicators = ContextIndicator(
+ sources=context_data.get("sources", []),
+ insights=context_data.get("insights", []),
+ notes=context_data.get("notes", [])
+ )
+
+ return SourceChatSessionWithMessagesResponse(
+ id=session.id or "",
+ title=session.title or "Untitled Session",
+ source_id=source_id,
+ model_override=getattr(session, 'model_override', None),
+ created=str(session.created),
+ updated=str(session.updated),
+ message_count=len(messages),
+ messages=messages,
+ context_indicators=context_indicators
+ )
+ except NotFoundError:
+ raise HTTPException(status_code=404, detail="Source or session not found")
+ except Exception as e:
+ logger.error(f"Error fetching source chat session: {str(e)}")
+ raise HTTPException(status_code=500, detail=f"Error fetching source chat session: {str(e)}")
+
+
+@router.put("/sources/{source_id}/chat/sessions/{session_id}", response_model=SourceChatSessionResponse)
+async def update_source_chat_session(
+ request: UpdateSourceChatSessionRequest,
+ source_id: str = Path(..., description="Source ID"),
+ session_id: str = Path(..., description="Session ID")
+):
+ """Update source chat session title and/or model override."""
+ try:
+ # Verify source exists
+ full_source_id = source_id if source_id.startswith("source:") else f"source:{source_id}"
+ source = await Source.get(full_source_id)
+ if not source:
+ raise HTTPException(status_code=404, detail="Source not found")
+
+ # Get session
+ full_session_id = session_id if session_id.startswith("chat_session:") else f"chat_session:{session_id}"
+ session = await ChatSession.get(full_session_id)
+ if not session:
+ raise HTTPException(status_code=404, detail="Session not found")
+
+ # Verify session is related to this source
+ relation_query = await repo_query(
+ "SELECT * FROM refers_to WHERE in = $session_id AND out = $source_id",
+ {"session_id": ensure_record_id(full_session_id), "source_id": ensure_record_id(full_source_id)}
+ )
+
+ if not relation_query:
+ raise HTTPException(status_code=404, detail="Session not found for this source")
+
+ # Update session fields
+ if request.title is not None:
+ session.title = request.title
+ if request.model_override is not None:
+ session.model_override = request.model_override
+
+ await session.save()
+
+ return SourceChatSessionResponse(
+ id=session.id or "",
+ title=session.title or "Untitled Session",
+ source_id=source_id,
+ model_override=getattr(session, 'model_override', None),
+ created=str(session.created),
+ updated=str(session.updated),
+ message_count=0
+ )
+ except NotFoundError:
+ raise HTTPException(status_code=404, detail="Source or session not found")
+ except Exception as e:
+ logger.error(f"Error updating source chat session: {str(e)}")
+ raise HTTPException(status_code=500, detail=f"Error updating source chat session: {str(e)}")
+
+
+@router.delete("/sources/{source_id}/chat/sessions/{session_id}", response_model=SuccessResponse)
+async def delete_source_chat_session(
+ source_id: str = Path(..., description="Source ID"),
+ session_id: str = Path(..., description="Session ID")
+):
+ """Delete a source chat session."""
+ try:
+ # Verify source exists
+ full_source_id = source_id if source_id.startswith("source:") else f"source:{source_id}"
+ source = await Source.get(full_source_id)
+ if not source:
+ raise HTTPException(status_code=404, detail="Source not found")
+
+ # Get session
+ full_session_id = session_id if session_id.startswith("chat_session:") else f"chat_session:{session_id}"
+ session = await ChatSession.get(full_session_id)
+ if not session:
+ raise HTTPException(status_code=404, detail="Session not found")
+
+ # Verify session is related to this source
+ relation_query = await repo_query(
+ "SELECT * FROM refers_to WHERE in = $session_id AND out = $source_id",
+ {"session_id": ensure_record_id(full_session_id), "source_id": ensure_record_id(full_source_id)}
+ )
+
+ if not relation_query:
+ raise HTTPException(status_code=404, detail="Session not found for this source")
+
+ await session.delete()
+
+ return SuccessResponse(
+ success=True,
+ message="Source chat session deleted successfully"
+ )
+ except NotFoundError:
+ raise HTTPException(status_code=404, detail="Source or session not found")
+ except Exception as e:
+ logger.error(f"Error deleting source chat session: {str(e)}")
+ raise HTTPException(status_code=500, detail=f"Error deleting source chat session: {str(e)}")
+
+
+async def stream_source_chat_response(
+ session_id: str,
+ source_id: str,
+ message: str,
+ model_override: Optional[str] = None
+) -> AsyncGenerator[str, None]:
+ """Stream the source chat response as Server-Sent Events."""
+ try:
+ # Get current state
+ current_state = source_chat_graph.get_state(
+ config=RunnableConfig(configurable={"thread_id": session_id})
+ )
+
+ # Prepare state for execution
+ state_values = current_state.values if current_state else {}
+ state_values["messages"] = state_values.get("messages", [])
+ state_values["source_id"] = source_id
+ state_values["model_override"] = model_override
+
+ # Add user message to state
+ user_message = HumanMessage(content=message)
+ state_values["messages"].append(user_message)
+
+ # Send user message event
+ user_event = {
+ "type": "user_message",
+ "content": message,
+ "timestamp": None
+ }
+ yield f"data: {json.dumps(user_event)}\n\n"
+
+ # Execute source chat graph synchronously (like notebook chat does)
+ result = source_chat_graph.invoke(
+ input=state_values, # type: ignore[arg-type]
+ config=RunnableConfig(
+ configurable={
+ "thread_id": session_id,
+ "model_id": model_override
+ }
+ )
+ )
+
+ # Stream the complete AI response
+ if "messages" in result:
+ for msg in result["messages"]:
+ if hasattr(msg, 'type') and msg.type == 'ai':
+ ai_event = {
+ "type": "ai_message",
+ "content": msg.content if hasattr(msg, 'content') else str(msg),
+ "timestamp": None
+ }
+ yield f"data: {json.dumps(ai_event)}\n\n"
+
+ # Stream context indicators
+ if "context_indicators" in result:
+ context_event = {
+ "type": "context_indicators",
+ "data": result["context_indicators"]
+ }
+ yield f"data: {json.dumps(context_event)}\n\n"
+
+ # Send completion signal
+ completion_event = {"type": "complete"}
+ yield f"data: {json.dumps(completion_event)}\n\n"
+
+ except Exception as e:
+ logger.error(f"Error in source chat streaming: {str(e)}")
+ error_event = {"type": "error", "message": str(e)}
+ yield f"data: {json.dumps(error_event)}\n\n"
+
+
+@router.post("/sources/{source_id}/chat/sessions/{session_id}/messages")
+async def send_message_to_source_chat(
+ request: SendMessageRequest,
+ source_id: str = Path(..., description="Source ID"),
+ session_id: str = Path(..., description="Session ID")
+):
+ """Send a message to source chat session with SSE streaming response."""
+ try:
+ # Verify source exists
+ full_source_id = source_id if source_id.startswith("source:") else f"source:{source_id}"
+ source = await Source.get(full_source_id)
+ if not source:
+ raise HTTPException(status_code=404, detail="Source not found")
+
+ # Verify session exists and is related to source
+ full_session_id = session_id if session_id.startswith("chat_session:") else f"chat_session:{session_id}"
+ session = await ChatSession.get(full_session_id)
+ if not session:
+ raise HTTPException(status_code=404, detail="Session not found")
+
+ # Verify session is related to this source
+ relation_query = await repo_query(
+ "SELECT * FROM refers_to WHERE in = $session_id AND out = $source_id",
+ {"session_id": ensure_record_id(full_session_id), "source_id": ensure_record_id(full_source_id)}
+ )
+
+ if not relation_query:
+ raise HTTPException(status_code=404, detail="Session not found for this source")
+
+ if not request.message:
+ raise HTTPException(status_code=400, detail="Message content is required")
+
+ # Determine model override (request override takes precedence over session override)
+ model_override = request.model_override or getattr(session, 'model_override', None)
+
+ # Update session timestamp
+ await session.save()
+
+ # Return streaming response
+ return StreamingResponse(
+ stream_source_chat_response(
+ session_id=session_id,
+ source_id=full_source_id,
+ message=request.message,
+ model_override=model_override
+ ),
+ media_type="text/plain",
+ headers={
+ "Cache-Control": "no-cache",
+ "Connection": "keep-alive",
+ "Content-Type": "text/plain; charset=utf-8"
+ }
+ )
+
+ except HTTPException:
+ raise
+ except Exception as e:
+ logger.error(f"Error sending message to source chat: {str(e)}")
+ raise HTTPException(status_code=500, detail=f"Error sending message: {str(e)}")
\ No newline at end of file
diff --git a/api/routers/sources.py b/api/routers/sources.py
index eda8df4..be9dcbb 100644
--- a/api/routers/sources.py
+++ b/api/routers/sources.py
@@ -1,8 +1,21 @@
-from typing import List, Optional
+import os
+from pathlib import Path
+from typing import Any, List, Optional
-from fastapi import APIRouter, HTTPException, Query
+from fastapi import (
+ APIRouter,
+ Depends,
+ File,
+ Form,
+ HTTPException,
+ Query,
+ UploadFile,
+)
+from fastapi.responses import FileResponse, Response
from loguru import logger
+from surreal_commands import execute_command_sync
+from api.command_service import CommandService
from api.models import (
AssetModel,
CreateSourceInsightRequest,
@@ -10,51 +23,288 @@ from api.models import (
SourceInsightResponse,
SourceListResponse,
SourceResponse,
+ SourceStatusResponse,
SourceUpdate,
)
+from commands.source_commands import SourceProcessingInput
+from open_notebook.config import UPLOADS_FOLDER
+from open_notebook.database.repository import ensure_record_id, repo_query
from open_notebook.domain.notebook import Notebook, Source
from open_notebook.domain.transformation import Transformation
from open_notebook.exceptions import InvalidInputError
-from open_notebook.graphs.source import source_graph
router = APIRouter()
+def generate_unique_filename(original_filename: str, upload_folder: str) -> str:
+ """Generate unique filename like Streamlit app (append counter if file exists)."""
+ file_path = Path(upload_folder)
+ file_path.mkdir(parents=True, exist_ok=True)
+
+ # Split filename and extension
+ stem = Path(original_filename).stem
+ suffix = Path(original_filename).suffix
+
+ # Check if file exists and generate unique name
+ counter = 0
+ while True:
+ if counter == 0:
+ new_filename = original_filename
+ else:
+ new_filename = f"{stem} ({counter}){suffix}"
+
+ full_path = file_path / new_filename
+ if not full_path.exists():
+ return str(full_path)
+ counter += 1
+
+
+async def save_uploaded_file(upload_file: UploadFile) -> str:
+ """Save uploaded file to uploads folder and return file path."""
+ if not upload_file.filename:
+ raise ValueError("No filename provided")
+
+ # Generate unique filename
+ file_path = generate_unique_filename(upload_file.filename, UPLOADS_FOLDER)
+
+ try:
+ # Save file
+ with open(file_path, "wb") as f:
+ content = await upload_file.read()
+ f.write(content)
+
+ logger.info(f"Saved uploaded file to: {file_path}")
+ return file_path
+ except Exception as e:
+ logger.error(f"Failed to save uploaded file: {e}")
+ # Clean up partial file if it exists
+ if os.path.exists(file_path):
+ os.unlink(file_path)
+ raise
+
+
+def parse_source_form_data(
+ type: str = Form(...),
+ notebook_id: Optional[str] = Form(None),
+ notebooks: Optional[str] = Form(None), # JSON string of notebook IDs
+ url: Optional[str] = Form(None),
+ content: Optional[str] = Form(None),
+ title: Optional[str] = Form(None),
+ transformations: Optional[str] = Form(None), # JSON string of transformation IDs
+ embed: str = Form("false"), # Accept as string, convert to bool
+ delete_source: str = Form("false"), # Accept as string, convert to bool
+ async_processing: str = Form("false"), # Accept as string, convert to bool
+ file: Optional[UploadFile] = File(None),
+) -> tuple[SourceCreate, Optional[UploadFile]]:
+ """Parse form data into SourceCreate model and return upload file separately."""
+ import json
+
+ # Convert string booleans to actual booleans
+ def str_to_bool(value: str) -> bool:
+ return value.lower() in ("true", "1", "yes", "on")
+
+ embed_bool = str_to_bool(embed)
+ delete_source_bool = str_to_bool(delete_source)
+ async_processing_bool = str_to_bool(async_processing)
+
+ # Parse JSON strings
+ notebooks_list = None
+ if notebooks:
+ try:
+ notebooks_list = json.loads(notebooks)
+ except json.JSONDecodeError:
+ logger.error(f"DEBUG - Invalid JSON in notebooks field: {notebooks}")
+ raise ValueError("Invalid JSON in notebooks field")
+
+ transformations_list = []
+ if transformations:
+ try:
+ transformations_list = json.loads(transformations)
+ except json.JSONDecodeError:
+ logger.error(
+ f"DEBUG - Invalid JSON in transformations field: {transformations}"
+ )
+ raise ValueError("Invalid JSON in transformations field")
+
+ # Create SourceCreate instance
+ try:
+ source_data = SourceCreate(
+ type=type,
+ notebook_id=notebook_id,
+ notebooks=notebooks_list,
+ url=url,
+ content=content,
+ title=title,
+ file_path=None, # Will be set later if file is uploaded
+ transformations=transformations_list,
+ embed=embed_bool,
+ delete_source=delete_source_bool,
+ async_processing=async_processing_bool,
+ )
+ pass # SourceCreate instance created successfully
+ except Exception as e:
+ logger.error(f"Failed to create SourceCreate instance: {e}")
+ raise
+
+ return source_data, file
+
+
@router.get("/sources", response_model=List[SourceListResponse])
async def get_sources(
notebook_id: Optional[str] = Query(None, description="Filter by notebook ID"),
+ limit: int = Query(50, ge=1, le=100, description="Number of sources to return (1-100)"),
+ offset: int = Query(0, ge=0, description="Number of sources to skip"),
+ sort_by: str = Query("updated", description="Field to sort by (created or updated)"),
+ sort_order: str = Query("desc", description="Sort order (asc or desc)"),
):
- """Get all sources with optional notebook filtering."""
+ """Get sources with pagination and sorting support."""
try:
+ # Validate sort parameters
+ if sort_by not in ["created", "updated"]:
+ raise HTTPException(status_code=400, detail="sort_by must be 'created' or 'updated'")
+ if sort_order.lower() not in ["asc", "desc"]:
+ raise HTTPException(status_code=400, detail="sort_order must be 'asc' or 'desc'")
+
+ # Build ORDER BY clause
+ order_clause = f"ORDER BY {sort_by} {sort_order.upper()}"
+
+ # Build the query
if notebook_id:
- # Get sources for a specific notebook
+ # Verify notebook exists first
notebook = await Notebook.get(notebook_id)
if not notebook:
raise HTTPException(status_code=404, detail="Notebook not found")
- sources = await notebook.get_sources()
- else:
- # Get all sources
- sources = await Source.get_all(order_by="updated desc")
- # Create response list with async insights count
+ # Query sources for specific notebook - include command field
+ query = f"""
+ SELECT id, asset, created, title, updated, topics, command,
+ (SELECT VALUE count() FROM source_insight WHERE source = $parent.id GROUP ALL)[0].count OR 0 AS insights_count,
+ ((SELECT VALUE id FROM source_embedding WHERE source = $parent.id LIMIT 1)) != NONE AS embedded
+ FROM (select value in from reference where out=$notebook_id)
+ {order_clause}
+ LIMIT $limit START $offset
+ """
+ result = await repo_query(
+ query, {
+ "notebook_id": ensure_record_id(notebook_id),
+ "limit": limit,
+ "offset": offset
+ }
+ )
+ else:
+ # Query all sources - include command field
+ query = f"""
+ SELECT id, asset, created, title, updated, topics, command,
+ (SELECT VALUE count() FROM source_insight WHERE source = $parent.id GROUP ALL)[0].count OR 0 AS insights_count,
+ ((SELECT VALUE id FROM source_embedding WHERE source = $parent.id LIMIT 1)) != NONE AS embedded
+ FROM source
+ {order_clause}
+ LIMIT $limit START $offset
+ """
+ result = await repo_query(query, {"limit": limit, "offset": offset})
+
+ # Extract command IDs for batch status fetching
+ command_ids = []
+ command_to_source = {}
+
+ for row in result:
+ command = row.get("command")
+ if command:
+ command_str = str(command)
+ command_ids.append(command_str)
+ command_to_source[command_str] = row["id"]
+
+ # Batch fetch command statuses
+ command_statuses = {}
+ if command_ids:
+ try:
+ # Get status for all commands in batch (if the library supports it)
+ # If not, we'll fall back to individual calls, but limit concurrent requests
+ import asyncio
+
+ from surreal_commands import get_command_status
+
+ async def get_status_safe(command_id: str):
+ try:
+ status = await get_command_status(command_id)
+ return (command_id, status)
+ except Exception as e:
+ logger.warning(
+ f"Failed to get status for command {command_id}: {e}"
+ )
+ return (command_id, None)
+
+ # Limit concurrent requests to avoid overwhelming the command service
+ semaphore = asyncio.Semaphore(10)
+
+ async def get_status_with_limit(command_id: str):
+ async with semaphore:
+ return await get_status_safe(command_id)
+
+ # Fetch statuses concurrently but with limit
+ status_tasks = [get_status_with_limit(cmd_id) for cmd_id in command_ids]
+ status_results = await asyncio.gather(
+ *status_tasks, return_exceptions=True
+ )
+
+ # Process results
+ for result_item in status_results:
+ if isinstance(result_item, Exception):
+ continue
+ if isinstance(result_item, tuple) and len(result_item) == 2:
+ cmd_id, status = result_item
+ command_statuses[cmd_id] = status
+
+ except Exception as e:
+ logger.warning(f"Failed to batch fetch command statuses: {e}")
+
+ # Convert result to response model
response_list = []
- for source in sources:
- insights = await source.get_insights()
+ for row in result:
+ command = row.get("command")
+ command_id = str(command) if command else None
+ status = None
+ processing_info = None
+
+ # Get status information if command exists
+ if command_id and command_id in command_statuses:
+ status_obj = command_statuses[command_id]
+ if status_obj:
+ status = status_obj.status
+ # Extract execution metadata from nested result structure
+ result_data: dict[str, Any] | None = getattr(status_obj, "result", None)
+ execution_metadata: dict[str, Any] = result_data.get("execution_metadata", {}) if isinstance(result_data, dict) else {}
+ processing_info = {
+ "started_at": execution_metadata.get("started_at"),
+ "completed_at": execution_metadata.get("completed_at"),
+ "error": getattr(status_obj, "error_message", None),
+ }
+ elif command_id:
+ # Command exists but status couldn't be fetched
+ status = "unknown"
+
response_list.append(
SourceListResponse(
- id=source.id,
- title=source.title,
- topics=source.topics or [],
+ id=row["id"],
+ title=row.get("title"),
+ topics=row.get("topics") or [],
asset=AssetModel(
- file_path=source.asset.file_path if source.asset else None,
- url=source.asset.url if source.asset else None,
+ file_path=row["asset"].get("file_path")
+ if row.get("asset")
+ else None,
+ url=row["asset"].get("url") if row.get("asset") else None,
)
- if source.asset
+ if row.get("asset")
else None,
- embedded_chunks=await source.get_embedded_chunks(),
- insights_count=len(insights),
- created=str(source.created),
- updated=str(source.updated),
+ embedded=row.get("embedded", False),
+ embedded_chunks=0, # Removed from query - not needed in list view
+ insights_count=row.get("insights_count", 0),
+ created=str(row["created"]),
+ updated=str(row["updated"]),
+ # Status fields
+ command_id=command_id,
+ status=status,
+ processing_info=processing_info,
)
)
@@ -67,16 +317,36 @@ async def get_sources(
@router.post("/sources", response_model=SourceResponse)
-async def create_source(source_data: SourceCreate):
- """Create a new source."""
- try:
- # Verify notebook exists
- notebook = await Notebook.get(source_data.notebook_id)
- if not notebook:
- raise HTTPException(status_code=404, detail="Notebook not found")
+async def create_source(
+ form_data: tuple[SourceCreate, Optional[UploadFile]] = Depends(
+ parse_source_form_data
+ ),
+):
+ """Create a new source with support for both JSON and multipart form data."""
+ source_data, upload_file = form_data
- # Prepare content_state for source_graph
- content_state = {}
+ try:
+ # Verify all specified notebooks exist (backward compatibility support)
+ for notebook_id in (source_data.notebooks or []):
+ notebook = await Notebook.get(notebook_id)
+ if not notebook:
+ raise HTTPException(
+ status_code=404, detail=f"Notebook {notebook_id} not found"
+ )
+
+ # Handle file upload if provided
+ file_path = None
+ if upload_file and source_data.type == "upload":
+ try:
+ file_path = await save_uploaded_file(upload_file)
+ except Exception as e:
+ logger.error(f"File upload failed: {e}")
+ raise HTTPException(
+ status_code=400, detail=f"File upload failed: {str(e)}"
+ )
+
+ # Prepare content_state for processing
+ content_state: dict[str, Any] = {}
if source_data.type == "link":
if not source_data.url:
@@ -85,11 +355,14 @@ async def create_source(source_data: SourceCreate):
)
content_state["url"] = source_data.url
elif source_data.type == "upload":
- if not source_data.file_path:
+ # Use uploaded file path or provided file_path (backward compatibility)
+ final_file_path = file_path or source_data.file_path
+ if not final_file_path:
raise HTTPException(
- status_code=400, detail="File path is required for upload type"
+ status_code=400,
+ detail="File upload or file_path is required for upload type",
)
- content_state["file_path"] = source_data.file_path
+ content_state["file_path"] = final_file_path
content_state["delete_source"] = source_data.delete_source
elif source_data.type == "text":
if not source_data.content:
@@ -103,53 +376,263 @@ async def create_source(source_data: SourceCreate):
detail="Invalid source type. Must be link, upload, or text",
)
- # Get transformations to apply
- transformations = []
- if source_data.transformations:
- for trans_id in source_data.transformations:
- transformation = await Transformation.get(trans_id)
- if not transformation:
- raise HTTPException(
- status_code=404, detail=f"Transformation {trans_id} not found"
- )
- transformations.append(transformation)
+ # Validate transformations exist
+ transformation_ids = source_data.transformations or []
+ for trans_id in transformation_ids:
+ transformation = await Transformation.get(trans_id)
+ if not transformation:
+ raise HTTPException(
+ status_code=404, detail=f"Transformation {trans_id} not found"
+ )
- # Process source using the source_graph
- result = await source_graph.ainvoke(
- {
- "content_state": content_state,
- "notebook_id": source_data.notebook_id,
- "apply_transformations": transformations,
- "embed": source_data.embed,
- }
- )
+ # Branch based on processing mode
+ if source_data.async_processing:
+ # ASYNC PATH: Create source record first, then queue command
+ logger.info("Using async processing path")
- source = result["source"]
-
- return SourceResponse(
- id=source.id,
- title=source.title,
- topics=source.topics or [],
- asset=AssetModel(
- file_path=source.asset.file_path if source.asset else None,
- url=source.asset.url if source.asset else None,
+ # Create minimal source record - let SurrealDB generate the ID
+ source = Source(
+ title=source_data.title or "Processing...",
+ topics=[],
)
- if source.asset
- else None,
- full_text=source.full_text,
- embedded_chunks=await source.get_embedded_chunks(),
- created=str(source.created),
- updated=str(source.updated),
- )
+ await source.save()
+
+ # Add source to notebooks immediately so it appears in the UI
+ # The source_graph will skip adding duplicates
+ for notebook_id in (source_data.notebooks or []):
+ await source.add_to_notebook(notebook_id)
+
+ try:
+ # Import command modules to ensure they're registered
+ import commands.source_commands # noqa: F401
+
+ # Submit command for background processing
+ command_input = SourceProcessingInput(
+ source_id=str(source.id),
+ content_state=content_state,
+ notebook_ids=source_data.notebooks,
+ transformations=transformation_ids,
+ embed=source_data.embed,
+ )
+
+ command_id = await CommandService.submit_command_job(
+ "open_notebook", # app name
+ "process_source", # command name
+ command_input.model_dump(),
+ )
+
+ logger.info(f"Submitted async processing command: {command_id}")
+
+ # Update source with command reference immediately
+ # command_id already includes 'command:' prefix
+ source.command = ensure_record_id(command_id)
+ await source.save()
+
+ # Return source with command info
+ return SourceResponse(
+ id=source.id or "",
+ title=source.title,
+ topics=source.topics or [],
+ asset=None, # Will be populated after processing
+ full_text=None, # Will be populated after processing
+ embedded=False, # Will be updated after processing
+ embedded_chunks=0,
+ created=str(source.created),
+ updated=str(source.updated),
+ command_id=command_id,
+ status="new",
+ processing_info={"async": True, "queued": True},
+ )
+
+ except Exception as e:
+ logger.error(f"Failed to submit async processing command: {e}")
+ # Clean up source record on command submission failure
+ try:
+ await source.delete()
+ except Exception:
+ pass
+ # Clean up uploaded file if we created it
+ if file_path and upload_file:
+ try:
+ os.unlink(file_path)
+ except Exception:
+ pass
+ raise HTTPException(
+ status_code=500, detail=f"Failed to queue processing: {str(e)}"
+ )
+
+ else:
+ # SYNC PATH: Execute synchronously using execute_command_sync
+ logger.info("Using sync processing path")
+
+ try:
+ # Import command modules to ensure they're registered
+ import commands.source_commands # noqa: F401
+
+ # Create source record - let SurrealDB generate the ID
+ source = Source(
+ title=source_data.title or "Processing...",
+ topics=[],
+ )
+ await source.save()
+
+ # Add source to notebooks immediately so it appears in the UI
+ # The source_graph will skip adding duplicates
+ for notebook_id in (source_data.notebooks or []):
+ await source.add_to_notebook(notebook_id)
+
+ # Execute command synchronously
+ command_input = SourceProcessingInput(
+ source_id=str(source.id),
+ content_state=content_state,
+ notebook_ids=source_data.notebooks,
+ transformations=transformation_ids,
+ embed=source_data.embed,
+ )
+
+ result = execute_command_sync(
+ "open_notebook", # app name
+ "process_source", # command name
+ command_input.model_dump(),
+ timeout=300, # 5 minute timeout for sync processing
+ )
+
+ if not result.is_success():
+ logger.error(f"Sync processing failed: {result.error_message}")
+ # Clean up source record
+ try:
+ await source.delete()
+ except Exception:
+ pass
+ # Clean up uploaded file if we created it
+ if file_path and upload_file:
+ try:
+ os.unlink(file_path)
+ except Exception:
+ pass
+ raise HTTPException(
+ status_code=500,
+ detail=f"Processing failed: {result.error_message}",
+ )
+
+ # Get the processed source
+ if not source.id:
+ raise HTTPException(
+ status_code=500, detail="Source ID is missing"
+ )
+ processed_source = await Source.get(source.id)
+ if not processed_source:
+ raise HTTPException(
+ status_code=500, detail="Processed source not found"
+ )
+
+ embedded_chunks = await processed_source.get_embedded_chunks()
+ return SourceResponse(
+ id=processed_source.id or "",
+ title=processed_source.title,
+ topics=processed_source.topics or [],
+ asset=AssetModel(
+ file_path=processed_source.asset.file_path
+ if processed_source.asset
+ else None,
+ url=processed_source.asset.url
+ if processed_source.asset
+ else None,
+ )
+ if processed_source.asset
+ else None,
+ full_text=processed_source.full_text,
+ embedded=embedded_chunks > 0,
+ embedded_chunks=embedded_chunks,
+ created=str(processed_source.created),
+ updated=str(processed_source.updated),
+ # No command_id or status for sync processing (legacy behavior)
+ )
+
+ except Exception as e:
+ logger.error(f"Sync processing failed: {e}")
+ # Clean up uploaded file if we created it
+ if file_path and upload_file:
+ try:
+ os.unlink(file_path)
+ except Exception:
+ pass
+ raise
+
except HTTPException:
+ # Clean up uploaded file on HTTP exceptions if we created it
+ if file_path and upload_file:
+ try:
+ os.unlink(file_path)
+ except Exception:
+ pass
raise
except InvalidInputError as e:
+ # Clean up uploaded file on validation errors if we created it
+ if file_path and upload_file:
+ try:
+ os.unlink(file_path)
+ except Exception:
+ pass
raise HTTPException(status_code=400, detail=str(e))
except Exception as e:
logger.error(f"Error creating source: {str(e)}")
+ # Clean up uploaded file on unexpected errors if we created it
+ if file_path and upload_file:
+ try:
+ os.unlink(file_path)
+ except Exception:
+ pass
raise HTTPException(status_code=500, detail=f"Error creating source: {str(e)}")
+@router.post("/sources/json", response_model=SourceResponse)
+async def create_source_json(source_data: SourceCreate):
+ """Create a new source using JSON payload (legacy endpoint for backward compatibility)."""
+ # Convert to form data format and call main endpoint
+ form_data = (source_data, None)
+ return await create_source(form_data)
+
+
+async def _resolve_source_file(source_id: str) -> tuple[str, str]:
+ source = await Source.get(source_id)
+ if not source:
+ raise HTTPException(status_code=404, detail="Source not found")
+
+ file_path = source.asset.file_path if source.asset else None
+ if not file_path:
+ raise HTTPException(status_code=404, detail="Source has no file to download")
+
+ safe_root = os.path.realpath(UPLOADS_FOLDER)
+ resolved_path = os.path.realpath(file_path)
+
+ if not resolved_path.startswith(safe_root):
+ logger.warning(
+ f"Blocked download outside uploads directory for source {source_id}: {resolved_path}"
+ )
+ raise HTTPException(status_code=403, detail="Access to file denied")
+
+ if not os.path.exists(resolved_path):
+ raise HTTPException(status_code=404, detail="File not found on server")
+
+ filename = os.path.basename(resolved_path)
+ return resolved_path, filename
+
+
+def _is_source_file_available(source: Source) -> Optional[bool]:
+ if not source or not source.asset or not source.asset.file_path:
+ return None
+
+ file_path = source.asset.file_path
+ safe_root = os.path.realpath(UPLOADS_FOLDER)
+ resolved_path = os.path.realpath(file_path)
+
+ if not resolved_path.startswith(safe_root):
+ return False
+
+ return os.path.exists(resolved_path)
+
+
@router.get("/sources/{source_id}", response_model=SourceResponse)
async def get_source(source_id: str):
"""Get a specific source by ID."""
@@ -158,8 +641,20 @@ async def get_source(source_id: str):
if not source:
raise HTTPException(status_code=404, detail="Source not found")
+ # Get status information if command exists
+ status = None
+ processing_info = None
+ if source.command:
+ try:
+ status = await source.get_status()
+ processing_info = await source.get_processing_progress()
+ except Exception as e:
+ logger.warning(f"Failed to get status for source {source_id}: {e}")
+ status = "unknown"
+
+ embedded_chunks = await source.get_embedded_chunks()
return SourceResponse(
- id=source.id,
+ id=source.id or "",
title=source.title,
topics=source.topics or [],
asset=AssetModel(
@@ -169,9 +664,15 @@ async def get_source(source_id: str):
if source.asset
else None,
full_text=source.full_text,
- embedded_chunks=await source.get_embedded_chunks(),
+ embedded=embedded_chunks > 0,
+ embedded_chunks=embedded_chunks,
+ file_available=_is_source_file_available(source),
created=str(source.created),
updated=str(source.updated),
+ # Status fields
+ command_id=str(source.command) if source.command else None,
+ status=status,
+ processing_info=processing_info,
)
except HTTPException:
raise
@@ -180,6 +681,98 @@ async def get_source(source_id: str):
raise HTTPException(status_code=500, detail=f"Error fetching source: {str(e)}")
+@router.head("/sources/{source_id}/download")
+async def check_source_file(source_id: str):
+ """Check if a source has a downloadable file."""
+ try:
+ await _resolve_source_file(source_id)
+ return Response(status_code=200)
+ except HTTPException:
+ raise
+ except Exception as e:
+ logger.error(f"Error checking file for source {source_id}: {str(e)}")
+ raise HTTPException(status_code=500, detail="Failed to verify file")
+
+
+@router.get("/sources/{source_id}/download")
+async def download_source_file(source_id: str):
+ """Download the original file associated with an uploaded source."""
+ try:
+ resolved_path, filename = await _resolve_source_file(source_id)
+ return FileResponse(
+ path=resolved_path,
+ filename=filename,
+ media_type="application/octet-stream",
+ )
+ except HTTPException:
+ raise
+ except Exception as e:
+ logger.error(f"Error downloading file for source {source_id}: {str(e)}")
+ raise HTTPException(status_code=500, detail="Failed to download source file")
+
+
+@router.get("/sources/{source_id}/status", response_model=SourceStatusResponse)
+async def get_source_status(source_id: str):
+ """Get processing status for a source."""
+ try:
+ # First, verify source exists
+ source = await Source.get(source_id)
+ if not source:
+ raise HTTPException(status_code=404, detail="Source not found")
+
+ # Check if this is a legacy source (no command)
+ if not source.command:
+ return SourceStatusResponse(
+ status=None,
+ message="Legacy source (completed before async processing)",
+ processing_info=None,
+ command_id=None,
+ )
+
+ # Get command status and processing info
+ try:
+ status = await source.get_status()
+ processing_info = await source.get_processing_progress()
+
+ # Generate descriptive message based on status
+ if status == "completed":
+ message = "Source processing completed successfully"
+ elif status == "failed":
+ message = "Source processing failed"
+ elif status == "running":
+ message = "Source processing in progress"
+ elif status == "queued":
+ message = "Source processing queued"
+ elif status == "unknown":
+ message = "Source processing status unknown"
+ else:
+ message = f"Source processing status: {status}"
+
+ return SourceStatusResponse(
+ status=status,
+ message=message,
+ processing_info=processing_info,
+ command_id=str(source.command) if source.command else None,
+ )
+
+ except Exception as e:
+ logger.warning(f"Failed to get status for source {source_id}: {e}")
+ return SourceStatusResponse(
+ status="unknown",
+ message="Failed to retrieve processing status",
+ processing_info=None,
+ command_id=str(source.command) if source.command else None,
+ )
+
+ except HTTPException:
+ raise
+ except Exception as e:
+ logger.error(f"Error fetching status for source {source_id}: {str(e)}")
+ raise HTTPException(
+ status_code=500, detail=f"Error fetching source status: {str(e)}"
+ )
+
+
@router.put("/sources/{source_id}", response_model=SourceResponse)
async def update_source(source_id: str, source_update: SourceUpdate):
"""Update a source."""
@@ -196,8 +789,9 @@ async def update_source(source_id: str, source_update: SourceUpdate):
await source.save()
+ embedded_chunks = await source.get_embedded_chunks()
return SourceResponse(
- id=source.id,
+ id=source.id or "",
title=source.title,
topics=source.topics or [],
asset=AssetModel(
@@ -207,7 +801,8 @@ async def update_source(source_id: str, source_update: SourceUpdate):
if source.asset
else None,
full_text=source.full_text,
- embedded_chunks=await source.get_embedded_chunks(),
+ embedded=embedded_chunks > 0,
+ embedded_chunks=embedded_chunks,
created=str(source.created),
updated=str(source.updated),
)
@@ -220,6 +815,131 @@ async def update_source(source_id: str, source_update: SourceUpdate):
raise HTTPException(status_code=500, detail=f"Error updating source: {str(e)}")
+@router.post("/sources/{source_id}/retry", response_model=SourceResponse)
+async def retry_source_processing(source_id: str):
+ """Retry processing for a failed or stuck source."""
+ try:
+ # First, verify source exists
+ source = await Source.get(source_id)
+ if not source:
+ raise HTTPException(status_code=404, detail="Source not found")
+
+ # Check if source already has a running command
+ if source.command:
+ try:
+ status = await source.get_status()
+ if status in ["running", "queued"]:
+ raise HTTPException(
+ status_code=400,
+ detail="Source is already processing. Cannot retry while processing is active.",
+ )
+ except Exception as e:
+ logger.warning(
+ f"Failed to check current status for source {source_id}: {e}"
+ )
+ # Continue with retry if we can't check status
+
+ # Get notebooks that this source belongs to
+ query = "SELECT notebook FROM reference WHERE source = $source_id"
+ references = await repo_query(query, {"source_id": source_id})
+ notebook_ids = [str(ref["notebook"]) for ref in references]
+
+ if not notebook_ids:
+ raise HTTPException(
+ status_code=400, detail="Source is not associated with any notebooks"
+ )
+
+ # Prepare content_state based on source asset
+ content_state = {}
+ if source.asset:
+ if source.asset.file_path:
+ content_state = {
+ "file_path": source.asset.file_path,
+ "delete_source": False, # Don't delete on retry
+ }
+ elif source.asset.url:
+ content_state = {"url": source.asset.url}
+ else:
+ raise HTTPException(
+ status_code=400, detail="Source asset has no file_path or url"
+ )
+ else:
+ # Check if it's a text source by trying to get full_text
+ if source.full_text:
+ content_state = {"content": source.full_text}
+ else:
+ raise HTTPException(
+ status_code=400, detail="Cannot determine source content for retry"
+ )
+
+ try:
+ # Import command modules to ensure they're registered
+ import commands.source_commands # noqa: F401
+
+ # Submit new command for background processing
+ command_input = SourceProcessingInput(
+ source_id=str(source.id),
+ content_state=content_state,
+ notebook_ids=notebook_ids,
+ transformations=[], # Use default transformations on retry
+ embed=True, # Always embed on retry
+ )
+
+ command_id = await CommandService.submit_command_job(
+ "open_notebook", # app name
+ "process_source", # command name
+ command_input.model_dump(),
+ )
+
+ logger.info(
+ f"Submitted retry processing command: {command_id} for source {source_id}"
+ )
+
+ # Update source with new command ID
+ source.command = ensure_record_id(f"command:{command_id}")
+ await source.save()
+
+ # Get current embedded chunks count
+ embedded_chunks = await source.get_embedded_chunks()
+
+ # Return updated source response
+ return SourceResponse(
+ id=source.id or "",
+ title=source.title,
+ topics=source.topics or [],
+ asset=AssetModel(
+ file_path=source.asset.file_path if source.asset else None,
+ url=source.asset.url if source.asset else None,
+ )
+ if source.asset
+ else None,
+ full_text=source.full_text,
+ embedded=embedded_chunks > 0,
+ embedded_chunks=embedded_chunks,
+ created=str(source.created),
+ updated=str(source.updated),
+ command_id=command_id,
+ status="queued",
+ processing_info={"retry": True, "queued": True},
+ )
+
+ except Exception as e:
+ logger.error(
+ f"Failed to submit retry processing command for source {source_id}: {e}"
+ )
+ raise HTTPException(
+ status_code=500, detail=f"Failed to queue retry processing: {str(e)}"
+ )
+
+ except HTTPException:
+ raise
+ except Exception as e:
+ logger.error(f"Error retrying source processing for {source_id}: {str(e)}")
+ raise HTTPException(
+ status_code=500, detail=f"Error retrying source processing: {str(e)}"
+ )
+
+
@router.delete("/sources/{source_id}")
async def delete_source(source_id: str):
"""Delete a source."""
@@ -245,16 +965,16 @@ async def get_source_insights(source_id: str):
source = await Source.get(source_id)
if not source:
raise HTTPException(status_code=404, detail="Source not found")
-
+
insights = await source.get_insights()
return [
SourceInsightResponse(
- id=insight.id,
+ id=insight.id or "",
source_id=source_id,
insight_type=insight.insight_type,
content=insight.content,
created=str(insight.created),
- updated=str(insight.updated)
+ updated=str(insight.updated),
)
for insight in insights
]
@@ -262,47 +982,47 @@ async def get_source_insights(source_id: str):
raise
except Exception as e:
logger.error(f"Error fetching insights for source {source_id}: {str(e)}")
- raise HTTPException(status_code=500, detail=f"Error fetching insights: {str(e)}")
+ raise HTTPException(
+ status_code=500, detail=f"Error fetching insights: {str(e)}"
+ )
@router.post("/sources/{source_id}/insights", response_model=SourceInsightResponse)
-async def create_source_insight(
- source_id: str,
- request: CreateSourceInsightRequest
-):
+async def create_source_insight(source_id: str, request: CreateSourceInsightRequest):
"""Create a new insight for a source by running a transformation."""
try:
# Get source
source = await Source.get(source_id)
if not source:
raise HTTPException(status_code=404, detail="Source not found")
-
+
# Get transformation
transformation = await Transformation.get(request.transformation_id)
if not transformation:
raise HTTPException(status_code=404, detail="Transformation not found")
-
+
# Run transformation graph
from open_notebook.graphs.transformation import graph as transform_graph
+
await transform_graph.ainvoke(
- input=dict(source=source, transformation=transformation)
+ input=dict(source=source, transformation=transformation) # type: ignore[arg-type]
)
-
+
# Get the newly created insight (last one)
insights = await source.get_insights()
if insights:
newest = insights[-1]
return SourceInsightResponse(
- id=newest.id,
+ id=newest.id or "",
source_id=source_id,
insight_type=newest.insight_type,
content=newest.content,
created=str(newest.created),
- updated=str(newest.updated)
+ updated=str(newest.updated),
)
else:
raise HTTPException(status_code=500, detail="Failed to create insight")
-
+
except HTTPException:
raise
except Exception as e:
diff --git a/api/routers/speaker_profiles.py b/api/routers/speaker_profiles.py
index 68700b8..3e3366d 100644
--- a/api/routers/speaker_profiles.py
+++ b/api/routers/speaker_profiles.py
@@ -1,11 +1,11 @@
-from typing import List, Dict, Any
+from typing import Any, Dict, List
+
from fastapi import APIRouter, HTTPException
-from pydantic import BaseModel, Field
from loguru import logger
+from pydantic import BaseModel, Field
from open_notebook.domain.podcast import SpeakerProfile
-
router = APIRouter()
diff --git a/api/routers/transformations.py b/api/routers/transformations.py
index 60465bd..ac0e73c 100644
--- a/api/routers/transformations.py
+++ b/api/routers/transformations.py
@@ -4,6 +4,8 @@ from fastapi import APIRouter, HTTPException
from loguru import logger
from api.models import (
+ DefaultPromptResponse,
+ DefaultPromptUpdate,
TransformationCreate,
TransformationExecuteRequest,
TransformationExecuteResponse,
@@ -11,8 +13,8 @@ from api.models import (
TransformationUpdate,
)
from open_notebook.domain.models import Model
-from open_notebook.domain.transformation import Transformation
-from open_notebook.exceptions import DatabaseOperationError, InvalidInputError
+from open_notebook.domain.transformation import DefaultPrompts, Transformation
+from open_notebook.exceptions import InvalidInputError
from open_notebook.graphs.transformation import graph as transformation_graph
router = APIRouter()
@@ -26,7 +28,7 @@ async def get_transformations():
return [
TransformationResponse(
- id=transformation.id,
+ id=transformation.id or "",
name=transformation.name,
title=transformation.title,
description=transformation.description,
@@ -58,7 +60,7 @@ async def create_transformation(transformation_data: TransformationCreate):
await new_transformation.save()
return TransformationResponse(
- id=new_transformation.id,
+ id=new_transformation.id or "",
name=new_transformation.name,
title=new_transformation.title,
description=new_transformation.description,
@@ -87,7 +89,7 @@ async def get_transformation(transformation_id: str):
raise HTTPException(status_code=404, detail="Transformation not found")
return TransformationResponse(
- id=transformation.id,
+ id=transformation.id or "",
name=transformation.name,
title=transformation.title,
description=transformation.description,
@@ -132,7 +134,7 @@ async def update_transformation(
await transformation.save()
return TransformationResponse(
- id=transformation.id,
+ id=transformation.id or "",
name=transformation.name,
title=transformation.title,
description=transformation.description,
@@ -188,7 +190,7 @@ async def execute_transformation(execute_request: TransformationExecuteRequest):
# Execute the transformation
result = await transformation_graph.ainvoke(
- dict(
+ dict( # type: ignore[arg-type]
input_text=execute_request.input_text,
transformation=transformation,
),
@@ -208,3 +210,38 @@ async def execute_transformation(execute_request: TransformationExecuteRequest):
raise HTTPException(
status_code=500, detail=f"Error executing transformation: {str(e)}"
)
+
+
+@router.get("/transformations/default-prompt", response_model=DefaultPromptResponse)
+async def get_default_prompt():
+ """Get the default transformation prompt."""
+ try:
+ default_prompts: DefaultPrompts = await DefaultPrompts.get_instance() # type: ignore[assignment]
+
+ return DefaultPromptResponse(
+ transformation_instructions=default_prompts.transformation_instructions or ""
+ )
+ except Exception as e:
+ logger.error(f"Error fetching default prompt: {str(e)}")
+ raise HTTPException(
+ status_code=500, detail=f"Error fetching default prompt: {str(e)}"
+ )
+
+
+@router.put("/transformations/default-prompt", response_model=DefaultPromptResponse)
+async def update_default_prompt(prompt_update: DefaultPromptUpdate):
+ """Update the default transformation prompt."""
+ try:
+ default_prompts: DefaultPrompts = await DefaultPrompts.get_instance() # type: ignore[assignment]
+
+ default_prompts.transformation_instructions = prompt_update.transformation_instructions
+ await default_prompts.update()
+
+ return DefaultPromptResponse(
+ transformation_instructions=default_prompts.transformation_instructions
+ )
+ except Exception as e:
+ logger.error(f"Error updating default prompt: {str(e)}")
+ raise HTTPException(
+ status_code=500, detail=f"Error updating default prompt: {str(e)}"
+ )
diff --git a/api/search_service.py b/api/search_service.py
index 22f823d..07d7b6f 100644
--- a/api/search_service.py
+++ b/api/search_service.py
@@ -2,7 +2,7 @@
Search service layer using API.
"""
-from typing import Dict, List, Any
+from typing import Any, Dict, List, Union
from loguru import logger
@@ -11,12 +11,12 @@ from api.client import api_client
class SearchService:
"""Service layer for search operations using API."""
-
+
def __init__(self):
logger.info("Using API for search operations")
-
+
def search(
- self,
+ self,
query: str,
search_type: str = "text",
limit: int = 100,
@@ -33,15 +33,17 @@ class SearchService:
search_notes=search_notes,
minimum_score=minimum_score
)
- return response.get("results", [])
-
+ if isinstance(response, dict):
+ return response.get("results", [])
+ return []
+
def ask_knowledge_base(
self,
question: str,
strategy_model: str,
answer_model: str,
final_answer_model: str
- ) -> Dict[str, str]:
+ ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
"""Ask the knowledge base a question."""
response = api_client.ask_simple(
question=question,
diff --git a/api/settings_service.py b/api/settings_service.py
index e9d1504..ed84e02 100644
--- a/api/settings_service.py
+++ b/api/settings_service.py
@@ -2,7 +2,6 @@
Settings service layer using API.
"""
-from typing import Dict
from loguru import logger
@@ -18,8 +17,9 @@ class SettingsService:
def get_settings(self) -> ContentSettings:
"""Get application settings."""
- settings_data = api_client.get_settings()
-
+ settings_response = api_client.get_settings()
+ settings_data = settings_response if isinstance(settings_response, dict) else settings_response[0]
+
# Create ContentSettings object from API response
settings = ContentSettings(
default_content_processing_engine_doc=settings_data.get("default_content_processing_engine_doc"),
@@ -28,7 +28,7 @@ class SettingsService:
auto_delete_files=settings_data.get("auto_delete_files"),
youtube_preferred_languages=settings_data.get("youtube_preferred_languages"),
)
-
+
return settings
def update_settings(self, settings: ContentSettings) -> ContentSettings:
@@ -40,16 +40,17 @@ class SettingsService:
"auto_delete_files": settings.auto_delete_files,
"youtube_preferred_languages": settings.youtube_preferred_languages,
}
-
- settings_data = api_client.update_settings(**updates)
-
+
+ settings_response = api_client.update_settings(**updates)
+ settings_data = settings_response if isinstance(settings_response, dict) else settings_response[0]
+
# Update the settings object with the response
settings.default_content_processing_engine_doc = settings_data.get("default_content_processing_engine_doc")
settings.default_content_processing_engine_url = settings_data.get("default_content_processing_engine_url")
settings.default_embedding_option = settings_data.get("default_embedding_option")
settings.auto_delete_files = settings_data.get("auto_delete_files")
settings.youtube_preferred_languages = settings_data.get("youtube_preferred_languages")
-
+
return settings
diff --git a/api/sources_service.py b/api/sources_service.py
index 03a123b..6e3fa3b 100644
--- a/api/sources_service.py
+++ b/api/sources_service.py
@@ -3,7 +3,7 @@ Sources service layer using API.
"""
from dataclasses import dataclass
-from typing import List, Optional
+from typing import Dict, List, Optional, Union
from loguru import logger
@@ -11,6 +11,16 @@ from api.client import api_client
from open_notebook.domain.notebook import Asset, Source
+@dataclass
+class SourceProcessingResult:
+ """Result of source creation with optional async processing info."""
+ source: Source
+ is_async: bool = False
+ command_id: Optional[str] = None
+ status: Optional[str] = None
+ processing_info: Optional[Dict] = None
+
+
@dataclass
class SourceWithMetadata:
"""Source object with additional metadata from API."""
@@ -89,7 +99,8 @@ class SourcesService:
def get_source(self, source_id: str) -> SourceWithMetadata:
"""Get a specific source."""
- source_data = api_client.get_source(source_id)
+ response = api_client.get_source(source_id)
+ source_data = response if isinstance(response, dict) else response[0]
source = Source(
title=source_data["title"],
topics=source_data["topics"],
@@ -106,7 +117,7 @@ class SourcesService:
source.id = source_data["id"]
source.created = source_data["created"]
source.updated = source_data["updated"]
-
+
return SourceWithMetadata(
source=source,
embedded_chunks=source_data.get("embedded_chunks", 0)
@@ -114,8 +125,8 @@ class SourcesService:
def create_source(
self,
- notebook_id: str,
- source_type: str,
+ notebook_id: Optional[str] = None,
+ source_type: str = "text",
url: Optional[str] = None,
file_path: Optional[str] = None,
content: Optional[str] = None,
@@ -123,10 +134,32 @@ class SourcesService:
transformations: Optional[List[str]] = None,
embed: bool = False,
delete_source: bool = False,
- ) -> Source:
- """Create a new source."""
+ notebooks: Optional[List[str]] = None,
+ async_processing: bool = False,
+ ) -> Union[Source, SourceProcessingResult]:
+ """
+ Create a new source with support for async processing.
+
+ Args:
+ notebook_id: Single notebook ID (deprecated, use notebooks parameter)
+ source_type: Type of source (link, upload, text)
+ url: URL for link sources
+ file_path: File path for upload sources
+ content: Text content for text sources
+ title: Optional source title
+ transformations: List of transformation IDs to apply
+ embed: Whether to embed content for vector search
+ delete_source: Whether to delete uploaded file after processing
+ notebooks: List of notebook IDs to add source to (preferred over notebook_id)
+ async_processing: Whether to process source asynchronously
+
+ Returns:
+ Source object for sync processing (backward compatibility)
+ SourceProcessingResult for async processing (contains additional metadata)
+ """
source_data = api_client.create_source(
notebook_id=notebook_id,
+ notebooks=notebooks,
source_type=source_type,
url=url,
file_path=file_path,
@@ -135,25 +168,108 @@ class SourcesService:
transformations=transformations,
embed=embed,
delete_source=delete_source,
+ async_processing=async_processing,
)
+ # Create Source object from response
+ response_data = source_data if isinstance(source_data, dict) else source_data[0]
source = Source(
- title=source_data["title"],
- topics=source_data["topics"],
- full_text=source_data["full_text"],
+ title=response_data["title"],
+ topics=response_data.get("topics") or [],
+ full_text=response_data.get("full_text"),
asset=Asset(
- file_path=source_data["asset"]["file_path"]
- if source_data["asset"]
+ file_path=response_data["asset"]["file_path"]
+ if response_data.get("asset")
+ else None,
+ url=response_data["asset"]["url"]
+ if response_data.get("asset")
else None,
- url=source_data["asset"]["url"] if source_data["asset"] else None,
)
- if source_data["asset"]
+ if response_data.get("asset")
else None,
)
- source.id = source_data["id"]
- source.created = source_data["created"]
- source.updated = source_data["updated"]
- return source
+ source.id = response_data["id"]
+ source.created = response_data["created"]
+ source.updated = response_data["updated"]
+
+ # Check if this is an async processing response
+ if response_data.get("command_id") or response_data.get("status") or response_data.get("processing_info"):
+ # Ensure source_data is a dict for accessing attributes
+ source_data_dict = source_data if isinstance(source_data, dict) else source_data[0]
+ # Return enhanced result for async processing
+ return SourceProcessingResult(
+ source=source,
+ is_async=True,
+ command_id=source_data_dict.get("command_id"),
+ status=source_data_dict.get("status"),
+ processing_info=source_data_dict.get("processing_info"),
+ )
+ else:
+ # Return simple Source for backward compatibility
+ return source
+
+ def get_source_status(self, source_id: str) -> Dict:
+ """Get processing status for a source."""
+ response = api_client.get_source_status(source_id)
+ return response if isinstance(response, dict) else response[0]
+
+ def create_source_async(
+ self,
+ notebook_id: Optional[str] = None,
+ source_type: str = "text",
+ url: Optional[str] = None,
+ file_path: Optional[str] = None,
+ content: Optional[str] = None,
+ title: Optional[str] = None,
+ transformations: Optional[List[str]] = None,
+ embed: bool = False,
+ delete_source: bool = False,
+ notebooks: Optional[List[str]] = None,
+ ) -> SourceProcessingResult:
+ """
+ Create a new source with async processing enabled.
+
+ This is a convenience method that always uses async processing.
+ Returns a SourceProcessingResult with processing status information.
+ """
+ result = self.create_source(
+ notebook_id=notebook_id,
+ notebooks=notebooks,
+ source_type=source_type,
+ url=url,
+ file_path=file_path,
+ content=content,
+ title=title,
+ transformations=transformations,
+ embed=embed,
+ delete_source=delete_source,
+ async_processing=True,
+ )
+
+ # Since we forced async_processing=True, this should always be a SourceProcessingResult
+ if isinstance(result, SourceProcessingResult):
+ return result
+ else:
+ # Fallback: wrap Source in SourceProcessingResult
+ return SourceProcessingResult(
+ source=result,
+ is_async=False, # This shouldn't happen, but handle it gracefully
+ )
+
+ def is_source_processing_complete(self, source_id: str) -> bool:
+ """
+ Check if a source's async processing is complete.
+
+ Returns True if processing is complete (success or failure),
+ False if still processing or queued.
+ """
+ try:
+ status_data = self.get_source_status(source_id)
+ status = status_data.get("status")
+ return status in ["completed", "failed", None] # None indicates legacy/sync source
+ except Exception as e:
+ logger.error(f"Error checking source processing status: {e}")
+ return True # Assume complete on error
def update_source(self, source: Source) -> Source:
"""Update a source."""
@@ -166,10 +282,13 @@ class SourcesService:
}
source_data = api_client.update_source(source.id, **updates)
+ # Ensure source_data is a dict
+ source_data_dict = source_data if isinstance(source_data, dict) else source_data[0]
+
# Update the source object with the response
- source.title = source_data["title"]
- source.topics = source_data["topics"]
- source.updated = source_data["updated"]
+ source.title = source_data_dict["title"]
+ source.topics = source_data_dict["topics"]
+ source.updated = source_data_dict["updated"]
return source
@@ -181,3 +300,6 @@ class SourcesService:
# Global service instance
sources_service = SourcesService()
+
+# Export important classes for easy importing
+__all__ = ["SourcesService", "SourceWithMetadata", "SourceProcessingResult", "sources_service"]
diff --git a/api/transformations_service.py b/api/transformations_service.py
index 6821bf3..876b9a9 100644
--- a/api/transformations_service.py
+++ b/api/transformations_service.py
@@ -3,7 +3,7 @@ Transformations service layer using API.
"""
from datetime import datetime
-from typing import Dict, List
+from typing import Any, Dict, List, Union
from loguru import logger
@@ -38,7 +38,8 @@ class TransformationsService:
def get_transformation(self, transformation_id: str) -> Transformation:
"""Get a specific transformation."""
- trans_data = api_client.get_transformation(transformation_id)
+ response = api_client.get_transformation(transformation_id)
+ trans_data = response if isinstance(response, dict) else response[0]
transformation = Transformation(
name=trans_data["name"],
title=trans_data["title"],
@@ -60,13 +61,14 @@ class TransformationsService:
apply_default: bool = False
) -> Transformation:
"""Create a new transformation."""
- trans_data = api_client.create_transformation(
+ response = api_client.create_transformation(
name=name,
title=title,
description=description,
prompt=prompt,
apply_default=apply_default
)
+ trans_data = response if isinstance(response, dict) else response[0]
transformation = Transformation(
name=trans_data["name"],
title=trans_data["title"],
@@ -81,6 +83,9 @@ class TransformationsService:
def update_transformation(self, transformation: Transformation) -> Transformation:
"""Update a transformation."""
+ if not transformation.id:
+ raise ValueError("Transformation ID is required for update")
+
updates = {
"name": transformation.name,
"title": transformation.title,
@@ -88,8 +93,9 @@ class TransformationsService:
"prompt": transformation.prompt,
"apply_default": transformation.apply_default,
}
- trans_data = api_client.update_transformation(transformation.id, **updates)
-
+ response = api_client.update_transformation(transformation.id, **updates)
+ trans_data = response if isinstance(response, dict) else response[0]
+
# Update the transformation object with the response
transformation.name = trans_data["name"]
transformation.title = trans_data["title"]
@@ -97,7 +103,7 @@ class TransformationsService:
transformation.prompt = trans_data["prompt"]
transformation.apply_default = trans_data["apply_default"]
transformation.updated = datetime.fromisoformat(trans_data["updated"].replace('Z', '+00:00'))
-
+
return transformation
def delete_transformation(self, transformation_id: str) -> bool:
@@ -110,7 +116,7 @@ class TransformationsService:
transformation_id: str,
input_text: str,
model_id: str
- ) -> Dict[str, str]:
+ ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
"""Execute a transformation on input text."""
result = api_client.execute_transformation(
transformation_id=transformation_id,
diff --git a/app_home.py b/app_home.py
index b78e236..21a88d9 100644
--- a/app_home.py
+++ b/app_home.py
@@ -1,13 +1,9 @@
-import asyncio
import nest_asyncio
import streamlit as st
from dotenv import load_dotenv
-from open_notebook.domain.base import ObjectModel
-
nest_asyncio.apply()
-from open_notebook.exceptions import NotFoundError
from pages.components import note_panel, source_insight_panel, source_panel
from pages.stream_app.utils import setup_page
diff --git a/batch_fix_services.py b/batch_fix_services.py
new file mode 100644
index 0000000..4db32b6
--- /dev/null
+++ b/batch_fix_services.py
@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+"""Batch fix service files for mypy errors."""
+import re
+from pathlib import Path
+
+SERVICE_FILES = [
+ 'api/notes_service.py',
+ 'api/insights_service.py',
+ 'api/episode_profiles_service.py',
+ 'api/settings_service.py',
+ 'api/sources_service.py',
+ 'api/podcast_service.py',
+ 'api/command_service.py',
+]
+
+BASE_DIR = Path('/Users/luisnovo/dev/projetos/open-notebook/open-notebook')
+
+for service_file in SERVICE_FILES:
+ file_path = BASE_DIR / service_file
+ if not file_path.exists():
+ print(f"Skipping {service_file} - file not found")
+ continue
+
+ content = file_path.read_text()
+ original_content = content
+
+ # Pattern to find: var_name = api_client.method(args)
+ # Followed by: var_name["key"] or var_name.get("key")
+ lines = content.split('\n')
+ new_lines = []
+ i = 0
+
+ while i < len(lines):
+ line = lines[i]
+
+ # Check if this line has an api_client call assignment
+ match = re.match(r'(\s*)(\w+)\s*=\s*api_client\.(\w+)\((.*)\)\s*$', line)
+ if match and 'response = api_client' not in line:
+ indent = match.group(1)
+ var_name = match.group(2)
+ method_name = match.group(3)
+ args = match.group(4)
+
+ # Look ahead to see if this variable is used with dict access
+ has_dict_access = False
+ for j in range(i+1, min(i+15, len(lines))):
+ next_line = lines[j]
+ if f'{var_name}["' in next_line or f"{var_name}['" in next_line or f'{var_name}.get(' in next_line:
+ has_dict_access = True
+ break
+ # Stop looking if we hit a blank line, new function, or new assignment
+ if (not next_line.strip() or
+ next_line.strip().startswith('def ') or
+ next_line.strip().startswith('class ') or
+ (re.match(r'\s*\w+\s*=', next_line) and var_name not in next_line)):
+ break
+
+ if has_dict_access:
+ # Replace with response and isinstance check
+ new_lines.append(f'{indent}response = api_client.{method_name}({args})')
+ new_lines.append(f'{indent}{var_name} = response if isinstance(response, dict) else response[0]')
+ i += 1
+ continue
+
+ new_lines.append(line)
+ i += 1
+
+ new_content = '\n'.join(new_lines)
+
+ # Check if content changed
+ if new_content != original_content:
+ file_path.write_text(new_content)
+ print(f"✓ Fixed {service_file}")
+ else:
+ print(f"- No changes needed for {service_file}")
+
+print("\nDone!")
diff --git a/commands/__init__.py b/commands/__init__.py
index e50e558..cd7fb89 100644
--- a/commands/__init__.py
+++ b/commands/__init__.py
@@ -1,10 +1,15 @@
"""Surreal-commands integration for Open Notebook"""
+from .embedding_commands import embed_single_item_command, rebuild_embeddings_command
from .example_commands import analyze_data_command, process_text_command
from .podcast_commands import generate_podcast_command
+from .source_commands import process_source_command
__all__ = [
+ "embed_single_item_command",
"generate_podcast_command",
+ "process_source_command",
"process_text_command",
"analyze_data_command",
+ "rebuild_embeddings_command",
]
diff --git a/commands/embedding_commands.py b/commands/embedding_commands.py
new file mode 100644
index 0000000..055e632
--- /dev/null
+++ b/commands/embedding_commands.py
@@ -0,0 +1,392 @@
+import time
+from typing import Dict, List, Literal, Optional
+
+from loguru import logger
+from pydantic import BaseModel
+from surreal_commands import CommandInput, CommandOutput, command
+
+from open_notebook.database.repository import ensure_record_id, repo_query
+from open_notebook.domain.models import model_manager
+from open_notebook.domain.notebook import Note, Source, SourceInsight
+
+
+def full_model_dump(model):
+ if isinstance(model, BaseModel):
+ return model.model_dump()
+ elif isinstance(model, dict):
+ return {k: full_model_dump(v) for k, v in model.items()}
+ elif isinstance(model, list):
+ return [full_model_dump(item) for item in model]
+ else:
+ return model
+
+
+class EmbedSingleItemInput(CommandInput):
+ item_id: str
+ item_type: Literal["source", "note", "insight"]
+
+
+class EmbedSingleItemOutput(CommandOutput):
+ success: bool
+ item_id: str
+ item_type: str
+ chunks_created: int = 0 # For sources
+ processing_time: float
+ error_message: Optional[str] = None
+
+
+class RebuildEmbeddingsInput(CommandInput):
+ mode: Literal["existing", "all"]
+ include_sources: bool = True
+ include_notes: bool = True
+ include_insights: bool = True
+
+
+class RebuildEmbeddingsOutput(CommandOutput):
+ success: bool
+ total_items: int
+ processed_items: int
+ failed_items: int
+ sources_processed: int = 0
+ notes_processed: int = 0
+ insights_processed: int = 0
+ processing_time: float
+ error_message: Optional[str] = None
+
+
+@command("embed_single_item", app="open_notebook")
+async def embed_single_item_command(
+ input_data: EmbedSingleItemInput,
+) -> EmbedSingleItemOutput:
+ """
+ Embed a single item (source, note, or insight)
+ """
+ start_time = time.time()
+
+ try:
+ logger.info(
+ f"Starting embedding for {input_data.item_type}: {input_data.item_id}"
+ )
+
+ # Check if embedding model is available
+ EMBEDDING_MODEL = await model_manager.get_embedding_model()
+ if not EMBEDDING_MODEL:
+ raise ValueError(
+ "No embedding model configured. Please configure one in the Models section."
+ )
+
+ chunks_created = 0
+
+ if input_data.item_type == "source":
+ # Get source and vectorize
+ source = await Source.get(input_data.item_id)
+ if not source:
+ raise ValueError(f"Source '{input_data.item_id}' not found")
+
+ await source.vectorize()
+
+ # Count chunks created
+ chunks_result = await repo_query(
+ "SELECT VALUE count() FROM source_embedding WHERE source = $source_id GROUP ALL",
+ {"source_id": ensure_record_id(input_data.item_id)},
+ )
+ if chunks_result and isinstance(chunks_result[0], dict):
+ chunks_created = chunks_result[0].get("count", 0)
+ elif chunks_result and isinstance(chunks_result[0], int):
+ chunks_created = chunks_result[0]
+ else:
+ chunks_created = 0
+
+ logger.info(f"Source vectorized: {chunks_created} chunks created")
+
+ elif input_data.item_type == "note":
+ # Get note and save (auto-embeds via ObjectModel.save())
+ note = await Note.get(input_data.item_id)
+ if not note:
+ raise ValueError(f"Note '{input_data.item_id}' not found")
+
+ await note.save()
+ logger.info(f"Note embedded: {input_data.item_id}")
+
+ elif input_data.item_type == "insight":
+ # Get insight and re-generate embedding
+ insight = await SourceInsight.get(input_data.item_id)
+ if not insight:
+ raise ValueError(f"Insight '{input_data.item_id}' not found")
+
+ # Generate new embedding
+ embedding = (await EMBEDDING_MODEL.aembed([insight.content]))[0]
+
+ # Update insight with new embedding
+ await repo_query(
+ "UPDATE $insight_id SET embedding = $embedding",
+ {
+ "insight_id": ensure_record_id(input_data.item_id),
+ "embedding": embedding,
+ },
+ )
+ logger.info(f"Insight embedded: {input_data.item_id}")
+
+ else:
+ raise ValueError(
+ f"Invalid item_type: {input_data.item_type}. Must be 'source', 'note', or 'insight'"
+ )
+
+ processing_time = time.time() - start_time
+ logger.info(
+ f"Successfully embedded {input_data.item_type} {input_data.item_id} in {processing_time:.2f}s"
+ )
+
+ return EmbedSingleItemOutput(
+ success=True,
+ item_id=input_data.item_id,
+ item_type=input_data.item_type,
+ chunks_created=chunks_created,
+ processing_time=processing_time,
+ )
+
+ except Exception as e:
+ processing_time = time.time() - start_time
+ logger.error(f"Embedding failed for {input_data.item_type} {input_data.item_id}: {e}")
+ logger.exception(e)
+
+ return EmbedSingleItemOutput(
+ success=False,
+ item_id=input_data.item_id,
+ item_type=input_data.item_type,
+ processing_time=processing_time,
+ error_message=str(e),
+ )
+
+
+async def collect_items_for_rebuild(
+ mode: str,
+ include_sources: bool,
+ include_notes: bool,
+ include_insights: bool,
+) -> Dict[str, List[str]]:
+ """
+ Collect items to rebuild based on mode and include flags.
+
+ Returns:
+ Dict with keys: 'sources', 'notes', 'insights' containing lists of item IDs
+ """
+ items: Dict[str, List[str]] = {"sources": [], "notes": [], "insights": []}
+
+ if include_sources:
+ if mode == "existing":
+ # Query sources with embeddings (via source_embedding table)
+ result = await repo_query(
+ """
+ RETURN array::distinct(
+ SELECT VALUE source.id
+ FROM source_embedding
+ WHERE embedding != none AND array::len(embedding) > 0
+ )
+ """
+ )
+ # RETURN returns the array directly as the result (not nested)
+ if result:
+ items["sources"] = [str(item) for item in result]
+ else:
+ items["sources"] = []
+ else: # mode == "all"
+ # Query all sources with content
+ result = await repo_query("SELECT id FROM source WHERE full_text != none")
+ items["sources"] = [str(item["id"]) for item in result] if result else []
+
+ logger.info(f"Collected {len(items['sources'])} sources for rebuild")
+
+ if include_notes:
+ if mode == "existing":
+ # Query notes with embeddings
+ result = await repo_query(
+ "SELECT id FROM note WHERE embedding != none AND array::len(embedding) > 0"
+ )
+ else: # mode == "all"
+ # Query all notes (with content)
+ result = await repo_query("SELECT id FROM note WHERE content != none")
+
+ items["notes"] = [str(item["id"]) for item in result] if result else []
+ logger.info(f"Collected {len(items['notes'])} notes for rebuild")
+
+ if include_insights:
+ if mode == "existing":
+ # Query insights with embeddings
+ result = await repo_query(
+ "SELECT id FROM source_insight WHERE embedding != none AND array::len(embedding) > 0"
+ )
+ else: # mode == "all"
+ # Query all insights
+ result = await repo_query("SELECT id FROM source_insight")
+
+ items["insights"] = [str(item["id"]) for item in result] if result else []
+ logger.info(f"Collected {len(items['insights'])} insights for rebuild")
+
+ return items
+
+
+@command("rebuild_embeddings", app="open_notebook")
+async def rebuild_embeddings_command(
+ input_data: RebuildEmbeddingsInput,
+) -> RebuildEmbeddingsOutput:
+ """
+ Rebuild embeddings for sources, notes, and/or insights
+ """
+ start_time = time.time()
+
+ try:
+ logger.info("=" * 60)
+ logger.info(f"Starting embedding rebuild with mode={input_data.mode}")
+ logger.info(f"Include: sources={input_data.include_sources}, notes={input_data.include_notes}, insights={input_data.include_insights}")
+ logger.info("=" * 60)
+
+ # Check embedding model availability
+ EMBEDDING_MODEL = await model_manager.get_embedding_model()
+ if not EMBEDDING_MODEL:
+ raise ValueError(
+ "No embedding model configured. Please configure one in the Models section."
+ )
+
+ logger.info(f"Using embedding model: {EMBEDDING_MODEL}")
+
+ # Collect items to process
+ items = await collect_items_for_rebuild(
+ input_data.mode,
+ input_data.include_sources,
+ input_data.include_notes,
+ input_data.include_insights,
+ )
+
+ total_items = (
+ len(items["sources"]) + len(items["notes"]) + len(items["insights"])
+ )
+ logger.info(f"Total items to process: {total_items}")
+
+ if total_items == 0:
+ logger.warning("No items found to rebuild")
+ return RebuildEmbeddingsOutput(
+ success=True,
+ total_items=0,
+ processed_items=0,
+ failed_items=0,
+ processing_time=time.time() - start_time,
+ )
+
+ # Initialize counters
+ sources_processed = 0
+ notes_processed = 0
+ insights_processed = 0
+ failed_items = 0
+
+ # Process sources
+ logger.info(f"\nProcessing {len(items['sources'])} sources...")
+ for idx, source_id in enumerate(items["sources"], 1):
+ try:
+ source = await Source.get(source_id)
+ if not source:
+ logger.warning(f"Source {source_id} not found, skipping")
+ failed_items += 1
+ continue
+
+ await source.vectorize()
+ sources_processed += 1
+
+ if idx % 10 == 0 or idx == len(items["sources"]):
+ logger.info(
+ f" Progress: {idx}/{len(items['sources'])} sources processed"
+ )
+
+ except Exception as e:
+ logger.error(f"Failed to re-embed source {source_id}: {e}")
+ failed_items += 1
+
+ # Process notes
+ logger.info(f"\nProcessing {len(items['notes'])} notes...")
+ for idx, note_id in enumerate(items["notes"], 1):
+ try:
+ note = await Note.get(note_id)
+ if not note:
+ logger.warning(f"Note {note_id} not found, skipping")
+ failed_items += 1
+ continue
+
+ await note.save() # Auto-embeds via ObjectModel.save()
+ notes_processed += 1
+
+ if idx % 10 == 0 or idx == len(items["notes"]):
+ logger.info(f" Progress: {idx}/{len(items['notes'])} notes processed")
+
+ except Exception as e:
+ logger.error(f"Failed to re-embed note {note_id}: {e}")
+ failed_items += 1
+
+ # Process insights
+ logger.info(f"\nProcessing {len(items['insights'])} insights...")
+ for idx, insight_id in enumerate(items["insights"], 1):
+ try:
+ insight = await SourceInsight.get(insight_id)
+ if not insight:
+ logger.warning(f"Insight {insight_id} not found, skipping")
+ failed_items += 1
+ continue
+
+ # Re-generate embedding
+ embedding = (await EMBEDDING_MODEL.aembed([insight.content]))[0]
+
+ # Update insight with new embedding
+ await repo_query(
+ "UPDATE $insight_id SET embedding = $embedding",
+ {
+ "insight_id": ensure_record_id(insight_id),
+ "embedding": embedding,
+ },
+ )
+ insights_processed += 1
+
+ if idx % 10 == 0 or idx == len(items["insights"]):
+ logger.info(
+ f" Progress: {idx}/{len(items['insights'])} insights processed"
+ )
+
+ except Exception as e:
+ logger.error(f"Failed to re-embed insight {insight_id}: {e}")
+ failed_items += 1
+
+ processing_time = time.time() - start_time
+ processed_items = sources_processed + notes_processed + insights_processed
+
+ logger.info("=" * 60)
+ logger.info("REBUILD COMPLETE")
+ logger.info(f" Total processed: {processed_items}/{total_items}")
+ logger.info(f" Sources: {sources_processed}")
+ logger.info(f" Notes: {notes_processed}")
+ logger.info(f" Insights: {insights_processed}")
+ logger.info(f" Failed: {failed_items}")
+ logger.info(f" Time: {processing_time:.2f}s")
+ logger.info("=" * 60)
+
+ return RebuildEmbeddingsOutput(
+ success=True,
+ total_items=total_items,
+ processed_items=processed_items,
+ failed_items=failed_items,
+ sources_processed=sources_processed,
+ notes_processed=notes_processed,
+ insights_processed=insights_processed,
+ processing_time=processing_time,
+ )
+
+ except Exception as e:
+ processing_time = time.time() - start_time
+ logger.error(f"Rebuild embeddings failed: {e}")
+ logger.exception(e)
+
+ return RebuildEmbeddingsOutput(
+ success=False,
+ total_items=0,
+ processed_items=0,
+ failed_items=0,
+ processing_time=processing_time,
+ error_message=str(e),
+ )
diff --git a/commands/example_commands.py b/commands/example_commands.py
index 5d8eafa..c1439e6 100644
--- a/commands/example_commands.py
+++ b/commands/example_commands.py
@@ -1,13 +1,11 @@
-from surreal_commands import command
-from pydantic import BaseModel
-from typing import Optional, List
-from loguru import logger
import asyncio
import time
+from typing import List, Optional
+
+from loguru import logger
+from pydantic import BaseModel
+from surreal_commands import command
-# Add debugging to see if this module is being imported
-logger.info("=== IMPORTING example_commands.py ===")
-logger.info("Registering commands...")
class TextProcessingInput(BaseModel):
text: str
@@ -134,16 +132,4 @@ async def analyze_data_command(input_data: DataAnalysisInput) -> DataAnalysisOut
count=0,
processing_time=processing_time,
error_message=str(e)
- )
-
-# Add debugging to confirm commands are registered
-logger.info("✅ Commands registered: process_text and analyze_data")
-logger.info("=== FINISHED IMPORTING example_commands.py ===")
-
-# Let's also verify what the registry contains
-try:
- from surreal_commands import registry
- commands = registry.list_commands()
- logger.info(f"Registry after import: {commands}")
-except Exception as e:
- logger.error(f"Error checking registry: {e}")
\ No newline at end of file
+ )
\ No newline at end of file
diff --git a/commands/podcast_commands.py b/commands/podcast_commands.py
index adf0c7e..486ff25 100644
--- a/commands/podcast_commands.py
+++ b/commands/podcast_commands.py
@@ -17,11 +17,6 @@ except ImportError as e:
raise ValueError("podcast_creator library not available")
-# Add debugging to see if this module is being imported
-logger.info("=== IMPORTING podcast_commands.py ===")
-logger.info("Registering podcast commands...")
-
-
def full_model_dump(model):
if isinstance(model, BaseModel):
return model.model_dump()
@@ -179,17 +174,3 @@ async def generate_podcast_command(
return PodcastGenerationOutput(
success=False, processing_time=processing_time, error_message=str(e)
)
-
-
-# Add debugging to confirm commands are registered
-logger.info("✅ Podcast commands registered: generate_podcast")
-logger.info("=== FINISHED IMPORTING podcast_commands.py ===")
-
-# Let's also verify what the registry contains
-try:
- from surreal_commands import registry
-
- commands = registry.list_commands()
- logger.info(f"Registry after podcast import: {commands}")
-except Exception as e:
- logger.error(f"Error checking registry: {e}")
diff --git a/commands/source_commands.py b/commands/source_commands.py
new file mode 100644
index 0000000..0f862b4
--- /dev/null
+++ b/commands/source_commands.py
@@ -0,0 +1,137 @@
+import time
+from typing import Any, Dict, List, Optional
+
+from loguru import logger
+from pydantic import BaseModel
+from surreal_commands import CommandInput, CommandOutput, command
+
+from open_notebook.database.repository import ensure_record_id
+from open_notebook.domain.notebook import Source
+from open_notebook.domain.transformation import Transformation
+
+try:
+ from open_notebook.graphs.source import source_graph
+except ImportError as e:
+ logger.error(f"Failed to import source_graph: {e}")
+ raise ValueError("source_graph not available")
+
+
+def full_model_dump(model):
+ if isinstance(model, BaseModel):
+ return model.model_dump()
+ elif isinstance(model, dict):
+ return {k: full_model_dump(v) for k, v in model.items()}
+ elif isinstance(model, list):
+ return [full_model_dump(item) for item in model]
+ else:
+ return model
+
+
+class SourceProcessingInput(CommandInput):
+ source_id: str
+ content_state: Dict[str, Any]
+ notebook_ids: List[str]
+ transformations: List[str]
+ embed: bool
+
+
+class SourceProcessingOutput(CommandOutput):
+ success: bool
+ source_id: str
+ embedded_chunks: int = 0
+ insights_created: int = 0
+ processing_time: float
+ error_message: Optional[str] = None
+
+
+@command("process_source", app="open_notebook")
+async def process_source_command(
+ input_data: SourceProcessingInput,
+) -> SourceProcessingOutput:
+ """
+ Process source content using the source_graph workflow
+ """
+ start_time = time.time()
+
+ try:
+ logger.info(f"Starting source processing for source: {input_data.source_id}")
+ logger.info(f"Notebook IDs: {input_data.notebook_ids}")
+ logger.info(f"Transformations: {input_data.transformations}")
+ logger.info(f"Embed: {input_data.embed}")
+
+ # 1. Load transformation objects from IDs
+ transformations = []
+ for trans_id in input_data.transformations:
+ logger.info(f"Loading transformation: {trans_id}")
+ transformation = await Transformation.get(trans_id)
+ if not transformation:
+ raise ValueError(f"Transformation '{trans_id}' not found")
+ transformations.append(transformation)
+
+ logger.info(f"Loaded {len(transformations)} transformations")
+
+ # 2. Get existing source record to update its command field
+ source = await Source.get(input_data.source_id)
+ if not source:
+ raise ValueError(f"Source '{input_data.source_id}' not found")
+
+ # Update source with command reference
+ source.command = (
+ ensure_record_id(input_data.execution_context.command_id)
+ if input_data.execution_context
+ else None
+ )
+ await source.save()
+
+ logger.info(f"Updated source {source.id} with command reference")
+
+ # 3. Process source with all notebooks
+ logger.info(f"Processing source with {len(input_data.notebook_ids)} notebooks")
+
+ # Execute source_graph with all notebooks
+ result = await source_graph.ainvoke(
+ { # type: ignore[arg-type]
+ "content_state": input_data.content_state,
+ "notebook_ids": input_data.notebook_ids, # Use notebook_ids (plural) as expected by SourceState
+ "apply_transformations": transformations,
+ "embed": input_data.embed,
+ "source_id": input_data.source_id, # Add the source_id to the state
+ }
+ )
+
+ processed_source = result["source"]
+
+ # 4. Gather processing results (notebook associations handled by source_graph)
+ embedded_chunks = (
+ await processed_source.get_embedded_chunks() if input_data.embed else 0
+ )
+ insights_list = await processed_source.get_insights()
+ insights_created = len(insights_list)
+
+ processing_time = time.time() - start_time
+ logger.info(
+ f"Successfully processed source: {processed_source.id} in {processing_time:.2f}s"
+ )
+ logger.info(
+ f"Created {insights_created} insights and {embedded_chunks} embedded chunks"
+ )
+
+ return SourceProcessingOutput(
+ success=True,
+ source_id=str(processed_source.id),
+ embedded_chunks=embedded_chunks,
+ insights_created=insights_created,
+ processing_time=processing_time,
+ )
+
+ except Exception as e:
+ processing_time = time.time() - start_time
+ logger.error(f"Source processing failed: {e}")
+ logger.exception(e)
+
+ return SourceProcessingOutput(
+ success=False,
+ source_id=input_data.source_id,
+ processing_time=processing_time,
+ error_message=str(e),
+ )
diff --git a/doc_outline.md b/doc_outline.md
deleted file mode 100644
index 49c7838..0000000
--- a/doc_outline.md
+++ /dev/null
@@ -1,318 +0,0 @@
-# Documentation Restructure Outline
-
-## Overview
-This document proposes a complete restructuring of Open Notebook's documentation to improve user experience, reduce confusion, and create a logical progression from discovery to mastery.
-
-## Current Problems Summary
-- No clear entry point for new users
-- Fragmented setup instructions across multiple files
-- Significant content duplication (models, Docker setup)
-- Missing navigation structure and user journey
-- Language inconsistency (Portuguese META specs vs English docs)
-- Critical gaps (architecture, API docs, troubleshooting)
-
-## Proposed File Structure
-
-### Root Level Files
-- **README.md** - Project overview, quick links, and 5-minute quick start
-- **CONTRIBUTING.md** - How to contribute (keep existing, minor updates)
-- **LICENSE** - Keep as is
-- **CHANGELOG.md** - Version history and release notes (new)
-
-### /docs/ Folder Structure
-
-#### `/docs/getting-started/`
-**Purpose**: Onboard new users from discovery to first success
-
-- **introduction.md**
- - What is Open Notebook?
- - Key features and benefits
- - Comparison with Google Notebook LM
- - Use cases and target audience
- - System requirements
-
-- **quick-start.md**
- - 5-minute setup for immediate trial
- - Single Docker command approach
- - Basic example workflow
- - Next steps navigation
-
-- **installation.md**
- - Complete installation guide
- - System dependencies
- - Environment setup
- - Configuration options
- - Verification steps
-
-- **first-notebook.md**
- - Creating your first notebook
- - Adding sources (link, file, text)
- - Generating your first AI note
- - Basic chat interaction
- - Understanding the interface
-
-#### `/docs/user-guide/`
-**Purpose**: Comprehensive feature usage guide
-
-- **interface-overview.md**
- - Three-column layout explanation
- - Navigation basics
- - Settings and preferences
- - Keyboard shortcuts
-
-- **notebooks.md**
- - Creating and managing notebooks
- - Organization strategies
- - Switching between notebooks
- - Notebook settings
-
-- **sources.md**
- - Supported file types and formats
- - Adding sources (links, files, text, YouTube)
- - Source management and organization
- - Metadata and tagging
-
-- **notes.md**
- - Manual note creation
- - AI-assisted note generation
- - Note templates and formatting
- - Linking and cross-referencing
-
-- **chat.md**
- - Chat interface basics
- - Context configuration
- - Multiple chat sessions
- - Chat history and management
-
-- **search.md**
- - Full-text search capabilities
- - Vector search functionality
- - Search filters and operators
- - Advanced search techniques
-
-#### `/docs/features/`
-**Purpose**: Deep dives into specific capabilities
-
-- **ai-models.md**
- - Supported AI providers and models
- - Model selection guide
- - Performance and cost considerations
- - Provider-specific setup
- - Model switching and management
-
-- **transformations.md**
- - What are transformations?
- - Built-in transformation types
- - Custom transformation creation
- - Batch processing
- - Transformation management
-
-- **podcasts.md**
- - Podcast generation overview
- - Episode profiles and speakers
- - Audio quality settings
- - Background processing
- - Sharing and export options
-
-- **citations.md**
- - Citation system overview
- - Asking questions with citations
- - Citation formatting
- - Source attribution
-
-- **context-management.md**
- - Understanding context levels
- - Context configuration strategies
- - Privacy and data control
- - Performance optimization
-
-#### `/docs/deployment/`
-**Purpose**: Installation and hosting options
-
-- **docker.md**
- - Docker setup (multi-container)
- - Environment configuration
- - Volume management
- - Network setup
- - Troubleshooting
-
-- **single-container.md**
- - Single-container deployment
- - PikaPods and cloud platforms
- - Environment variables
- - Data persistence
- - Scaling considerations
-
-- **development.md**
- - Running from source
- - Development environment setup
- - Database management
- - Service architecture
- - Hot reloading
-
-- **security.md**
- - Password protection setup
- - API authentication
- - SSL/TLS configuration
- - Privacy considerations
- - Data backup strategies
-
-#### `/docs/development/`
-**Purpose**: Technical documentation for developers
-
-- **architecture.md**
- - System architecture overview
- - Component relationships
- - Database schema
- - Service communication
- - Technology stack rationale
-
-- **api-reference.md**
- - REST API documentation
- - Authentication methods
- - Endpoint descriptions
- - Request/response examples
- - Error handling
-
-- **contributing.md**
- - Development workflow
- - Code standards
- - Testing guidelines
- - Pull request process
- - Issue reporting
-
-- **plugins.md**
- - Extension system (future)
- - Plugin architecture
- - Development guidelines
- - Distribution process
-
-#### `/docs/troubleshooting/`
-**Purpose**: Problem resolution and support
-
-- **common-issues.md**
- - Installation problems
- - Runtime errors
- - Performance issues
- - Configuration problems
- - Platform-specific issues
-
-- **faq.md**
- - Frequently asked questions
- - Best practices
- - Usage tips
- - Limitations and workarounds
-
-- **debugging.md**
- - Log analysis
- - Error diagnosis
- - Performance profiling
- - Support information gathering
-
-#### `/docs/migration/`
-**Purpose**: Version updates and data migration
-
-- **upgrade-guide.md**
- - Version upgrade procedures
- - Breaking changes
- - Migration scripts
- - Rollback procedures
-
-- **backup-restore.md**
- - Data backup strategies
- - Restore procedures
- - Export/import functionality
- - Cloud backup options
-
-## Content Consolidation Strategy
-
-### Files to Merge/Eliminate
-- **setup_guide/README.md** → Merge into `/docs/getting-started/quick-start.md`
-- **setup_guide/DOCKER_SETUP_ADVANCED.md** → Merge into `/docs/deployment/docker.md`
-- **docs/single-container-deployment.md** → Move to `/docs/deployment/single-container.md`
-- **docs/models.md** + **docs/model-providers.md** → Consolidate into `/docs/features/ai-models.md`
-- **docs/SETUP.md** → Delete (referenced but doesn't exist)
-
-### Content to Extract from README.md
-- **Provider Support Matrix** → Move to `/docs/features/ai-models.md`
-- **Installation Instructions** → Move to `/docs/getting-started/installation.md`
-- **Docker Setup** → Move to `/docs/deployment/docker.md`
-- **Feature List** → Move to `/docs/getting-started/introduction.md`
-
-### New Content to Create
-- **Architecture diagrams** for `/docs/development/architecture.md`
-- **API documentation** for `/docs/development/api-reference.md`
-- **Troubleshooting guide** for `/docs/troubleshooting/common-issues.md`
-- **Migration guides** for version updates
-
-## Navigation Structure
-
-### Primary Navigation
-Each major section should have an index file with:
-- Section overview
-- Links to all files in section
-- Recommended reading order
-- Next steps navigation
-
-### Cross-References
-- Strategic linking between related topics
-- "See also" sections
-- Breadcrumb navigation
-- Back-to-top links
-
-### Search and Discovery
-- Comprehensive table of contents
-- Glossary of terms
-- Tag-based organization
-- Visual flowcharts for complex processes
-
-## Implementation Priority
-
-### Phase 1: Core User Journey
-1. `/docs/getting-started/` complete section
-2. Updated README.md with clear overview
-3. `/docs/user-guide/` basic files
-
-### Phase 2: Feature Documentation
-1. `/docs/features/` complete section
-2. `/docs/deployment/` consolidation
-3. Content deduplication
-
-### Phase 3: Technical Documentation
-1. `/docs/development/` complete section
-2. `/docs/troubleshooting/` complete section
-3. `/docs/migration/` creation
-
-### Phase 4: Polish and Optimization
-1. Navigation improvements
-2. Cross-reference optimization
-3. Visual enhancements
-4. User testing and feedback
-
-## Success Metrics
-
-### User Experience
-- Time to first successful setup
-- User retention after initial install
-- Support ticket reduction
-- Community contribution increase
-
-### Documentation Quality
-- Reduced duplication
-- Improved search findability
-- Better mobile experience
-- Consistent tone and style
-
-## Notes for Implementation
-
-- Maintain backward compatibility with existing links where possible
-- Create redirects for moved content
-- Update all internal references
-- Consider automation for maintenance
-- Plan for internationalization (Portuguese support)
-- Include screenshot updates throughout
-- Test documentation with new users
-
----
-
-This outline provides a comprehensive restructuring plan that addresses the current documentation problems while creating a logical, user-friendly progression from discovery to mastery of Open Notebook.
\ No newline at end of file
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
new file mode 100644
index 0000000..aa562a9
--- /dev/null
+++ b/docker-compose.dev.yml
@@ -0,0 +1,25 @@
+services:
+ surrealdb:
+ image: surrealdb/surrealdb:v2
+ volumes:
+ - ./surreal_data:/mydata
+ environment:
+ - SURREAL_EXPERIMENTAL_GRAPHQL=true
+ command: start --log info --user root --pass root rocksdb:/mydata/mydatabase.db
+ pull_policy: always
+ user: root
+ restart: always
+ open_notebook:
+ build:
+ context: .
+ dockerfile: Dockerfile
+ ports:
+ - "8502:8502"
+ - "5055:5055"
+ env_file:
+ - ./docker.env
+ depends_on:
+ - surrealdb
+ volumes:
+ - ./notebook_data:/app/data
+ restart: always
diff --git a/docker-compose.single.yml b/docker-compose.single.yml
index 662ed02..ad641bd 100644
--- a/docker-compose.single.yml
+++ b/docker-compose.single.yml
@@ -1,11 +1,11 @@
services:
open_notebook_single:
- image: lfnovo/open_notebook:latest-single
+ # image: lfnovo/open_notebook:v1-latest-single
build:
context: .
dockerfile: Dockerfile.single
ports:
- - "8502:8502" # Streamlit UI
+ - "8502:8502" # Next.js Frontend
- "5055:5055" # REST API
env_file:
- ./docker.env
@@ -13,8 +13,8 @@ services:
- ./notebook_data:/app/data # Application data
- ./surreal_single_data:/mydata # SurrealDB data
restart: always
- # Single container includes all services: SurrealDB, API, Worker, and Streamlit
- # Access:
- # - Streamlit UI: http://localhost:8502
+ # Single container includes all services: SurrealDB, API, Worker, and Next.js Frontend
+ # Access:
+ # - Next.js UI: http://localhost:8502
# - REST API: http://localhost:5055
# - API Documentation: http://localhost:5055/docs
\ No newline at end of file
diff --git a/docker-compose.yml b/docker-compose.yml
index 9cf2fda..7409876 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -10,14 +10,14 @@ services:
user: root
restart: always
open_notebook:
- image: lfnovo/open_notebook:latest
+ image: lfnovo/open_notebook:v1-latest
ports:
- "8502:8502"
+ - "5055:5055"
env_file:
- ./docker.env
depends_on:
- surrealdb
- pull_policy: always
volumes:
- ./notebook_data:/app/data
restart: always
diff --git a/docs/assets/asset_list.png b/docs/assets/asset_list.png
index f2ac3dc0dc57c3e2a453769c2d3d0d29762951c7..87327a8022ecbfc8a9544ac8c607f501f084f5db 100644
GIT binary patch
literal 241289
zcmd42bzD^6`UVP;iiia$Er>KIT|){;cehB#&>aE-Dy6g_-QCTAfP#Q9#4yA#gmm{1
z!@%8s&pE$yK3BbeUp}9K&Focst+n6xd7t-L@6=S}h;LBbz{0{JmVfbF0}BhE919D_
znGhfNhCN*5GZxm3I$LRJHF;@iIyG0Ym92v%7S@Y*iOB?7np0%qP{{hDM-o!Fd2eR(
z8nHB_NFW00*X3WUN!`S~-^wi*Vj=k)H{s61d}cvb8QDA4W|Fv+DC~##VJ=l>(#ytX
zcRu@ULK@eCdpM!PmOKA=_v%VFes!Ox8B3y*kZF&*d7Us*qQ|=8#b-RM
z4d%NOLw$FZ_c`9!zRZ(&BUd138{VPh(#4y*T;SFvjl2WL#!~(1{8b+7!;g=G3~6a^
z);<)#xycW3rQS?D+WMT4(J9@8V~+D!?#*6;7|()8dR~X^lNWqKpC7Y8w=AdkXhM|K
zDXhO;`A}UUM#mo4+OIK$Ca|Nf%ZO47)rxEIT;#DU5#CI-#0CoorN%Rg-KqUC=pLKC
z{G;wEiv3WKCQu{2PU1>m<-WJP8?OG=p8krpZ}?!1AcIzmQ_f^M-vHIV5pf)h&{p&l
zG-+e?Nrc>>YXcp(nt`ER%A8S5D=%l#8-^%p$T6_NZcK1EDO&@{8`lXSQH7!*bfbU-0_qsNYK7(!+y?`l}H(gc@hv
z)C$4L!l%OSl^_zt8w!b>XI;Z8JGB72Zko6n{4KI$?
z(<9*z{q&Rw723V<&G~NI&7rWv`Eq9#BRtyhw1wI|?NfeHtd7^c>R61SIk~**Pne(d
zQ2oa`4r-)H8?mcu5B((z2*W#MTIT$sso(_z*nf
z@Cr$RTxc#hhhW5F4g7#xmeeB3Z+DP)B0`rP{9;coc#1}jlxRNnhiq3Sncf8+_zHxkGCq=m+Y@YH|GJ$F1Ui#aG
zulIlYzGSHm_6|P%aH=CKE34RJ^5nTJU9C!i)=JS2#!r%;R3GET-RVoeFBkWrA@0^k
z;TI<_zaW6o=@+j1zItkF@dP{Ti0amQyvNQ}s2=HG?$*uT0em?;GEUta%fs
z$dXM}XjIsi!0>&&uX(^_3I08CAU$C-nKHSs-|#y@U*Q1wV`Y2?yLirP&0)QjA#g_XKTk8B+bZj^uUd;^!J)YMX|(}~hDs?e+OHgI_z
zle0N8b;NtlfBx*8`h3;L?ojXW^pGEFpW~tzqnr{
z!7zcgI7>T9OG?XC3sr2S<|CBJKj$Lm71()UakyLBRU%uGUBaeKQsh}0Q(9c=Q~C~u
zDoWS%D^}6<(pFXH(gEwzj%V2k+7t4==NqySwPxnli{h1KSroNU($pT4)68i$t(9|=
zQZ!|gvJchCg)=zj@N0^wx9YWO`YFjtGf7w{sz8PUj6*mz1@yfdosxG5KmibDh_Ww-
zFT=UQxdX-sQ94Mg$Euq8Cs;{2CvCUGZ;h@IAe%$VOdU`Sy41u0k5RX6kVim7kIH+qVVp#1Mx?PkW
zW87vOW-K1xbJNPn+@z*=vURGmUflAkuM_@gU?HX3H0DW6ag0X{1JN=00l5wrmFuYM
zPr+?33TG?9UXiOU(fOS9E|+3gpPwb${cfPO^C-*j3g*;S5LX_rd#!xU0nT2|rrWa%
z!3TRN=DoFjt-Yu{_Cx=z{N=ESN)*?k#m4&zgEw;9kxAoK)4L&PU6SRHZqj
zbi+Ku)Vq_y$HO2bUPNEQ`Lly?jUeKHqIeGKlcGU6eu;T>&2%
zIpssYr=I>FS96Rb!xtTX9xU##!%f@#_m>%26-Sdkn<(h_>-&n*+@mEmCBF*0_qF(E
z6jRkb*1J7TJ2S``a#3BAXW)e0uUtRH72Ors6{zH06{bJ&rf$U<#a(F3b28s+Q`wg<
zRyzAYk>Jbh!Og*1{){@4CNof@h)#(#1F8nq%NSweCR>igPZDBB)H^6`F^Q#Ix8xjh
z7|eMTtnB#qsZCZP6;>dZfQ@r|${zyv9laZ$*;yE77|gb`Wz)6=6+x2+
z$yZOnt}|L^4Civ4v7oea{@tmI^4Ge~`NM5hd3yt7J4tNf`WIT~V@vR_}lpH_)BnNHxP{@x)HGmB0uJt|SSWk&Lf*4uzuA9+l4
zm^>ZINaDu0?1$fLVPZ`!QpOupJAA)VowSq`3{Qc#$G0gW)`Y7P`BOzYad8ZsBANDkr5*nH`c-n`_#
z4zr9HV5)ptS!YxanVxYRL%I*H7-+^isvdf@Zk9t(5XS%_^fb6>M$m6;apjxgmZ2_~
zV0NeVp3ACJkkQo_<5}cKKid+mY!s`h7MXX(A5
z=F4#q8HoSF;KKJ{FsZR0OaiG5oDiZBy*v##N53P%qfHct`@am_IHx*3n|eM(K9f9J
z8E%SbKMU{^{3IsC^7F0GSUJ{>ht4nQvB0;mC1S8&PZMFej3@a~0LJQA>d9LvD`T+&*MwNu5w=*kz!f&|p#VMr
z+KmgxA^`s01wPNSuKnjOK6w_-f39(ye-)I}l$MtV{%TsdT3R}}*?`@1sy&|oO-ACAED+ybG9XZWkfnQp3dOQA81xwUh7`Sw_bT_B-c64xZ6ZQtt|5HL3xc>Du7d_oS
zMcnN{^m@u_bkbl~OFDker<_me#ct5i(TTdgvJ%z+9PhuX1OI{OZQR|Rg}J!Ayu3KQ
zcsRkX)?CkogoL=Ba&vKWa{whc+YU}K73wEOW)vozV
zu!lQ{p8i)y|M&B+jx4=x|J#$3+rJ+RctEaScetK$KIQtqwt=dmzrGb#v-P%g(0^|0
z2uu&qhnS$?Q_+9Q|90oUJ^oKkz5mvH#>Xf0zg7R|*8jVzwwtA^G}sa7(_QSpOY`r_
z|9kV_6-BvzJ^KGl#lPnHpKpPQ7P}$J^?!>dcEcmzCo0OJ9`59QXXT
zjKt&ar~9Ri;RtsDkdLnvv?HT%eJ5?V(h0g;?6#?;p`)YIKAuzKFm=No>Dy}H-t(FD
z8X?_ltUvxDW&2=Mon~SG*8+reAJ(qnlHY}d3yI_2mcYXP<1bdd*Yi?%p*2E(y!%hJ
zyT(9~oQbEa5q};DFailYqT5DAW@`NZA4PJ2A}@|8RxbWJmfJ;U*dauuH~UxqUgU*4
zP~@FS_h;Ha&dNWtLCP-SoE;)LMfLY0?>Hr#AC1_j97_LXUT#Zdh2Rl#YCUC6`+E_(
z5unIpUt+hwzpsK;6;@~14SY+tzZZGj0u*^(X+A{q=QRRmBlrdxU6Wd8mcy;T7kN%j
z*YshH@5kArzib&oI<-zUAW6Zfh(
z`mo%br}pTm>sIXgUlvGW;aYREPeULEf8fW=E5OXkGI%WLET`x8Y{Fn#1K(dXo*!@O
zD=3`ZT~za0F8|1}sq2n(m~vgnWaVi3P(M_r!cc1}Cf~qiBHutv9Pbd&?T^UU&7k(Q
z$w)jDX1}=kDfG|p)pe#Ux~_0yO2_dMzV_-?C*;kEz4Vufm?O`wPBIrMZfIX$eOVbh
z8gpU^&y!~;)va_Y9xQI9|1cm}nzK>B+gh10AL7f-YD*4-6a@f7qC)G@Qx#
zQHhH^9;%^PZ1SEBXjO_-q=VN(GYQlStlTQT^N<0!p%23E@#{%V*Jm{LOdP0o$ru*Z8E(IzuwIAt$qkKI?Wpcia+w-&YdH
zcZTD4DcbZ|d|ni4f;ME1)jrVMN`Hiq7q#|RR>r}V^<&vNEjX86r4CYlbg1o6t!aZ>
ziBNyGrWRf@w3BV89v7KNz5EP5oM4*buV9jf;`<~eo9i^jG)F>zU>vI+C({o`L*kyw
z;C%k^S!m4aM~eBJKTr4SBZM1tr$D#S3CRvZ%9z;s_-qtsSD+J0{dGO3i%Y4!W?aS%
zmJXpERZ#IOQTHfaV-HK_%9&6SvHPj*SC?o+FagbcM=a&ZyW{n-iK~xDzg>u9*PH$9
z<bG?KPOGlO~~Mi6mq_f$Z9fVjB|9$>;t|1HoX
zR7_w!!x~Max%Nzr
z%^rG|CrCq~X4
zn@J*Rd%Lbi^%Bq{l>(9W|NGG&;wRGhxEUxUyZW>UMj2l%Z2ql;lK&>QA?u!8sdGdVVsd
zYAtl)O@#0C>InWEHrj0A?Bkxv+Kx6B&Wm<>z1qXcsL{Te|6x-sjfi)t*`DLuvEBC{
z{PsN6q;UU0`UzraJgcBkuyu$KRBMRNPb0QPmQMpFLz^bZOsXm
zGN`jz5HEd%Y2CG&rt#Zyil%HWQfGYuUQpaQM*{&vFHN)#uUJkFx6P`}nbR
z-aCn1|8AMxIhDWeV*3zFOTv+V?gU%k$$Uy(S&RjGyOGPC&zMMI5msR=(h*e=M(3FD
z!~tll0&TjL36_UGy6|@2cBmDSiLZ!nc3VBgdQo*HMdE&VGdj82d4b*#;#*%2o*N4B
zWpx5=+b?uw8mqKN@0@SgMN{yxJCUw1{BCC$$sP!~<^c<746iipxLcWvx7&_(66dtZ
z6bn$OHz^A{JUH-g&75|Haq8ZYDV`j^Ea1=4TW;NLuZ5-SPXSVdtY26TaO|Gs
zC)M~hYE&BD(|L;&)*oWfmFSEveTitHx$eU`nT4I2)^bwMc^puZ_TlpizKP2rALJv^
z6Xx&EU!DQ5Lbhhmd3%h&y|2*%(Zc+km@|@`D@O1<05!$*v$fsGZ=3H9@mD)D;{J!$
zpDph(sT`U5p|)&k1CAu04s-4>OAw0eR4F3+sDe@1fg
zVV;fC^5C)gqC|%)npY^28+{Stsgwl1`_d$;Z;Mw^?dT}1WlC~y~4+)?WZr_AMXS>*o=IO
zB>xn%O?%bEeV^H8G_N?_`=z+oO_B&vTHm!tl-x@h!@lQL?&}(xu;SWG)FpbiIYB$(
z%}O%LUz#oHXE#Hv=tdNN1YEDxv!3KD?mc*!zJSOsiQYaZ3{APFXY#)-ZrtOh)04hlz2y
zNXAKqIz7wBA__52EtUv_w;qiumIY$YvzJ3j&I4>9UUSqte!IA1fcukcJsb)$oNHh1
za3Me}QU8Xv?J8eScWGt#?M!Jr;+|e}+u%s68+{+v|
zgl0$3toZ57ICm326F7t$f)V#rMHi-5ho07tkHc!#v*kFv;0`C}BXI)a57LegN(*$o
zM_hdw9b2RGw`RsiD_iLOJ&VOFk-NihJ>n?%t6-Ru`lHQ?qu{8wpJe=}j(!j&`Sc>;
zuaGr&@t9oOPe)yug3Q--uOM&Bbu#r`u5+#(anDFEJZ#86ksJyiWI_tgxf-+PDG4~c
zL6XiI#Ky{kiDY@fabO%Ha0K>vZ-J3qPKD;eayxpb(K<8W$txZG^vFonuRJJBQy6Hd`)ax@GU>_+Qv~kOQ`y7
zRm*M9xIssjQkH>9t3R3b<63DuP}1UhPsG_Yd>p4oJ{^Vc5O{gEYP{%KfH~<GwVN<1U<{yluy%O@SkhPY^|h$SzQN~K@l@EMw;L>k=-_)m*SCR9Ild8n@S)v
z&ZhZ`w>}&4E?w6H9FowBG{(Go^}Z|2g#4~c&K-eOQNB7y>tj&7EGvR|solesK{uSN
zO!L+B6LK#ce^K6al0P^NA6%GD{g{gJK-9mq1^Xn%sm(
zo&y!Jj#ms=s`+_wp&u7;=Kh{?OW%TR$uRcbVcF`ePlcqQZ*(fg72TwFM@A0obB-I3%K9j-RJEh+kdQia+5S&h&Q;LjR2H
z%T5rqV;0;0swov5R$<(FUidw6!{&5vq5YA*BtD-Yt;^Siz}GK3s;h!v-eqR*6%Y-M
z0o|_VEdE_OY$Lrv7mKX4OKRX4+MtVP^p>Ngm6uRnwF7K|Eiihkw$a$Nj}ZmaRF!ei
z9n&&sblRR{`{|Df?>~c%>zY#D!5SUAl%w`YU~$e7rW8G`kUCMjvsz_0&N>mlY>*jP
zYS?J(*q^|~;7f1hg*+BZxev?LU_IXUd&yjt^-RY1Xl-G89PzyK7U{8n^Kz1*@p&cs
z!Yf}=c>im>bTD3iq40WMJmC0IPE>K#s6M7ej4J>HL#Tm$%1P)Fay2CWhh3nLi<-9Q
z%H+Pt@T2E3y5Go=|8UN0E+EpdZ1POOQOtFTxplpO(s8!!tUyBA-8tK^5xG?}aJe$A
zSulm(jdj`8^iM-blYkr;WTTun2M@P70Yekv_rkmJZNO|Gnp(_kBv1ZcV{Ff=~Lq%8&
zYRBZK&rEPTb&SVRz+xe>3ZE%H6^X@&pr}6>VVU=ovNXVn56ZrD37?2$eFwBHo~jU1QK(sIx81c<_4X|Ok4yKPG*!}HIaHm
zYkd`Bw$z%aFrE=4b2Bz5IKiOR<)|MDKgr+H;Oxk1!=MaN#}lRmrp7nQgQnkHUsSe;
z^gmx`;y3$Zo9)vbj^VmrSm;IE9}Y$8c`}epj${X)h*X7i*mT?r+L2=IAfzr9RCcf&u4)wTIDI<%n0Xg?SEO?0q>XH^?^aFe;&+~V`o
zVZCukoF>+XoDv^Bf80@EwAh3SLTkh+R}4<$MN
zIIwp{daRFgECwL=J86D6-W?
zZgMy`2O<3r2T2={Ys4&ARf{p{NWrZ1tq;<->H(^1+P)G(14;4Ngb#)j`+Xz%0tjao
zLW#*uyt8Tb&dX|t+@mqwTY&W^wv->i=ha!Uhy|^T(b|4i&6fh+8xE3Cd=YiZ^wAvq
zt${3NY577Gk#m|m{i)d|#JU$mEst#B+v%v%0PU6
zMgjCVdNnIf#G)Vw+BSf9BSx%igE#=T@tN<-Mh7Z~u{F3s03-&3#-8@#-GNTPnUKq}cH)i#GDn
zBFd0`vtED$EHAmo{E{~DB7bz0K7Yck^YKK$qjJTJa*xh89-K0x1_CK=?S9}*-)E2Q
zMkJcMBRUo-<>g{&wlj7z*2d=84Oj5@CyiiD;$~m41a`-6?ApPK$8L#DKipE>jiwav
z3=;Ju%&+_O%pyFqoa!9lGe|gmrz~|RuP}4D*}LaYt%p7jpPU^A9ef<5(^jzwM^{~4
z61)j2uY!&kcQhb94X?fid^j;(_f;@pJ@8j-f$9x}2~67ul1z-VwQK=2z%#HYsn4wu
zVhjPTzuH&xH;BNq<>YK111@3jf{aHLW+E+bDhnys1?C*CJJ?R`NoF5oyy>tu_})dakZIwA^9ec)ST6L
zLCkYO;$wwE;^%9*A@c3yVcfEB$m_}RWLUbi6VmB*)RW#Bf-!~Q+%m()3e_O@LB5Go
z8S5X+Hkm#vpRU4txV3LEB~I0@J{QH^WP<+;IuJOalDAu!E}iLK6mr_LOe7{Zy`#~G
zJ{Ay=?M~u{naS?_*bPv9Q)XU?mMjTf_l>P@D_C`M5S_aON7OYJKbK6<*2M(tR_YG5%UmY7RA@o~&xfsJtyH|q0#Yx_S(glL3&CaWJlR|B4!yoGcC62m9EdsA9gE)_
zkqOFLRM*gKyci*F$oxz-#r8PGWX4Y>1MGrkwY!9hb48_;Cm4({!g)C~Kd#;mlpbry
z#lcT4$k`OQXt6;wD0fD^yo@$cjf?;@!8a&q9}9oKV=q+j2ZlfYPSa*ZS8pTxpyzu7
z9SDw=iO(|W)eTeBVr*~A{u955CycEQT#_RxiL4JF$~0Ey=Q+qLWW}IQylkQ1
z!$omkR~%C7<{6BnfqlN3hV5vcEpt;9fa=ps2Ga!C>}uerId%zT)HRgnhI$X=4dRWoW5%Tq!6-vWJ0={wb`hc%!%qI5kHl_-jUQ5
z%W+-{AFC(BeJnY$uxremh5P*J`(_pSFYrz(b)Cw1G_lfHM&*
z?ZYOZGNXuonXW#iKev&UaJr#B0!Hed#J-x?IGM@|+UWFs^O!%io$E
z#G~-6X_4L`e74MGY$eZO_@`zyK$vJf4YOQZ?uju7%Mi{q(
z@%{V9PHF*>bbI0-2_
zwOVg%Ye?jZ+MvT4iW87F4>Y?q##c;ykYwmoU)gavU7&hZ&^6W+;c1jXQen(j@h#f)
zBj_Mtz;~H%z_bs2GcGOBZ}OI4GSTP
z?2DNK`saLCgdBw=@)Zz)^115``*0B@+e8gdLQjIE!;(9}*shkBX
z^V^TU--DCdqzMqtm*0|H=FAj5J!q)Szz^`JgXhm1c-KvJI-PbrM
zlJpYS4O5SCzB{2PR^3bBc{VhO@Q8L=HY+-N-?pI@#CczR2VIiHke&XFWI>%Ss=i6$
z*a-DiwHda_&r=^klj>eDzzn6vV72tjl7ajIK|nUaka(Kgd(W8jrSDtBilvwVMVYSg
z9N(Io8bBQWWT@!$g6KvZZVzXN?s}3@O$3}EcL^rdPwM#W;Vt5*P{)hY$j_f&;+Q;7Z`wMOV-xU^L&SDsEOi-TMT+woeyA!p
zC?*mabcNI4V&U&fBfFKr_51ty#VS}nUaa&s5grLlHiu5qT?mK?FPh?k=G>r9
zPYfzeAX@;A_awX#n4fdR4sHk@;03s{(?JAi$?qj(Sd4f0EZL?he;0sshn+_
zfb*kzVAyl91Lpv(rZezGDn6Rcp8-O#aV$i`7-ZbpR6&VG!4*gg`5LKGy(R)OOF&cc8<YiDMxd*+{L1UN1?Ip3zFH_Z)JCAp;`J6vMKs
zxo>wnt^y*qtEA<~kBVtl{Z5df+oOtCD5SFbkq@j
ziBYWwmh60%*bPbBLvHFnGG}BB&Dd_4bF8wj0+Ia(y@e&In(rgN&jhyN@27`|
z1!mM>TXzRF{i02(`|rq6nFn1q=c`}DiYI~V^?ZF>?EQSE#p%jAvl!4Fkh!JYD
zF_B5?8L(*V{x&Vw6Fre);-eVY_^LUK7?$obR$ri7^eWE~LVmd@Yd2Icnk?2w@_lQR
zHtfBo*+QiQ$j(#CmF4i5pe6|3q%V45h>FgSEST@`8NK)u(2Xd0}
z*4F+JwU4&*h5M*
z)g>e=Js=G`cy|LoKWv;)v)Oy(-|QWL3J!^LngQL;n6X1kC9Ak@H;;T%jeG|3ZSKEN
zaefptjcoO5?2BibDsP;%8Y@sfj4oF2$jw<>R1w);YI(T5N@03s*`H5l45VA`2n*u+OG1DMKi+wMlmp`p#;1*{`svGbA6}YWkZ!LmJo|WJaw|Ne0tD#9
zN#QNWp4zTm8_ijCk{TK{9y@Khcs^^2u1|eSKp;>z9zF$Gt9PYIZ;PnIJ&f!7y1=6m
zg3H=#L_}m4TXF%qu-eBmI0MXu?J*Fy)uC~2sDb>D^gx+GeSh?$YFMfO*lGdCD5BET
z7z*Il-P$0Gy&}o$Fd}lTQn%qr!bmdA&mKJeWgInyIqkTdPvx4{ZF^>v#Dar@c!fxa?z}AB$*?5B4XPB5?`_1JbOILaSlM=br{dAP=H}{ZZ0DECABk6*ho?nblDw8Bb-GjB>KxsVHi~Ou*&Nv;
zVlSoIe54k9O@SlN<(bwK)yo-Po#%jnz(_UO2h~YHd)iqaiZu3J&+n9*z4M#OEZBhI
zHm^}(;lzG%$8}Dv%o5$6`{R)cYN`uzqDAPLE#4(C=O2%BA0zI3{Krlhvrr#fJa`<=4#YVa7$91mN84+M?ca321c6h6|
z=QK%%+0RbtR5A=fp(mlXY_T00kN6xWzf4WAG)zc2lfjJX)c1Fn%hPJ8?sSIL6jk5)
zoi_TFtkrra@nKe`*kl8)?5a9c*b=_Oj%aep1)u6LurZ2VtI@@h3;w(^n0A^a_2Cvv
zQF0E7id`B>ny=O1^Q-Z##Ca)wm>$7znYe%8eu)yDO`cA7gD=+_#0lxHX%;*$bx|U-
z*yP9!$n(qhQI8O)qX<%J!Qc&0ktHqO(!bekz$cUQ{CG
z{3uZS9xnbh^gN)beo&^0Ow_5^TMlg@bO_xIt5m-f!X@tqe<%Ab^7v<4WWC1?>CxKG
z_#$<|mQ7DtUflY=yz+fnS{&8G&agP<>YIO=FTdb^C!=x{9p4&OaVv$Eei})AR;AjY
zh*QjiKM(m|Gw}jYX_9u<8ved*{=^HUsuFT$q{
z^rT!a6hryf;RLhZ1zu>Gy)Cl87fEFWdMbcZ^+x}`A^#1+BuK9VvQm0)&&ad?
z(zK6Rl>N_R{ANHz|L!TK^t&Nii^A
z3PmX&8G0wjb?08@$_43_K!_h61fhUvgtU2Rx_l#UAkg
zuD1A>JJHe%hS~WVU3b&lD6iUX7<5cFZZW8}Qa0WVdk8Pqj3^)4?WJ9v&XYHt48e=M
zIAF>Il7j;_U0>e;p~7u|O=+a|Tr=yX@q6`SIfrROuqQV2)j$%)e0_Z#4)GJ%94k!D
zFCYuN{>swwR|@TTJv>3m1`6`3<##lY0kRDT2V0A=;>QEF=yF3axR*94%42Pqziz81
z`V;53F`G02aIkIbHlndUUPX8oaQS2gvMd>5=QeGF86YUU+;DfO5;8aBiKyQj@IUB#
zoW^J0KAGhIB`QhURSK*qOkjROYWS?X|?#t%v1X*
zS3|Fz@+|;&*DeoSRmD&v5VnQsHpCRL6Z8V@U*23gb=8p^g+X1por#Pb|9uD
zu=#D+OaHTt(ub&nfl4&~5f5~0mvXh^;=rC#>;m&Zkz7fm^0j&QY>n+$`+hJX-ZOxs
z6qq}ug^U83#e?8GcxozQXWoj6iuJ(CO#|wzli{2z%&ublM9;Hcs-2|u#ZUrOu`{FZ
zQrHoNnSp(HWGmBvd}AA`iX9H59j1>@^|l*xU1)$vq8Nff*&sY7(*41=2fz3A?L(yR
zFA-jGcEe!zMfSfg8a>U82ErpbF~S^6)MRejcI&Q5;MqNuHA{#uDXwi!BWZw91dVSN
zhtX*yG4**!xoJm!pnDmB-t6L;RQDmws98T%+864>!GB1J{bODpr%EiumNQL%(&;=a
z5(v45YxiAeeKtVgN()UQ{=)@3B3)W8@3my94u|9X7dLjHX1dwj!edjP<>+B3W}~YxgVA&;`#&NT<(yfH0*Mbg`#4ms080|6^v!
z5BF$yZbG2V{?YO5aLWzCyu7?76VlAH!={~<0cps?`%yj%VY=I~;+M{#yA){0fx}Tn
z=m~`3;tl
zsZ;V;m8H8XWDeWixC_XGjw`HRP>UeCowMPhSl)7LhOp{YG+)1tx8R*$Yc6DsN3*acN#_Ljyr9pOg^jwtj
z%G_YL!7LBxri9P{EBF$KsJT}TQk%T@!4=leF=}R#V+m67#bI5M89yhEe2+KI5)>68
z>t23S$eflx)Z*qI4Y0WU`p)$9b(jV_pB-#C)55@@Q}8-)G}^XsDEmtYeC)vua%1<3
z-M;KyR)uGIhb;2y5|7%xtAMy+IJtShp8tWCI1xE#%%oDG3N@e<7@V{%A5aK4zRSKbRKJJ9;Fax#}}n9M06%SChOXWm;s$UnFp$yIPRA3P){Bbl&w
z_vX{|WZmbDaeJhvQGao_u-m}l3zw3jPdwIV90E==-vOZ(<#Z&zhfXYZ^tUXgbv{$#
z*&A4I$U){ER)a0Wc~*lrMo~6Gd=3Pb#~a^|L)I5|?S>~#u>f)*obpvuI*A=1M9YVq
zt-Rg)MlWU`bhv`K|4i9br`+J)Gv(DWz=#3L3O0rU0@DyPiGd`Ji%ZLF?4(4ewS|bx
z8UTrp2=ohhpaIcfq-Is1Z2x$s)c!=YyLDh><434agn6q*sVA>WNf7dwL@9WOqvf$8r7h#WcDbT=&@ORrgw!lkEKsGU@o@DCP|7CgJT@fu3J&
z2h$qA920)OrEEW5LPx50@Di?DW|;e
zxG5(v@9M4IUs!KH(K7$dm-g}S7}~eEQ0BJQ6o7~o@s7(aEiKXRFDP0UqIgtL5atjp
z+13U}|EJQ6Ja3ZkF!Y>aCv+Dk9vznH14rvIC*tSYhK)|kbD>{eU%M5SsMnhx+`{K1
z+v-_PWElO|^9)i%2}P@2CB#->|3tfctBg}K%T}MJw5{V7G>*Jx;al_6hubh3A$Xq2
zIJKg=AN<}ytu?eZH&%>&h2C$wJ|ddpIM`1kD0;Wq8KxdF4MMbl
zqsg;mPl3co_xaKKas~>#F@}X%=y4$AGH%^ji^-e
z9jXnw5(|`A1w5AfoL3{DB7_l;_Qa&m`#IFvPxOFy=0i^eqSg0D6)RgUZ@Eam#Rn8W
znNtZYV;86R3uyLr>-1dw5FIs{MR8`e>@TuU1?y1hsi`Z(d@tMvl6Jb@vySxUyK4iY
zOB6_XNNBaLC_nO2IOr1U^m^l}Su#tNkoW-8#}q$9Hy01U1pM>;C;U!7>Ii7OXNXTB
ziZ=5*IgR~BfAQ*B2YByqa<0iA3v{T|`iZ@$nX!xGHwlj3o#{TB!z8_uW-6E>h8&b%
znM{er1}z_cn_Y~0$P;i*&8OGPnp!a*lCc8lgG2puTQd%>j>e!4GfnTru$-rDnOb0y
z@$@?4XG@C#;P8d+xklwQ{xke5x1on-v7+3t^q{XfcTDkb3XWt1c%}dfeuh9_G|EvK
zJNjf6&HS=?kBfq9zKaOTjfa=v=aeFzIh5|@92`E2>SqJLK^10@lN&iF5j|;J#KECZ
zlp@|!ILpg=qX3XyTU)iNwT*K$w!rnt*!?m!@GaskDt5OKnbjss*(pH#xOh@{e0h+}
zV?+XYbORazU*Z|3@pH=Y*hN^HUr+VKo?=T8BUOiB@9
zzS)eY3BxMie-Og7M`Xz9+tQ=>LPg~U-ov6Z28FaVN16H~a??-FOZu)1$`o|24gt{Y4U
zg`eDiXCgc2VI?gQn
zZ%{L@YPTu#wOeU!S{AE;SWJ!77u{`fh^DRLpG3b$xc@kN`0(MgLOE5~7Z|`BMP^X#
zLmv0ujzZk7eks(>{6bwz9V@p9@R}1YiH*+*$m$?#wj*-`t*Y=p@E!l)0agRLmDfX?
z$dnd^Fs=CaF}fDgYJUL=zgC5={hA>jIn3UC5>{u&(<3FXYHA_Xw$KwXysv53_!>W_
z{b=-1G5xaGv(yFkJAd_$carGCdx=l}WUy2pLKZSM?sCtFkk@XvwdeWTOmlpGe*U-#
z?hw^qw!qt5i68S_GnOth7E}V|i%*(v%bD^2Jq&$)?=2xGF7e?fA-=IEf;;n~b}RbM
zu)mB87@QV3`0aeit@;9i9}`E5m9xo2a(_AL1hgTc1r6)RT5n0$UnNVdWsJ@2*;Uiy7XZ}2+3FZLVJ`rrBz%6E&7fKa`wxX1$F71IYl@j!LTPHS`c
z^;r-j`x}#ytor);{6Ux34BP!>)FZXid^RV*&b{Yy^+Fl_f*$~y8OaYMMaL`non~wl
zlh~1VzP3y@=$2cowF$2>H{s=tfP|mik63rbg}lBpkxh3B@U)QzZ+!kZH^3D?vEST22Y698k9~tRznmc8n3vncyv2~&
zh)&^oH;LO;b?;^C_|d3}_g*KH!~Rc3Mgzx@oNvE^>m-}095PnBObV~%K>@k*x%7$!
z09#B1RXQzJ1NysHni7wgf#l#bjME>;RA9PcxWTYg=MQ}Q55`l_nOALQfSTqtpmX(s
zXGsPC;3WE{}(vt@Q*XA9{9n{->mE!-2O%$_g178NY??LdA8bODo$gla2X@Jtwmv
zmH=uzgkC{qW2h`}@IPy91e7cY(Da@!C!6Z0@s|Ti2UNb;6P+UOV4p*>$&u$O7s6>4glt)v@~nr{8Wx;@6_@eI=+Ns`EYJyh$w%w?;1XqSlB%!x-Q93f)9I~ABfHT+RRsUdI6xgX
z85(90sLv1wZ?|&;_{C2^m0$Fs^83)C$&+g{w%JqMN(eo#gdS+UCi3|U88jOPxmY@@WV`!
z5O5J(zkx2s>z=^<=;7h@dBiV+Fr4F-CjbPrY?=T`55nMBP$q96s@Ecr7qhoEl?*D%
z?i)8AG{m&N1PVsXgpMt0V*fWMy?&Ny9&<|!U!`nsk>imTkirW~Q5j4=g59J@vytN!W%k#btx`;WE$lmqY&IEC}fRQu=$U$CHFHk=b#_iGD#o-PoUb=94l9wVt7i
z2^;TIZmJ(q=o8UBQcPXWo8$x!Ky_A0IJfB2uNlciqu(9s(3(|RER(|HIKUbt2Cc_`
zYazMiRI1N&KHhSWJWZv>v2qVkriNX!qDy5P#nu$R=QGSzp}*dt5I>}r_QRs!)h(VP
zTy*1;@12dw+HEcjF{_RrGFOrE|5M11OZu!ir#jft)*Q?@xBsmeF=KVAS_1zOI_(=8ctVY8iq&8G7W`+6XmrJ`~oJ3t`t`4qD`>~OWP
z0%03!WnoFfxZ^^7CEw`)eM`9lMBu~y?aeyubKXhUi@D)kF~FWo-q*nc$fUwrR7wAG
zYn1qU{VW0!Nm4cIoidEzxZ1<4wUqE_mOzz7eKDc)QVV*_6YOY!3c-OvPxGsAa@q0s
zhkK9CJVywZ8>1FNGd^w(`IS;U){SGgWysQIzU5
zceYVD5rFAHFM1cjD#k%T$=IVTp?s57CHBr!=jxQzuD7eErHsT=MCD!++qM<=7Lo6dw@)qk5rQe;
z1mR!(U7!1?&BI*~h%t<=-sg>K`V_{M7!OPvr}
z=#94(n+u-XKUJMKuiLne3gy*vC~nV?0`JGFcf(7D8vVa-HN9dI*KJny-8I=6-*!Bx
z&%J#%1^U5^OPeoBtRq#2v)+HLkRJ2RZ1eqvx9x0LZma)Y0F^Pf!c?&aRZ>*jxS~kA
zHMNzfS|g3Wd4K%j{%jQcK5*P`^V=tx({_}5Lild8(K<#R*n7{1hs)L*!9Y*q^3UBb
zTrQ?n?-ofkdq8blh1v!~+p%^t_o-9ke`@9&FO~W_0&p%DnF8GAeNLP969ivqo3^(b5=j@vV;<(O%;;#i>wcCX5An
z34#D#xOsb~RGaFJ>MB7~#_Dk+@;z3Hr`8sfSk#-BCt=MWg=7^Uk9!b_)8$Zu(rLUg
zUJ*QAZ_+7#BzOTsZa^9YN($7MPJXDyw+k?|w6valgnqx&Kr?l!)ncJ^SRvxKtq>;|
z`PdGzW;(sNCu|qaVf=MTCaisZL65JNU5Nes!o&R`V+kS_?N&za4`o$JRxq9BuF={m
z0dY@|G+VcEjv^w9Mfu`Z$9A8y_7#ik!ykFv-RKmluC-!^kTcu~htq_|1<)2;I
zK?}?}kVfCAXA_Au44tcwaw;cC66R>3(A~_YD_IHAZXYtMr8@n?3DjqX5^Zuxct3N_
zuI7Aa;vD6NnTExvUjJDK|Jz)S>*F5ogOEE8e!ll*L(!B{BK6Y~@Wrq;eB90U+jePe
z;wY)E!f#ZrLG7f`kSv{$y16=Hfznizo1j{xw!K*0403khMqzvT+b50kv0p9GSBHOA5Z-Tns1a8
z;%7YbiI|MM%u|I`7db3%wHk~1$AWiiFs!cxt9=p8mQ;pAIzHpAt{3{0zPOTGwN+7x
z=hEEx#o4EppX{5N`@0iA21$W?f2lM<2R9|lReQ_yD!ukCi|irKZ<4a~L&WcSwpu+3
z3%^PuKr6tbF!#M+_awswmgc2!C)
zM~FSObRYfUDAp;h&ne`eC3=$biCg$6P%uea1b^mz!+~g3t0}d(MQq+^Caug9tw!k#$=k^{
zSQ5Ctl+&K%G@U~-@ddA9qAjnx!?MP7L*am68=p(xQg+O@`v%C8RINPLKb0y{4+^-H
zzWWgy+uKh{nR7K2lggk6;t$Qr
zesc$4r+l(`SRV5mM*QiY=7(1^;YXq}YS%#I7kGSRI4(%kd^kSND1^;eND?|kKDo}Q
z=+Tu-+%lNS+mV38zl(##fYi$AG;3-6=4=G&o{VzZxU6`~DNG`h6*1PTs4SIwbT@Xb
zkzpOrlBwTj^N8n{V^(RU8qil0rueyrneaY87i~Cf>Cr&jxmWvoEEL}EM_gsfbVQ*Z
zwwm!uNvLmlnEu4scej+HwN>Zq|JJrVodKA9rWm8E`$?FLXZvy(6*|8w%;woL)9m;h
zbzu(Aa7=1!xRPNZ-{237HR55PLmyjNH3g*mD38TGP?&6e<#j2LO0moCchMe+EQXyS
zk!^?%V*-)`yB@hoILz_L!c16w4w8anHCYq$R#(X;{T^Q*jcPtV($?J)%`7hLn-c2t
zIA-l_13K4ddV~=QJq#bu3J;6X_LSN~0c8fKy?RojMTCbh$Q*`{{%+<&j61CecAMS8OfBUuqhmHE3J{=93@AIs2nammJm%3l-!NGZb#E|xx*45Lc
zrPfSYuGF`mp?VYg?S)~BNN?u
z6We^~H)nXpAxAHm-sY=-Vd)~d;O^}W%g!L4eU;I0pYZCKmlS2nYIRx{qEG8fNf
z=I5F>&POC%o}KSK@!vAKwL#ooRlVn_0W{5((#NcK9y)lhK1_ae9@Uw~gA;;)=7^8L
zWgp57`o)HaFS_k(p8|{r%2}%jIYbMwHZu%AR!w!I;uW1cc+h%h-6-D>O_YB}b#QrH
zbk?tX$?TZsnUUx9s-#Vdxp^6H9`#t?33}3yg~WAyCFh|al|(WrXE7MkW(W7I9d6Z~
z#4JG*5w-j6q4>*o>sPK?MTa*j0vA(b{3PkmLU+Vk=he2$&bai>cd9cM8$G!i;<44o!M+I)At&4%$(;Ee&4OE;`l0`=^l5gI97L4shwmfGVl0loQ!)^Vz)W
zp5$62SXSK2lm7q(b9vw)8dXRi5Q29YXC;0}hgYS>{aZQw&4I~d*GU-H!m=L};5-w}
z&*r0vwPab%BMnf7X6jF$JA&5y8SO{clID}2UoT3%=2!6@Cbmj^-24J{QQ+@7kV>cd
zE_qijGhnwfyYWV1M(I`VwzVsb1}}D@ACYnG)bkSFvJ^8M;^jFHCOed5TsbUm&DGhG
zCzRwx02yAp;v?nsh7a7q>ae;B(A2w>7w$EE!cG(30G!DQrqyf
zWnJcdTy`F*##=wMLUfI{w-XNNmb9nWr`T6o=tATTzl`&@Zkm^TD=ntVxaOCZMBI-=
z{SfJ$vv%_+h)yOHPaB?d1lRjg@OZ%}VcjXTWdJ!tMk?};{V`&-aRJ2ewKaprwqCVI
zL?e1M#|h;GXzl*2@||ww5_<;cA30788InG}IL%17&Xi<4f%EKJ{>$;=a%!IQz_6BO29sA2
z{84}bsndd*eJdM&>jZ<~dnS@Ios=4=O0HKFXmB1V*N~+q%VbVmAjR=}XL}=ki`Jbl
zj)*_f9F!xji0UYj%d(R}!=(ko%iHQd?T~p&LqpFnO6lIfaGpUs>rN_^vEmjojh7K<
z%Ff7)9F?q};{&S#ICnH*h^(Vba|#?}UU%2Gp1(;(`}!nUcP|65A2x@VyD%e8+X*yP
z5m~zEcvNz3Kl5uds>h0zF=__QI6CuCTUM4)A6IM1-y+wjPahU&YpRc$hDd%Q$Z6)R
zZgORU-VXgdli=#RmHg%0ZIZL&kuxu-2h^<=Kvnuf1?p|W<)!L4E89Q$mZcjZc3E%_
z*eJ4pGFw2{t}8STnW$5Z)@R(2*rIx0LsUuX{?+U%%<4`#099fOC#Kp+dHlu9_z9(EZkfKKjd5?}
zwi1tj
zFwGwK6UuLW^_u%1NeZ86c+u>)zYrx)kI%AS;q&AhT+c~W#QglYvEpa9pXpao>-tf^
z{ZTpYCG8!~zml~8T7LghJe=Q|?|RWoc2e;0Zj*mFOZ)-ZeefvJ*z3|4kaG8u3^xzHs*}sS
z>&ai8uw~y64$EVEj`1&0ykI&kk_AV4<9np*4^KuxP5o7?{Stf!kfA;B=pg;;q*kkw7=6t4rZA~f6$I+S#L~<|
zWo2IyVH=C~%K{#vi~pi=v&5uft3%U)M!CMxit`CLT1DVL4gUvZ;6FhdM=zJTBz#3W
zsz{>YH~g<7f*3<7}{}Vp{|6u-2=J@~R%qW=ehV+~^dEK91I$Nb$
z{yPUHr{@D89s1)6O3lb6VTl~cat&U(0<1=3`F{_%6*|F>*>xusw*USS3|SCwD-*QF
z{{Q+G2x@>}>WWJU`}e^`IzRcNmdbFG|NC!Rp+mcbNTGh8th9wdU_T-hFr*LrZ|Mh&
zMM>7D4i-BQxxVM1#Z5fQnBVnGoV-_#0-gQeLpo0d()RtA?6!{gT=OQt#WB>GHa$Hv
zz2ms&vOLq3ONpNGIqZMJ{~x5P3?=O_*F^Sm!@@~^T1e11h?IK&WrPzlBnRyQnL}!(
zF;)%RfpVyUnem21|LuuTed@yN(Y~z=B(d)w_oSU^Gq+rk5lhQ8WUqAV!~fp=-yTW-
zm-z**e>rCwK;DOtVryn5#JdVCbl}4}(L&qNJ_e)ydtxGyvxK3`*%ZD!nBVWCCN8ak
zC&2kHV~o{ES8!zt#bN!0$7QvnHJGXO-g2#1o>Jgryy(BZF2GZj4m&S46{*?nvIc1K
z_&5599LD7@CRE4thV;q)%WqQ>@R8iEmm4?D&CbT+`1;Re=U)ti+#ob~j1IsD>UdrE
zyBv(iWXcFEYPqZ%n*YkS9k*}!N9WDI(zu|*i@ew+;WU2gshtUWB#y*arV1ISiT{ls
zo{%CrFpZ*me^WHY4&3Q}Soy>J-?jK(e@j<*!jYH1f3>Ot7EUYLKT^WtbN-1L`TO4l
z}dSf=1PW@v*@u_6e&nMS5Lon8b#x2|
z-5?b}0p342SPiyuu2Pj#o0C5yL7vXQ@$vCSxVYN~d=t@gp-ajsGJJKa
zyD7I04h6X3f9f@o1<1L|RmOpfzusw@RzZgbZA=XsFOQdv0Y~o;VX!zYrnaCEfy&~E
z0bU%5j*i~LSNlAVgY6dReXEh+X6A0e+$X<4#B&+7N~4tZwKcWQKon^dI;_8P
znOUJXDQ(dX0$KUThTZ&;Y}?M)2T||;N~3a>&??1R=3m*^HN7)3a&jm0qhCqPm74$N
z=p6m?HA-R6F82y-jZ6Pp;MjNR6^E0^TN=qTv_C4AQC>W?muyGCGsHqhzWp(fgqxR05)cgq1p@*4U=
zA^{2{#g4KDbn_zjH|Hy5)SMxIHUz;z>=8bNOUZe5hePCptYiL49f7&hRmQ=;mJH*q
zx0_?;IK}3lmnpIrGR~zHh(LnqCBj~~xVU(R284#%Ln7I-7{&Z~j1bE7$Gff6{NoUN
z)QRR-3jwzdO###Ve}+SbXfrI+^>O{%pRr=RPp>$bgvMU6S=E*+1p0xB&`&@a
zdED|88J%%}0E?d}{)!&NSSj&WoMC7SUdqi&t!mb3_oyfC|A`XdXMMT%KYrFE=+G;F
zPp{~8=GfS7mu~<6%mr8`rORK=*awrxFO6>hnrx;vzBm74%`Fb?9{vSu~5E8WEzhZ?HkA#g9X6E%s
z#h-!ud?ZPCGciQj8n|zGsUi36PretNgRV2~35!ShJJxj}kDIi|AhmQSuPi_9!
zd-4A!%4JZ8ba4EaliV}NA5YbDeDHtE=Oh}SD5`@^`}ayiFkQ`OTuq8Wy@u@f8Xjvg
zpl3K%UUQ58HQ0a3rDg>{DFumg#!=0~ZT0UTTVVzz+ZBE#+`oSWLlTTmSecsoe>YJ6
zXG#R`o=UdGM85xbcK-9S&*&Q+Q#IoOu{^~e;l;CuZGWMrPozdQ{H4pClS&c4(n
zrTSY#>p1%K^wE^8!T2JX3i=3&Ho5wnJrwY{;8yV9SNN*GiRz41<6qtWrxzN5C7pc|
z;!jdYg7&2*KwyXsDScOx-Jrt!VuJ6leT@~mE{*yf-@7!J%RLkHmcJ`!LEfk4A(1%C
zr`N7pP>?$*4lzWb5&FP!n@tJ$6F$vM7uu3%D$Y61E(3Uj6mcZE
z7}1Ypxna5-5e!y)c)gi|C3H`
zXUX>g>f=OdMJmJYpW~xasP_RB=7!2Z&6MOIrri84UizPy8MlBq*yLz@khD9StCY&@
zV*5LGWf6r`
zg)|vr-T?v=h2Ef@3^ml>wYipL=ZlX#{I~>2uif)l}fIsZDlkBqM9TM!lAi{W`Yn
z3CuW4dw;Id+h4d~ii!wHd|h3G-3dJ0o;+i#k5_xtjf}px%I!`R8ut)<3grV@;@`j!
z+rQbFn-zp2k59T#=1#abpKA59Nx?qzCBPj31BK?HQk&%f>>1!^wx~j%X8((p(_u%c
zamg16i0_D|%{*-}fu7d1^|59A*Vve45FyXY+SjK~bIIJccrs&4DmQFcY783l&v`iY
zpprR_25!;E=yv*(AB_)r+p=krSu~(2#W6%FKCRJk!y#Z`DN#_rK=u}Ym+4J{OZoY8
z9yAOzgy;9tp0Od=yD~@MjX-*mN%szO#jch5dzv@`P4bD<|qR`^%Fe9t4=UByYlG22durpi~>Hq*Q=woFSj>J8ewAAYsmD-zAt
zN0-6-fBgi(Ma*U0p-*as|8g%!P+)Yi%W=$~*>e4-@_F|LyI-cobOn#|=8J(yjTZNM
zrP2p)9i80BX9$sk8kBNd{85mX@4OD9ch+w1W@LK#d}`y~#E8e+xAf<}-yU}nV_P&V
zqgKR=I&l%Tn=2;0zr$W%5Eazf7)YwHoShUB7(*4%ATjKZkLeC0Ms=Mpii8&Th9)CJ
z(_D42^Z5AVD&?4Im}rXibxQb;17ACuLvst6G|9E?0^QwDrQ~e=HD1$f{(L_#Sl0=4
zFKwmnNaa9T;L*}T0nCuhv^dO^P+d25<(#kAbvLg0+%5?07Ck=X2>J-~;OZu8G&&AW
zESzmyOYMxCW~-L+msrYc3axzvD7nHVxZwYa~I_@HsKR-pnnS
zNsZY|SN_*tHg%h+Za~cFX**$zeX_}Hashw@$41)pVW2o2JY%Kfi6!6$UE8v$yts1e
z?KB40pp%^gXQKQ$Mlc4%ooBWAW%klz8E8qWaM)F~yFPh)+QC+%OZ-!9)zkpLqw5m%
z7m8h!iO2PHotL)E&|O((i>7+Zt_o`)1z?TlXS9Y4)_G4q!)dZx>m>zU(NWATQw+D4
zkFu?3+P%O7pEL-SI3B23-}#5%N_azyll+E%MNj@KQ{#+Fo3ple?>QMUFL^W{0gRwC
zAI_vw_9@izzHN>2(C+wl|1$`f?(BFndMz7%Y*Y34Le8+Lm4wS)d3TDD2
zKPErvv_5@r+gCyRu+7As3nTR?EeAez3*~lPzqF#2`et*+C|ucK9R(q0g;#}m*aac6
z1KWFyQJUG7KaM0}Yxap!1if#+)Vq2`v1e_vDxYxQlCdAuj<%%B#?}cf0f);IU{;kl
z$#*%I^>fcnN!Rg5)1KpPNQHb9Xseef{#d;Z%vNK7coT?vKL8_V_Y>e}EBW{Ul`R&L
zDxc5Up^pZUp;3>h(0Jj89v^1m13GdLc{$w9wl;XGBx1gQ=i8$3rA$^&%s~
z2SDemmHW0$^h$33$^jP63#tj(tkHapoAcW~Py)$cV}X(UE~`uS`>&2{#zVIka;G6j
zl0cTq!R&8hlK@me-4?{`MjD9~iPuMWrst=y|HrYxpTs_8ZaS9-83@+mnJ633PZ3L)b2ASxfdef@|IHqg20MtGITFwobj<
zL#I*7i(J4;P@wz)k;I~_AV|pV_0WV1{0E%HD~tw7!LAT(yV~tGu;8*+-d$2A56IsUgzFs
zoPT~0wr3KIB$D*J7=34x&~ph9X04#ng67>1aoxjS|Fa4YkH05fli
z>pV9tydXpG%Y%$3wZqPMd=Ig8X8qn-27j+Yks2$v?TT?o$YL>x6Q7I{c8TqBt24&)
zzON)VVQ^brWH?o%AmK3;s(S6&9srGQFx~TpK0nmJA!3jGqW~st1d$i)$u=R}?tV
zbFEi8!I4tV_ouRX_*64sZ;qYzJotub)z`eY)GiX10eM?QkA4&I`WLY~i|WJGDk_S$
zgUG3fMLg)0o$Jhl@OwA*ebAw%P5k<1w-g3}S$7&NN$~Zt;4E65yh^^pdvHq!saQC{
zD1HlBGQ&pQ2d^cck0p*Nzuc)IFi0Sxux{EO1%;v*3yAN9wOL*{BXv-p{r(Uo)fmjM
zN1;#{>4Xqv=OU_Q&3N(wHA)`OWupX3x4+K1eXsY>3v)X6T^Ld_#%5y-~naM+z1
zMSYF&dWa<1VKw%LoS4QyK003;X_ci3h+()(+(bFw0hgJ0o0~a>w0SKTX#+pkK6P0{
z^}`u0GR2P}SOXZY$g#w-Dld~J7P!;O--~u2ZKJtDC-@32fg%E^eLCMOul7W!qIQyd
z7XFx;D*AxL&Pf)v=Jj|_kI!Wlz1*oC1>UztVM?8sIgJ_(_7Wvp6J8DKY&J_I?5D>-
zc@bY_I<{5kA`NI~P4+j30DP=Th!PZ$Z^nM-iQI`lH~xpImj{tSX{>bqAjud^F_g9zUy$JTq02aaJZdZ4RS7=&yH
zxQfiwSQf6XkVL2g3oG8=`<}5sSYLCpRT%UsD7trtZYvajRF;T_b}}VH(@K%PqeFna?Uk!K5
z`3XD6u8wPqWcuV*)+c+H6Jxd4AoL_rle&!U3NX
z)r@eCSf9BfX`r)CDd#Kv7zn-s{>^KHT)9blEKI;yfO;WF)Xd}df-t#mTjU;tNbxY(
z)0g=sMwZ{O#b8)q@k1@K>ZJ_wPPqAIn-6pT$!dXVmUb(|4qf%&9Sz}|ix`5}N6p{{
z26Z-ry4iv8L@w(VTA-$VSFQ1oeT=O56-`dP#^Tp`=SCg)(6&qdM&zFj$O^h(16#r)
zw^wCi)ze|B&*K@4cVk=Gh}sLtub$&@b$n#QN~fVc$N`Tg95jNn4|*
zn-1Wjc3?zPi-`)31d3xAbS&x|J88olQwx^|D%`yayF*ld^qJp$^G_T_JHzM3pM&uz
zyDcX~M8Lt&5p_9~it4YuG1^UOl7Y5O)72?M#PRp{K7s6x=i;KpRPH75$28JX$c%VK
zbVB4_7Ve_^?cn!&MoCZTk_T;%^ElkyTB~~nOwKeVsC!t61f~>Rd`pH^hh6>zAY_>s
zj}`FpWZyM9UU_Ro_%k41QF~JV*RIG}0-pLcTxF|9z1_#f!|p_H-=oMf#OI;wvo#($
z?}F3L9y04=Y2-U_?%oHulY><1gF{tznm9%xOdu?|sgTvk|3j}SaBBa^%6sqij(Z>%
zzp!qE?TdF0gFG9Lj#9*LJSux93uMK=pNEeh6D>fUIt4S#fToq=IIf2#xB{}
zpLiFuf}X{$M*>%&Kvs%2cpVhPgEShX@}A^4_yrlWU|>|q@=o6_y%rglZHjj{9EWvI
ze-bzoO&pC(&>UnmX*byC)Ai5P{{&Y&_45XGo|XasU8(^LEXyO$u2ZL(s_krW6;;Uz
zFyWMg(Rex8pqVF^;fpYQ2HGrU-tSPB4OytSdN!M1uwZDAhbm?W2T*>fFu>#7srVq^
zcJYluHdSNx7ygQRH1{VX1cbg%=7ZP4TkgDwjm7EB!0S0#A_#bh+0L%k!^;ke5rI?<
zY#p*}#GL8<`F%I3o8(AbPS#k{E>Jw)<(Y0Jq`GWypOX=iCq~NG+Zi?|+>AQ=4W&J1
zFdp3akLnDk@)gt#n4RG#_L?4+%E3pGjxf!%0)MXe$T!<#xx44qdr{1o*H~!x5Z^$X
zKE42O6rtU={Nf_`-|d;bn%
zv8n#K0tDhuVbchGs3@R`L_#(Kb-yY#sWn$CvvXkkBj?FzTV&8W=!`1dba;Q-++|JH
za(cz&xHY0hZB#tz-oX`y%{9~IguA%RVa`9Q!6g#5jHbJ?)E+hrT5ggXSg&!1&51o<
z%Z7Oy+Uu#}PjC@&&Uah4SD9Ly;S(AxwLKWIRior?6g~(Jaq%Y{voKL${csoNjtG7z
zKiNPT^%q$*h0n}*hv2o+h2S8ST>QIQg(?;T3sL0=h9C66e(<x=D6s`(-Wd>h{+wM=DC&?@8wO2ZH#gKuSrqMz}_FgV{)?#?ttJPHGcpCNR?
zm5l_V3-|dnMQG;3eQ{)5hI~~eW1>6ClP@ifpiB4gAR~$ezkKvQTtTPc0hwuzmd!mb
zYpT}6M6KhPXsdCGmg}8m`K?#ATj%4qtZv^QH%-23LsF?uO(@BQ)R55v!s$!SnH9Yi
z%>`H5)F)B#`?&(MaukA5=(ybWlAWwu<)<}oB#|y%1s;A+UtV8^+d{)a4-9=Pe`~Oz
zdP8y&)*HDl!-P1Hmn}-{1A51jerkhjb1-*wPB$1gjK)>Jz|+Iy^4JF>?oA#v5DN5-~})kVPcFX<|_&l-;=8E`UL}$uCh3
z9jvDU1Hg}bsrtx|tHOEc85%@U`o`aE@5{oy)~#>}bqua{D|*rAjedBz`$H~jwoKUG
zjj7*+$vPDAUImHMHd70E;9%=Ewbk9!geN$hYe9A+YWuQ!)P(1=5S-=Xd*JG!+$oBDwK2)@-Ti~vjY7D&|A-3*m%9I+b@?+P~vefh~p9o5gOjbT_>K@}Asosz{^0gI88}
z79~tEbe~r7JH}cNx9eGOp!#sACX>)S%U1L40rIIEf(Mn|d`H%-v_fk!IjINLIe*esNn*s|{bdUgdU38)f=V3n{kX5AAY9T9v-tyq7MvqF
zlf@RlZp*tlrSeakOs7eJdQxt~zR3ICjdeOyx+G8I{i2KJ6ZY%|xFpXSmK&_?o5NRJ
z%ys$gY(;)YcHW@sRzX9WX#wfOyk-`c1SC2gWO$q|U5SB|H=@a}OpRbY4t?ecdbz+p
z`dUQrJQm`M3<3P-Y(+PNYDluhR2!pPeb`6T))3zBsJkkCsMbCdH%l$^^#W}B!nn4}
zW+vE(2J!)Irs&Q@YCahqX@ViFmVV-u+dR8AVZGcb6H{5?y*TG&L7C5VpN|AtiV2(h
zL@(FTNu$+}viW|hz|d;;(o!1suO_1&`n)!F`eR@K_9lk!NOv=!D>0Hc>6>he7&F8`
zATW@sH8mJXPmrd8np
z)Q#qt^BfLeZ{VQVtf3hCozm?Y4~Tg!6$ktOt`RsI>2;}Pmw-5z^Obrw419R=xjd{O
z^}W7JC;h;?FQ;#^oC??9&*m?>#WepWp2QZ=s;7G~QL3#ivg4n)5APSs`z6r<3H6yc
za7(d|f3W9}HbAFIL5F#^@EJ{-2U%&!;aP;#mKxemIESS?B^J-H5x`ETh#O^Mws!_MI;(r2Yi?%E2u
zGOOsme00q&SIa@?mzzlXI9**^H9I-yW3plnndhh*ZqXd(oE&Wn`kq<3(0q=2iN{^8
z=M%>pNmS;)_ox^M!hJ|ac|&U`rQ#K`L?wK36Kd0~aIojYo&y6d)^23=m#w8L0Bb3D
zCnDy8CVqcW=YN*OOC^2FsWCe@PH=epei0z!%K^f*=MJAH`1NpabC2&)ibwbiaR
zhto>Aq!07nOw{6i+Gjr%b~m}((s6_i0I_y}ekGq~`p#ozn%S8_u&MTqy7gnTif(Rr
zpsRLxLZi}8`jhqL5jnw0K5C+nTOQ%(nv>JP2X=#nO+b`vND-ygGVH>q{I-)s*-1~7
zd<`ACl3*TYH~+w9aizYQ5)0CfCx|gin%5=jXj&t39)JFTuK4S@W_{4jOJDM+1vw9s~8UCbjnD9~(
z$i)eO5N2EEaNKj|SYc*km2Hk#=K*N<3OO47Dz`9Udw-dz0OP
zC_?rI7ZVoBUvs9xp^$z-Al3su0@@Gii7u=~WmeYfiT|
z*-$XuI;0g2W~#qpF>4L@(^mFu4yDZEK%s~H*kPAVy&Lq`sxWl-xG5lB*hKhn#z}nx
z&FUFN10SCgtSu|5Ery{pD901a@ZlrCWS67VpmRUl!2SLh=0l{=;?8}Fu-hVTSI8lO-?s#5Q_XOHO;6dr-
zoCfu1Yi_FRv%yciTzqi|hh)bs3#S$cPAlW>JHL4}sn;7tFp58Y&zX8>+eaks|A_
zY;i7A3r_0*&VY$_l$mGj6yM|Jm%zE$S1v@r&L@T=OQhB8#!XwfjCk#
zHTKwK-DIygl!$&UrW5MVu!zsY-*Op;`EF!^`8B0`tL)WWpRK_myBx;5Xa1sg#HZzi
zyDWQ+_%AKA%ZM+ba+ON&H@=TNxF1!=#4PV{#HK^XJPUyo@P&trjUm0{?)o(8s5_WPhYw|b)|(%uPZS_p2QAZwY^h_*)2N*tUzjgOr#rvC
zD(#d=fzcj5v~6)*c}>%Yenkh7ys)awGWnOgnK^{$Z8Bu8+n
zJYdV#f#Ma(i1k-HM5)ZQh8KVaXV0v|Wu(~9NPsit=>dWk+m59UsR?Ti22n!1M1%VW
z9+CSw)TILp^flAwonSz2qQ^#(vGf~o#~o)B=>X!QI0+sCfB@!jhWT%{es?;&wrtOP
zM(+0__QxZ4c$oN6(HRgnnvBP!79{ZMoh2TXbX#k1pYOb*A-f}LzLi5O)>4<#jfSF~<3z(yM?xr?XlS7f
zaO6W%RU@^BJnm(|$M6**_Wg?ArRbmD#PM85#gFkz;%79Rm!Drcyt1n_Znd7fDkJ>t
zMxppz$-=|t{wtqb&2AY;fBEO-)d<5n`dlgnYgJ*yuytG?aanZIZI1zNR6h+auw>wf
zReJSdx*USw$rxKhsNC~lj&YDsg$D2#A<56M?k+dck_u6mw>8bT~T
z!jrcEG3vloz!&YUoEC{YB!)QoBZM9|wlYj-q#_Yb^0syIVlh
zBnyfX0OKq~^jM_nyq7kxQ=g-~8Pc0}-EUVWu3RXZ>VpMbWy>xp-Yv_;Wa(q%v1A*M
zRU$BQ)2Cw8vFL7YgF+p*d9Si`g(wLQeLhoU-sLxubj$d8{E(8Ks~^%+>N61?{rQFS
z1ZWPI8F6sAGkXMCo-cm=&HC}KAOcf_xPF~7N+O0uprAD=g^wENwOGJo_)#Fs)GYK^
z&3J+JH_=g9)fcphZ+mnJ-K2K-;A+%`yd<~zpbij765%6Bu=p2^id_%E(`t+)fh4<#t
zbfwo@x;Zo|3}KhBtEJvje+k&5@GAB|%X>$*WrsmHLkB3wv|H*9-?q*bK*~(G1*B};
zrJsppnhY?czHv&WC6X7{XAKjOBP#b7Kf4WiNQ@2<#aguoN^(x@+FTn3Kly
z4d*4D7OfTnmQ!YHT~(MOV5`FvSIv_fa$tA~PA*>|X97M^Hin*gro`w**{Rb4y|wWaBMS^
zuDc&k>p7_T00&7|cu?l+xmi)r;u3W1Dmb5*2QEJ&{;Ch&YhAnIrKAB(xyxuuV$)xn
z!yNt!Dlt%oKV$odhP4jX=M$jv7n|LMKgbZJP(zw?ha$ypkpkOC;uOZR9tuvTkK^Td
zWK|HbSnztx;Mz<~Bixrq^`90e4TRo@Mm;Ns)D7t2@GV>d?a*|x{P|7=wa|SMkzLR~
z0|ADNGYnaWojLb2j^2{BH7yu5*kG)*>EyHh_AlkirdG>XVr|ds9PSdSgiTE%B!IWE
z?2`f~6d^EI3v(LnE5R+Oc(2+B{D=|e7}{nt^?v*`D`qI>LTjwv(_JkYucAGyy4S)w
zpzskDefAl0;6Y?X#Cke?VH8Cw52B{*RPONvF$z5_rR&$D>9SMH$kcaQk%*bgFB|XG
z=ublFpug9SiALw9u(7iz3GPGzR>5wy1rfgmUIP~A}FY79@G^?9<7V2g+q?&){E-r!qaR@UJX%Lm?uBY
zEX8hN^}97`2}F9<9Cl1MxTk;84qTd0vN-PHoTFV0#L(q8=q(xPwY8*$RW7#BAccWqlKOF^YvT#JF4Ah>rrtsFKH5sjvmIsdsemPV%)7F
zmW-v*;jIaft=~PiM?m2c3aIP}))irRFj~C`|ezySYCCBNHu?X6Zzh(E+n}I7S1afnu;t
z(naH?+NxpVhj|Nn4J3M<^rgt`*Yr>Apjje-N@_~7_i@VtV&S~Z@(PJ9vxpvQVRk~k
zo0yP^7PGjA^t>k;$9)A{NH2c^td{aqH)!>Xc(S4>)3>@rM_N2?lkMt^#F@fE;;PLB
zXi)I^Es)=g40?Y84X>K#x}+|4J#DP|-S+J0m>%FVPRmcITk?_g6leyJ`?CUu$c9&5
z4@-XB@A8SYFr8D&A5Q^KQQ!g0~AZ`vJNJF&$%
zaYs}aq*eN{`m;oKxG~YVaoA@7Rv#Z#2+qS>2O+39VkGQyVU#!aXTC+!C~VnrdLf>U
zRB({qr3xutQsr_=MOfsz{l1-x=@4d)MyG~ckGe5P${V}$?)nqQ*Bd4MSArVvaBx^g
z$Q^N8yj5;44+}%WhLVJ^LV_Zp@kn0)3qbQP&}ShA;Ku1F4fZBxtS;Ct6oo)!r$Tc@
zrx>|D6-fFA;MF^TG6JkY3`R2}26VJDMG>m4WL{NL%DwcX{n=jTJB{hQ3o7Eked%u-(LSrM7iFiq_+2SyEdO0Vo|;UZYVXnk#qe`t%KW0
zKhv!vx4)*yHcjFXH&`Uh?G9$&za=xfj6^beq|3EV`nC1&MJ5r!jaNWF{FyLu7%uxq
zz?2hwmmkOm?ru4__)!e#m{3PY2nlmQ(BMpgDJx||C
zKwqZ}6FS`)hsVAwso5`ZS*6Kz>|F}hFU>>7n#5yb)cLy6?tz{sU+!&Ut|wWQHziui
zYQ5#^J|mg43lbId(mLaESj3~^eZlP&pmX#&I^bs3#*Q?$*oLPA_i=_j5O`nA?^dqc
z5RYnvK`pR}g*|sBm8fh^u=zjKy=7FDUDpN*n+~N@xt6R-b6)ehuE`dRg5|N80h1Ai
zNhoq4TNZ{gdVTI{AuyjC3zPIBuEG1XgH7~{+3puuGILy=RQL!7@>FCPT9PPe>gF5(
z;BmMFhiYwVK-DA-z5sxb9BJC$ELlJ6RPpp!>mX*fg&9#TNNWL7m<;buZ}q{D_$fE@
zkhHTUF#qVMH%>Qcg*dTI49S>>$5w%MXvE8{V3d7)RHka>2y?{2NKK|o#dYJ+Md+!E
zQOFMuct!I451Ug$_2D}{yet;xkwo=WKmE7~mlxiLGuD0B$)
z8Rk~6x5jYw4Z1xZbmu_Tm{}IO-r*wiiUsxfj|nHAwpiy9CMzt`YB7Hp<%Q`54gBaC
z{g@ct*?qM7_{4ATL88wU@J{34k9}fwfA4+E0hsp*8L$z(3_Nl?3mPn%D3_
zVhjCG=ZCu6*^0h=e$SD|Z9k_QjXogH>mg5xi!F$M;43y+gzRU3a6~&NiKO#TUmdK~
zp)^FQxv)^tm}l=qw=upoR_n8N19k3QMS92*LB#~rbzF2JUFD)LvnfmX!vHtOPGl$#=|5QyHJ#=Rq*?UE9h^<}bxx5&I(tR|{
zA>JH=moG^d`wnqZ_5)41{lSB#<=tI3e!7m4RLf;IzKLZH9x{=;3Pn_hrNgP+Mc)iz
z-zh>gL-qswKHc{In*Q~L2J&q$d&{XH*j9tD8wW9~JRrQh5D!@Pys1&InpTUMjHix}
z@ranUrG~(JJ)t}EJsl&U3lh!h^mr=(7(rW{377d9$vR5SoNApGgT=DgFQMzq6^fdEq@^>^X=KIw>uX+_(g`rhVos8^e}
zmkEn3h>uCvJx@p?%8f+~fqZOBK{4%<(lV&h|IG%rS{h~Bf`e4qb&1Z@uS5|Hg;XiKcs#Jtjli+mh^xB
z6)XmXRu;GLSxd0=lI9_48lB?@k->!D1ExTQUtoDjQO+K!Qj{Lv?h2_sYiz~nIBl%(
z2V(`VYvPx)skM3=>8svqtk-2c1%5k&o?u29osGo|`X
z0RdtX$l261G<0FKNBhqu8YcwYKB#Ki#
zqY9D9M&G)o*%_((Pp84`L|%aXE>V+qbVks52OUHQk
z&y@}%O^ypOcIz8KT!cP@>pm8kvOQ>bQk69woqp
zrQpC^@W^l|b4oh`Z@o9R`tI+yo}tA2+z|?(DB0(f@kx7>v!ew{y~T7L6$yR>CQL1Oj9k}P|*4&P~+oC
z^N~cD_UJtN!kKl)N)y6(`j8UZKs9!B`bw
zdM0W8lipj<9(}NWX?(K0Y7gGQ_E7fylaGxb*rnaly!luYe=ovze+bFo@;UdnWv-V6R2df*~n~hG;GPw0GF$k#yd^yIwa!)Mp=l
zPfRm8?6vpc;S70xb#`{CQU(!6@j%ZCI;zn{&UU9nE}38KlF+5
zTP<+sD^>Zh=1U2$9G1d8%)Bfy(Z~yW0z&Jx*>6Xu-fllrAQSJ;(4{|bp
zy$Z}gQ=#lK`7`YQ_?5x|VibkOi;H??+MPh86yOhyo<>AG!5~B1V0(ZTXyB1T^PeyG
z_wHGQEloBTs4jyMDg=+}6V0F32%;~3fWIDG-b$_p$e=VT@{Y~5i@&z}mCZT$K?CHc4=IZlK*5lvH
z{Xc!&{m&`PQQ%E=9rm$>5N50#-u>g<0>>daEd7AWl7M{%$A6iQI8j`bp=wMC2f?WS
z_MVSn!CKi)UibX-9N(WO|L5!fC+|oTyW|+pzXNcguTHB>lQqt7b*mko0q#C4a2Kw|
zWznWlJuPSd4`ck-z(sd(BHRW;fqNBTZdAMM%}Ds&90mZ=m;~T?(}Q>^9t7LnY)u~`
zd{K8`|LeOo-*161ZLMlb-``3`6ZgbZBbi+unE}2z^W-{)e1T)YB`FjD744wB}t$SV(-CXn@@%i`a>XhPe8$;vz~Mo?sY`^uj2$KrfmlT
zjeV*5XIa8ngnc#tlVPd80w4eYL2
zw0>p&YsdcI58S{15c3*7r@uOmTFQ;j@5UptWK|GW)>-v&<5-+wM}
zBkuw1IzX2q^&312##obC>pPddJX*a?Syue-^Y`q2{s4zIK7~c6D*zU5<>nG?F=O{}
zSyUWg*LU}_VLWsfd3s-=`Nvng(EWXF{`ioH^EqJTFaoS30eI$pieId!bhtRcBI2>B
zzQg^dGrGG|FK&$n-*ut1I-su(b0f^
zoDC1K5uV*!qyH~g7bi+W3xXnq51@b9Cil)T=H_s28eo%3vFg_|NYCx0n*<_;@m`U)
z+Wi$o{im@85CmF2Wt}+VH0`1QNn2$xS>}PJbOO!mPM*?#o*?ksb_EEBxo*V9NuB#X
zz6#dfZ_4{WO+*A7d=7WEGYPMVTi}1$52Y!(;4O8LIhED_#JvB$%KrEFDCr;>K)FRd
zzWJve{(h7H|Kc4JkZOe033}l{$1IUAWq>NLVK-OL2)*03d9zyj#q;-U1oFyL$RyAt
zJh8%-0r9FI9bU<_2FadV{qB?qR5!6$1DXOv|J|T5cK-yay;2zIpNWjXgAf)1D1y#?
z&a*I2Hd$3QubNB!szow4ycz6>Ifcf*WPd-33ser{dLQ+Abt8zuA(GROGZ2h$2_k)P#c*W~@?z
z@nO@gwnT>lDtJ=eo^08N#76^F1_H1CtZ!*~_zxCMk08OPAWrV52?UV$r!R{!?ID5d
zf35}G8_YN0&P7OEv{+9O<1c)4xV>`CEpL9XO6q=4z^NIfg>iokG^P-CVI8<99S@^u
zm4n(A0$e>fMJ)}h8=#Or(U2p0-zyjE7t*fY$%mGI=J2@^%ra+(RsBd_jU1~%BNJ%g
zl)o&mO=`i5!!-;I#{7GQx5I!rFotn*;W2uE2HG7dn+-kSl0$g<_D?_$c5Eit;-Y|f
z*b2)1YU-MIda+lQh7rUs3GrF=YNFxb+lNjzaHm#c*E)rnu?}7ehbv$=7lu8
zb8zv0Uhv^C%Pa8VCMBa{ZDJ*c7PsB;)GBnWe
zAj^nST3+kH?>`s4fM=(KS$@z#8YO~@9l-{FFFpasq=nBW>ork0$_?OpeFdEYmZM3-
z>$*$hMT00oZYkK1q}~!dgL-Vj@i!scbA1g
z!bdZr2d=dQl>dA79Xn&m-`?x{a^-U+W2N7+vU%9N>3#fVc>bW;lM56d8$mTvUKr~0
z`MlJlB4(Y{$9=s&jo_oO)M>RecWW!nf<%B)jlSqoFB7f+PbcK~ar@iijz9JB8yds5
zT}pXukl-T0wNWFuv0$lh&HF=5VnUl8zTp0ur8s1Y=h$LLMkKQFO9K>RJ=ROq_rs6|
zwW&6j28Tr|qskv^VS#W+T_&+YOz!jPlst8d!)Ks4#sauhZ-H3O(*~nMc70$_?(XJt
z^>n-tF}L=)niR#%dmdXVP=`ta5eh9atDp*YgaYIVYdhU7L#1s4{s&CF94^XtCX|s3
zo`iocB>F+Wl?~erRu&$VciSTn(9uVb)HY!NMUJu?!M%w$SWR(K2o6YNQ9QE
z=?&CB-U5FP>&E{z9AiEHJ+^iK0(Ca_HN%}2LetLN
zRLiHrRgSyF7EvAR+ntT~aipjZ9ZW8ygAx_xf_ezB(5+@J9Gzn=dhhGw&!qD~gY{H}
z-D&sx8hy`5zFfZc_bt_G2bI$>4}HIjMI`TtZD-JQW8|yZq$8ho$aoNvCEbmn=VlL0lD
z_@by>@Nz}X>FUHvW4iGoyBP=oS1;G=2n)c87q5~liE@OY+2SiW
zr4RV3v!=pWUWGQ3M`Y7}SDW;kzdon&HOVG3^m}#No|o$ciO-16{kR$
zD2a;CY+8D}U8Zzg)Ome1oqpkeFQx5+8b6=e;i&94@Z&Rj+6^iL`OkvG0YM-0qxSdr
z&5scVxf*FcLH)7P!@a#Pguh#lP5Edrsh5~GKAQ7wdBab@_d^p$p@ziF*@Qw{giMFP~_US$R0mZJ=77)~)`f64?#J|#c(X?NRS{+A$
z4Dt^QHfldzc7_O=>~S%{dcPgc`xOn!A~A)!*j$L7^S~q3Idac$zS>bOfxa-;(#6^Q
z_Rj({`m>-ZVg2C|(Fu!QC%Jb=5$&fMgXi?Vq
z{Um@sPd?e?-$cwr#&xZXx!7uGSWmXBles$g%#)2_J$@HUHv5Ltl;)x(zXnH#=eOTWGJVhZPIF|9qF#XVbqb*V
zKQam3jW|E8nl@snb=geNCx_11c=}9%&VuzuJmM;I*=;{n>pdJj*VBovnhqlBZ;sfT
zF41$YpajF76Mn7DaN^1!Xx@NcygIsAj@>?g6}9=6C3p}tF+@S<0%<@J5d0Nquj&yA
zxP9=BM}9jy9n``I-kj608Rm9iki9eIvBrSzI8a&DZfRH9rC-LBXdTzQng+cxD4XuC
z?)9|8Hg)Oyb;YWzG{_L2sp#;x!^8x8^@(4&mKRq|{wP-UaRRpa>_$JbW9)Dm_nBW5
zHEaI8uRDNswP+f!z!@F;-FCi_T0UU`Aj?vC+cBz5K<_dFxUobce)UhRj;fiakT#>B
zsQkV4!JV68Ej4zt+Ua-(qc&20A=YEoUG1+EUoi4d_08-L=e||Z28F}6r27`^VJxyt
zp#P6D!X-{ArliGS#!HD)V@8#n5UEFVkJ)*AK%375kf&6jMIN5drG4y(Bjtw^ihVT=
z`oHUji!J(ZjWSNwdnTIpr4HMafyNPs)d8JtXlyQy$qmpuY)Pv(4>n$9i%Gy&RFivT
zg?Z79s)X*F(5;IBD!5)JE~~EeF-?7G&v{CzZK2|=74ClN13&yMLn&YTCZVIggy-6-v>>}$ivw*_7~Z~5;IdGRWrA$7JrwOKuA1}uNiv-y;oSc@
zUgZJ0$q95xX596tXyO$um7V?%A$3QE{f2XW&wIXYBS{zmA+OU~->S-hH!Ty?f6g_e
zDn_|tw(k7NdW^7E_j~T*o$0EFQQ+#-RjaE@S
zn-7ThQAbr^HSp{BmW9E+Coy+|k3#)b9$;dWRitzN>~by5UjF61{3vXp=9BC+r+EQ)
z6qX025S}NBZcs3kp6^WNE+&c%4$dE9
z?G~%#hJkncxE!NOjco%Tv#r~<8>mv~HL(1^4Nx@-|$QXV!eCR9HkaQsvkb!o%Q@hwX&g7Ip
zr!e#S&C+DPiTm%u!85G#Wh{$T`l8;R#6K`F$Q7gw;Q>ddV9|T-z+3%5em5)IKF%Ko
z)pQ$g!4EU3cMcmnK?*c5QmY{9k(LJhkN(2*M^R|Voe-2K-PJe=KPw7=sK>+ZKwDfy
z@dK}s@u>Ii5xaw-!PGY^p-7MnO;Jk)b`Iz~Mn>cEI50e2nlQv|R~Bs>!e&0)Qv)|e
zeoWEQrV!8GkJljNAu0+A%FhQj*Nfr%Z8qmsu&5X2{jK^^9xR4y^>y$g21$eI0KJte_m#b}O3>Y(SVP`0?nXWE_Khx)H|ZwLIBn_}3#qhw?kPdocUEMnnc#FN=D|x`j2V
z6WJL~JfMmL_RCGf>Pq*#u91_|ibLzF;hFQM!iVul8+9C9k6??9u)JR5l-S&Zkg1;J-pjex
zgYfM}VE=q6fnx;H*xpfq`Y0s65K=3MBJSC;dwVXcd)XMyXJiKw=~2Z_)U%?e))tYH
z4Txla7a#7662qV<0pRVaeAGh?3H^DBqka@({2VU@!#G$G2Y4li{J+A4+!XQOgG`
zCvQR5&SxMb%&(}e^Q3|8@T5SGtUDq`L}@px^2Cu%sh^h@1;zNPL*(YDSM>uQ_)N_C
zsLj3QrNQ2=0I_J_sPb)I2e@S`4QEV|h%#=SH@fI#^pGqFDcBbyZ63n!JIG;;y}2!V
zbc!I(eoaPMeBbFScFPd`0(7v?ciB`m7f9Nvaa!d>H9ZfSuCzT)QM1T>t`bMztOt7inyBwWF_L!Jd4ViBw>b4d12r}=fgqq6;-ju3D}{ha;2q5r
z^5%E9U2P4W7H#AeLD>{$t5PU^Ca=Cor7aDP6sPMsl&_BMQzn>wRvf{>#cN{(lwa|=
z2=jQ4IMFi!`onWmZu7v!arN5ndSGyZ7BpF&SS8Q@l|GpxljFjS@mDe(Xo%_1kz+X8
z&Tsst+0?hg`~1y>wa{Xvx7sOuSqVO$r+%JAdWd(1AlILi|Kb^pkxzYc>W1g!S2KCl
zanKt6mD6IC8F;+uJZg0w6e{jcerqXsZ2^o__U>JC
z6pkzjl>u3b6gYQuLhQbPoBiJ3_>(A>{x?gGbhAwHtP(wtD~x`6DL0ZQlUuQ%@$L*#
zdZOL}2#c?T9>eodiHRLa1N9G;PN>u+=o{7z`lvE?Jr4MQiJbhzZpI@$I%9nKgKgb%
zXVa(gOV~vAM|>Pv->M+XBNmyThueR~rqRq-?*ogMP#IC*UF}gp6bFSk5je_7hk*?X
zK(f;1<7Rrnz=oVS3WDETku7uFc>ouWIS4LbPd%N6b~%U(0SK8LC4cb;EH#Rg8ITh6
zWH=28!|`Dd*44W27i7Qhcz;8D@Q@csR6*=JedGTukenGV*j)!4D`Us=wBq3plL5KJ
zJ7h#`F7#f^>sl?2lcm8We*ld+1=ui11MKT~e5Xeyk
zH_t6?5X@<{{ObsaVoovt`$K2TLoMPM0&S!JQ
zbzgHsWKP0)Mw>R&p90=H&MVT|*|7bF@uu$t>nTFlzE#sv^UL@C(Uw+l2HKiz;7aXp
zgXmwP6&{<`YlmM=8q#?v$T=nch|eYCfZ+jWGO
z?x5~jf^0samh%>7#Kd;9mjr!S?%j8QT3k@aDRDcKl2
zjYVTDiClP)fF$)@-y_NNUh2AC{E$%3@ZiwwB=D{~jR;+Cq1DiARnmVqc)paw{ZOY0wt@yHve|4=d05x~n{Zj5r0^4K@~MPm0}MLBGA!_
zD41prpqSX$*hbeaOqgAsSuFc2n=RHN^%%?u=0i354*YMl
z?l(poLR-FE)KycQJv}%V8Fz4bo;^LXVOq;{DK<{6eRUWcA?vXh4)MEIqQbyGeZuOI
za-{n>SXVLs{JjZy9{g6fuB}3i5;cP{;AIk
zW@@8jFfu=Y&nzH4bpM|^ji?P)YF5RKg)g4pb_0Q@-M;e6yjZ3lxcH_rB-I{9HO)5U
z`L~LB^TzQtgAptJ%tt>nvB>8Oa=I$1smkj5xBNXryM`pGF2BXt`{UJ6zmtyTry4`$
zHA-BwC3u!&)zx=KC3hY%%1opBLY&{&?*2~0UmB7SAQ
z#9?C3`|()
z`1{vS81|qyBGYU~OH0ksXm_uUXVB~s2q7_
zSUHJCfzA3)#Y^O}*Xf<|9FrxhHhvMc)++%>kSH{aWCuZ#%1;*Xri-lc=4PbJkYDG;
z9PP8^!=9yUK0}EzDKFqDBlkw@=5>k2BC;eCjx{76-mHa4FeSXCSs_zM5gI%UYQ78q
z&cbl#r@JzBi@j&FeDz`RHw<}*0&&
zZT9I)+ZOr<=zA{J@_(oQnDfXR%Xq&f0DmI|rI?}{nXV$d4K4VN8y~pYd-mHQ|>n7jvi2t-_mDTQd*4^ow7sTiTJ;!gYBmr`?UF17z-^bu@PPp
zfso)JC$NQp*6v@G)`)F*JE4Lfr*>ceq9)#NnQ^?xHoO$Z&P+aKvYvgTO(#|BHFaes
zYa;NS@>aB}QWj7o@hx0OYrbOC(--dim#F0k`tWyAXbjHELh#l_^sNa(F;ryP;R`hf
zAHrrN{Ee8av%?bYj;Tx0PHTFfZgH327Ij@u&s2pqp|`=b(na4rH~ma(u&4MLOihRZ
zeg-~tX6QJL@cnd4N=Oj9tt_`gQDFO;vZ#JIT+=l64NYK()Iys58Ja#E;z=Bkef4}b
zBkw(0C6}vnv&PGt;ep+r3{WB@Jj1BlWzzaLT1PwYeQl3~@6o@}GUnmq)L>)_madqx
z^&xRS=lq#xv%a$_CTftb>JvOaw`Qph=s+qO!r(C4KgPp2c%t&1q5Mw!FIM)JKPo|io?z!QL78MQV_HjUW+&Iw3bbBFbg_0sXZja?&^dQ|MBQi8yR>*
zOhVWNgd;$70(Bk}#R{23NvG8wmUGfF91+*8Z=g4{tl#Bo9`0jx>APEvCWopO
zfbb>K%BiCW1HHP=dv5RNN2_S@r`uIsApTxG>^yzrCP$~}BR5W-BIxM^Ai77U*JrcZ
zL-NR0J91?$~}>dIHT(7ig%JbjLSl;K_!wV)Z)qUF|k
z#U-niEX|)kyQ>18{|uC-sE7L)ZYE7pfC6M>nND{%2t-0&3Bamp%7b;r_3a(IM>l8|
zYqr+snefz}jEJ|qFife35_+9ki#nSSE296{tY?OeS`t0M>(shC2;piuOBGZFCL69F
zwH(Ib&?|ykco2X#)qr^_6Tl1;$%Odj%JkD$hq5zQ21QPLH)tET9`35y{Va@&n*pY<
ziirY{mp<3K7+;Or2tZ{FXLF_|_~z!gtRr>trLVHO{T+d}0_TJ8h?}ZEIjHu@*KU3p
zKKi$vZ;ARIJMAp)5dIfyjK%}vPUn1t^V}t;%P8P$plmiX)%=G817f4_Ptj;PYwJyd
zRpA;PWZ$o=k@XL>11tjj$&Jpb{hzsW^JuQtJ$M>&t1$O9|HELQocMVB{=yiyHKyiv
zi1t@cOl9Mhtsy5jtI=g1f-=kPEeYR;{EBd{eKkxZETSv928mU{eb~61d
z7=9#r-l3IR?}dqY7Gw?-x$WWOOcsIBo4Gmkr&3iiAS`x=le5Mv32VJFzo^R`+;7U{
zL=38Ref9nw6om8%=>q4LSW?AvUzQUKc7sz}gl1c8Uf`wWb8~3T=X(`(SN2pLPJeH@BtHDF
zSr%mg`;eHH!|wz8&l$9Y9+(|#DV_0>YW#ytgYY#%8>9!3sVfvS-jQ;s7Qz@VhdEU|ITxY@WG5Y>a4R8-8kd^^!aBm6dy+aTUT{Jb+)OTuQ#&=PVsq$
zB2j1;K5OsX%>5gEJo9`mM*!o5PdP4CVjHLa!1WIv4K6gc~oH~?N#%F-z
zM032(7QddkZ+OJe?451UWVH2e0XNKXaZIDI`*67_HE($G>#T)abA)n}pY6*?VG3Z;
z`LgVF{?=-oA}W60J_+Ho*>{{~FHFK}wQIT!{L>Qwr|>-s;19Hf7=EpbzlP-_3bNG{
zz2)Zi?|)9tS5B_HOx;=rqLo@?o9Rl0Ha}90vRtK6Q4QAG_eOu^DX&vuoLOm>-VCb!
z>v)#-foGRH%U@8^mG_~5<27+uD_MaN{UZH=M!8g%@sxKW2PL-bp7>)5Dw!Q^&2<$3
z=~AI?mAKrigEJo=&J}rN0;|4|4HN2HX``i3M7^RS+C_WImB6Z}J?owc+^@VCb$CYK
zqsE4;_KI+HL<60_wd&@#sSdOCES>$kEy;9CIGJo<)W&AuS2IhKREgKDX8g0DEb0q9
zB8I`bdGGuHeSI#_>*~M*-iQ{4`as<`0ib3DBrq!;La}9#C
zmX|KKR}Y*{HaMCj9Jwk?Iv?_R9K_8#xvk%O{~q)EUVWQgIrIeL&