feat: improve dev commands, update all langchain dependencies to their latest major versions

Makefile

@@ -1,6 +1,6 @@
 .PHONY: run frontend check ruff database lint api start-all stop-all status clean-cache worker worker-start worker-stop worker-restart
 .PHONY: docker-buildx-prepare docker-buildx-clean docker-buildx-reset
-.PHONY: docker-push docker-push-latest docker-release tag export-docs
+.PHONY: docker-push docker-push-latest docker-release docker-build-local tag export-docs
 
 # Get version from pyproject.toml
 VERSION := $(shell grep -m1 version pyproject.toml | cut -d'"' -f2)
@@ -45,6 +45,16 @@ docker-buildx-reset: docker-buildx-clean docker-buildx-prepare
 
 # === Docker Build Targets ===
 
+# Build production image for local platform only (no push)
+docker-build-local:
+	@echo "🔨 Building production image locally ($(shell uname -m))..."
+	docker build \
+		-t $(DOCKERHUB_IMAGE):$(VERSION) \
+		-t $(DOCKERHUB_IMAGE):local \
+		.
+	@echo "✅ Built $(DOCKERHUB_IMAGE):$(VERSION) and $(DOCKERHUB_IMAGE):local"
+	@echo "Run with: docker run -p 5055:5055 -p 3000:3000 $(DOCKERHUB_IMAGE):local"
+
 # Build and push version tags ONLY (no latest) for both regular and single images
 docker-push: docker-buildx-prepare
 	@echo "📤 Building and pushing version $(VERSION) to both registries..."

README.dev.md

@@ -0,0 +1,408 @@
+# Developer Guide
+
+This guide is for developers working on Open Notebook. For end-user documentation, see [README.md](README.md) and [docs/](docs/).
+
+## Quick Start for Development
+
+```bash
+# 1. Clone and setup
+git clone https://github.com/lfnovo/open-notebook.git
+cd open-notebook
+
+# 2. Copy environment files
+cp .env.example .env
+cp .env.example docker.env
+
+# 3. Install dependencies
+uv sync
+
+# 4. Start all services (recommended for development)
+make start-all
+```
+
+## Development Workflows
+
+### When to Use What?
+
+| Workflow | Use Case | Speed | Production Parity |
+|----------|----------|-------|-------------------|
+| **Local Services** (`make start-all`) | Day-to-day development, fastest iteration | ⚡⚡⚡ Fast | Medium |
+| **Docker Compose** (`make dev`) | Testing containerized setup | ⚡⚡ Medium | High |
+| **Local Docker Build** (`make docker-build-local`) | Testing Dockerfile changes | ⚡ Slow | Very High |
+| **Multi-platform Build** (`make docker-push`) | Publishing releases | 🐌 Very Slow | Exact |
+
+---
+
+## 1. Local Development (Recommended)
+
+**Best for:** Daily development, hot reload, debugging
+
+### Setup
+
+```bash
+# Start database
+make database
+
+# Start all services (DB + API + Worker + Frontend)
+make start-all
+```
+
+### What This Does
+
+1. Starts SurrealDB in Docker (port 8000)
+2. Starts FastAPI backend (port 5055)
+3. Starts background worker (surreal-commands)
+4. Starts Next.js frontend (port 3000)
+
+### Individual Services
+
+```bash
+# Just the database
+make database
+
+# Just the API
+make api
+
+# Just the frontend
+make frontend
+
+# Just the worker
+make worker
+```
+
+### Checking Status
+
+```bash
+# See what's running
+make status
+
+# Stop everything
+make stop-all
+```
+
+### Advantages
+- ✅ Fastest iteration (hot reload)
+- ✅ Easy debugging (direct process access)
+- ✅ Low resource usage
+- ✅ Direct log access
+
+### Disadvantages
+- ❌ Doesn't test Docker build
+- ❌ Environment may differ from production
+- ❌ Requires local Python/Node setup
+
+---
+
+## 2. Docker Compose Development
+
+**Best for:** Testing containerized setup, CI/CD verification
+
+```bash
+# Start with dev profile
+make dev
+
+# Or full stack
+make full
+```
+
+### Configuration Files
+
+- `docker-compose.dev.yml` - Development setup
+- `docker-compose.full.yml` - Full stack setup
+- `docker-compose.yml` - Base configuration
+
+### Advantages
+- ✅ Closer to production environment
+- ✅ Isolated dependencies
+- ✅ Easy to share exact environment
+
+### Disadvantages
+- ❌ Slower rebuilds
+- ❌ More complex debugging
+- ❌ Higher resource usage
+
+---
+
+## 3. Testing Production Docker Images
+
+**Best for:** Verifying Dockerfile changes before publishing
+
+### Build Locally
+
+```bash
+# Build production image for your platform only
+make docker-build-local
+```
+
+This creates two tags:
+- `lfnovo/open_notebook:<version>` (from pyproject.toml)
+- `lfnovo/open_notebook:local`
+
+### Run Locally
+
+```bash
+docker run -p 5055:5055 -p 3000:3000 lfnovo/open_notebook:local
+```
+
+### When to Use
+- ✅ Before pushing to registry
+- ✅ Testing Dockerfile changes
+- ✅ Debugging production-specific issues
+- ✅ Verifying build process
+
+---
+
+## 4. Publishing Docker Images
+
+### Workflow
+
+```bash
+# 1. Test locally first
+make docker-build-local
+
+# 2. If successful, push version tag (no latest update)
+make docker-push
+
+# 3. Test the pushed version in staging/production
+
+# 4. When ready, promote to latest
+make docker-push-latest
+```
+
+### Available Commands
+
+| Command | What It Does | Updates Latest? |
+|---------|--------------|-----------------|
+| `make docker-build-local` | Build for current platform only | No registry push |
+| `make docker-push` | Push version tags to registries | ❌ No |
+| `make docker-push-latest` | Push version + update v1-latest | ✅ Yes |
+| `make docker-release` | Full release (same as docker-push-latest) | ✅ Yes |
+
+### Publishing Details
+
+- **Platforms:** `linux/amd64`, `linux/arm64`
+- **Registries:** Docker Hub + GitHub Container Registry
+- **Image Variants:** Regular + Single-container (`-single`)
+- **Version Source:** `pyproject.toml`
+
+### Creating Git Tags
+
+```bash
+# Create and push git tag matching pyproject.toml version
+make tag
+```
+
+---
+
+## Code Quality
+
+```bash
+# Run linter with auto-fix
+make ruff
+
+# Run type checking
+make lint
+
+# Run tests
+uv run pytest tests/
+
+# Clean cache directories
+make clean-cache
+```
+
+---
+
+## Common Development Tasks
+
+### Adding a New Feature
+
+1. Create feature branch
+2. Develop using `make start-all`
+3. Write tests
+4. Run `make ruff` and `make lint`
+5. Test with `make docker-build-local`
+6. Create PR
+
+### Fixing a Bug
+
+1. Reproduce locally with `make start-all`
+2. Add test case demonstrating bug
+3. Fix the bug
+4. Verify test passes
+5. Check with `make docker-build-local`
+
+### Updating Dependencies
+
+```bash
+# Add Python dependency
+uv add package-name
+
+# Update dependencies
+uv sync
+
+# Frontend dependencies
+cd frontend && npm install package-name
+```
+
+### Database Migrations
+
+Database migrations run **automatically** when the API starts.
+
+1. Create migration file: `migrations/XXX_description.surql`
+2. Write SurrealQL schema changes
+3. (Optional) Create rollback: `migrations/XXX_description_down.surql`
+4. Restart API - migration runs on startup
+
+---
+
+## Troubleshooting
+
+### Services Won't Start
+
+```bash
+# Check status
+make status
+
+# Check database
+docker compose ps surrealdb
+
+# View logs
+docker compose logs surrealdb
+
+# Restart everything
+make stop-all
+make start-all
+```
+
+### Port Already in Use
+
+```bash
+# Find process using port
+lsof -i :5055
+lsof -i :3000
+lsof -i :8000
+
+# Kill stuck processes
+make stop-all
+```
+
+### Database Connection Issues
+
+```bash
+# Verify SurrealDB is running
+docker compose ps surrealdb
+
+# Check connection settings in .env
+cat .env | grep SURREAL
+```
+
+### Docker Build Fails
+
+```bash
+# Clean Docker cache
+docker builder prune
+
+# Reset buildx
+make docker-buildx-reset
+
+# Try local build first
+make docker-build-local
+```
+
+---
+
+## Project Structure
+
+```
+open-notebook/
+├── api/                    # FastAPI backend
+├── frontend/               # Next.js React frontend
+├── open_notebook/          # Python core library
+│   ├── domain/            # Domain models
+│   ├── graphs/            # LangGraph workflows
+│   ├── ai/                # AI provider integration
+│   └── database/          # SurrealDB operations
+├── migrations/             # Database migrations
+├── tests/                  # Test suite
+├── docs/                   # User documentation
+└── Makefile               # Development commands
+```
+
+See component-specific CLAUDE.md files for detailed architecture:
+- [frontend/CLAUDE.md](frontend/CLAUDE.md)
+- [api/CLAUDE.md](api/CLAUDE.md)
+- [open_notebook/CLAUDE.md](open_notebook/CLAUDE.md)
+
+---
+
+## Environment Variables
+
+### Required for Local Development
+
+```bash
+# .env file
+SURREAL_URL=ws://localhost:8000
+SURREAL_USER=root
+SURREAL_PASS=root
+SURREAL_DB=open_notebook
+SURREAL_NS=production
+
+# AI Provider (at least one required)
+OPENAI_API_KEY=sk-...
+# OR
+ANTHROPIC_API_KEY=sk-ant-...
+# OR configure other providers (see docs/5-CONFIGURATION/)
+```
+
+See [docs/5-CONFIGURATION/](docs/5-CONFIGURATION/) for complete configuration guide.
+
+---
+
+## Performance Tips
+
+### Speed Up Local Development
+
+1. **Use `make start-all`** instead of Docker for daily work
+2. **Keep SurrealDB running** between sessions (`make database`)
+3. **Use `make docker-build-local`** only when testing Dockerfile changes
+4. **Skip multi-platform builds** until ready to publish
+
+### Reduce Resource Usage
+
+```bash
+# Stop unused services
+make stop-all
+
+# Clean up Docker
+docker system prune -a
+
+# Clean Python cache
+make clean-cache
+```
+
+---
+
+## TODO: Sections to Add
+
+- [ ] Frontend development guide (hot reload, component structure)
+- [ ] API development guide (adding endpoints, services)
+- [ ] LangGraph workflow development
+- [ ] Testing strategy and coverage
+- [ ] Debugging tips (VSCode/PyCharm setup)
+- [ ] CI/CD pipeline overview
+- [ ] Release process checklist
+- [ ] Common error messages and solutions
+
+---
+
+## Resources
+
+- **Documentation:** https://open-notebook.ai
+- **Discord:** https://discord.gg/37XJPXfz2w
+- **Issues:** https://github.com/lfnovo/open-notebook/issues
+- **Contributing:** [CONTRIBUTING.md](CONTRIBUTING.md)
+- **Maintainer Guide:** [MAINTAINER_GUIDE.md](MAINTAINER_GUIDE.md)
+
+---
+
+**Last Updated:** January 2025

README.md

@@ -195,19 +195,6 @@ Thanks to the [Esperanto](https://github.com/lfnovo/esperanto) library, we suppo
 
 ## Podcast Feature
 
-### Audio 1
-<audio controls>
-  <source src="audio1.mp3" type="audio/mpeg">
-  Your browser does not support the audio element.
-</audio>
-
-### Audio 2
-<audio controls>
-  <source src="audio2.mp3" type="audio/mpeg">
-  Your browser does not support the audio element.
-</audio>
-
-
 [![Check out our podcast sample](https://img.youtube.com/vi/D-760MlGwaI/0.jpg)](https://www.youtube.com/watch?v=D-760MlGwaI)
 
 ## 📚 Documentation

open_notebook/domain/base.py

@@ -2,7 +2,7 @@ from datetime import datetime
 from typing import Any, ClassVar, Dict, List, Optional, Type, TypeVar, Union, cast
 
 from loguru import logger
-from pydantic import BaseModel, ValidationError, field_validator, model_validator
+from pydantic import BaseModel, ConfigDict, ValidationError, field_validator, model_validator
 
 from open_notebook.database.repository import (
     ensure_record_id,
@@ -211,19 +211,20 @@ class ObjectModel(BaseModel):
 
 
 class RecordModel(BaseModel):
+    model_config = ConfigDict(
+        validate_assignment=True,
+        arbitrary_types_allowed=True,
+        extra="allow",
+        from_attributes=True,
+        defer_build=True,
+    )
+
     record_id: ClassVar[str]
     auto_save: ClassVar[bool] = (
         False  # Default to False, can be overridden in subclasses
     )
     _instances: ClassVar[Dict[str, "RecordModel"]] = {}  # Store instances by record_id
 
-    class Config:
-        validate_assignment = True
-        arbitrary_types_allowed = True
-        extra = "allow"
-        from_attributes = True
-        defer_build = True
-
     def __new__(cls, **kwargs):
         # If an instance already exists for this record_id, return it
         if cls.record_id in cls._instances:

open_notebook/domain/notebook.py

@@ -2,7 +2,7 @@ import asyncio
 from typing import Any, ClassVar, Dict, List, Literal, Optional, Tuple, Union
 
 from loguru import logger
-from pydantic import BaseModel, Field, field_validator
+from pydantic import BaseModel, ConfigDict, Field, field_validator
 from surreal_commands import submit_command
 from surrealdb import RecordID
 
@@ -142,6 +142,8 @@ class SourceInsight(ObjectModel):
 
 
 class Source(ObjectModel):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
     table_name: ClassVar[str] = "source"
     asset: Optional[Asset] = None
     title: Optional[str] = None
@@ -151,9 +153,6 @@ class Source(ObjectModel):
         default=None, description="Link to surreal-commands processing job"
     )
 
-    class Config:
-        arbitrary_types_allowed = True
-
     @field_validator("command", mode="before")
     @classmethod
     def parse_command(cls, value):

open_notebook/podcasts/models.py

@@ -1,6 +1,6 @@
 from typing import Any, ClassVar, Dict, List, Optional, Union
 
-from pydantic import Field, field_validator
+from pydantic import ConfigDict, Field, field_validator
 from surrealdb import RecordID
 
 from open_notebook.database.repository import ensure_record_id, repo_query
@@ -114,8 +114,7 @@ class PodcastEpisode(ObjectModel):
         default=None, description="Link to surreal-commands job"
     )
 
-    class Config:
+    model_config = ConfigDict(arbitrary_types_allowed=True)
-        arbitrary_types_allowed = True
 
     async def get_job_status(self) -> Optional[str]:
         """Get the status of the associated command"""

pyproject.toml

@@ -17,25 +17,25 @@ dependencies = [
     "uvicorn>=0.24.0",
     "pydantic>=2.9.2",
     "loguru>=0.7.2",
-    "langchain>=0.3.3",
+    "langchain>=1.2.0",
-    "langgraph>=0.2.38",
+    "langgraph>=1.0.5",
-    "tiktoken>=0.8.0",
+    "tiktoken>=0.12.0",
-    "langgraph-checkpoint-sqlite>=2.0.0",
+    "langgraph-checkpoint-sqlite>=3.0.1",
-    "langchain-community>=0.3.3",
+    "langchain-community>=0.4.1",
-    "langchain-openai>=0.2.3",
+    "langchain-openai>=1.1.6",
-    "langchain-anthropic>=0.2.3",
+    "langchain-anthropic>=1.3.0",
-    "langchain-ollama>=0.2.0",
+    "langchain-ollama>=1.0.1",
-    "langchain-google-genai>=2.1.10",
+    "langchain-google-genai>=4.1.2",
-    "langchain-groq>=0.2.1",
+    "langchain-groq>=1.1.1",
-    "langchain_mistralai>=0.2.1",
+    "langchain_mistralai>=1.1.1",
-    "langchain_deepseek>=0.1.3",
+    "langchain_deepseek>=1.0.0",
-    "langchain-google-vertexai>=2.0.28",
+    "langchain-google-vertexai>=3.2.0",
     "tomli>=2.0.2",
     "python-dotenv>=1.0.1",
     "httpx[socks]>=0.27.0",
     "content-core>=1.0.2",
     "ai-prompter>=0.3",
-    "esperanto>=2.8.3",
+    "esperanto>=2.13",
     "surrealdb>=1.0.4",
     "podcast-creator>=0.7.0",
     "surreal-commands>=1.2.0",

uv.lock

@@ -671,15 +671,15 @@ wheels = [
 
 [[package]]
 name = "esperanto"
-version = "2.12.1"
+version = "2.13.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "httpx" },
     { name = "pydantic" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/7f/7d/856ecb7ab6b05fa212ef4cc9186be5008373aeced2dc1ebc99013c96fa3c/esperanto-2.12.1.tar.gz", hash = "sha256:177f001363d7710a7bdf747adc2c3f5e00aebd6b759c3ac6642e2c8df8a4e1cf", size = 811783, upload-time = "2025-12-16T00:53:46.798Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/b5/8c/9f655703422fc895f4c327316a8b4c824ec334374a8f6e2dea61f2512362/esperanto-2.13.0.tar.gz", hash = "sha256:78df58492700d4cdfe9dd715313c48e5e4b816ecb87dc22a56d03610431c640e", size = 742118, upload-time = "2026-01-04T22:14:24.996Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/65/48/5d8c6bc2b5db29f3b8f8be7c0964f8d78bd40b3586133f71b773530f67af/esperanto-2.12.1-py3-none-any.whl", hash = "sha256:341c239d3c7a14d556a3b4225c6e1b5f66d635e34ec8026117fc18f4f608b3c3", size = 152050, upload-time = "2025-12-16T00:53:48.023Z" },
+    { url = "https://files.pythonhosted.org/packages/8c/44/e1c9aa604f252a351d67398c886bb9b1cd572881289864d3e0d45212e61d/esperanto-2.13.0-py3-none-any.whl", hash = "sha256:5b9d12eb3d03f63acb7a77c0c17ed90b32b761ded1f122bc44bb4e8b6625cec0", size = 152019, upload-time = "2026-01-04T22:14:24.031Z" },
 ]
 
 [[package]]
@@ -2439,23 +2439,23 @@ dev = [
 requires-dist = [
     { name = "ai-prompter", specifier = ">=0.3" },
     { name = "content-core", specifier = ">=1.0.2" },
-    { name = "esperanto", specifier = ">=2.8.3" },
+    { name = "esperanto", specifier = ">=2.13" },
     { name = "fastapi", specifier = ">=0.104.0" },
     { name = "httpx", extras = ["socks"], specifier = ">=0.27.0" },
     { name = "ipykernel", marker = "extra == 'dev'", specifier = ">=6.29.5" },
     { name = "ipywidgets", marker = "extra == 'dev'", specifier = ">=8.1.5" },
-    { name = "langchain", specifier = ">=0.3.3" },
+    { name = "langchain", specifier = ">=1.2.0" },
-    { name = "langchain-anthropic", specifier = ">=0.2.3" },
+    { name = "langchain-anthropic", specifier = ">=1.3.0" },
-    { name = "langchain-community", specifier = ">=0.3.3" },
+    { name = "langchain-community", specifier = ">=0.4.1" },
-    { name = "langchain-deepseek", specifier = ">=0.1.3" },
+    { name = "langchain-deepseek", specifier = ">=1.0.0" },
-    { name = "langchain-google-genai", specifier = ">=2.1.10" },
+    { name = "langchain-google-genai", specifier = ">=4.1.2" },
-    { name = "langchain-google-vertexai", specifier = ">=2.0.28" },
+    { name = "langchain-google-vertexai", specifier = ">=3.2.0" },
-    { name = "langchain-groq", specifier = ">=0.2.1" },
+    { name = "langchain-groq", specifier = ">=1.1.1" },
-    { name = "langchain-mistralai", specifier = ">=0.2.1" },
+    { name = "langchain-mistralai", specifier = ">=1.1.1" },
-    { name = "langchain-ollama", specifier = ">=0.2.0" },
+    { name = "langchain-ollama", specifier = ">=1.0.1" },
-    { name = "langchain-openai", specifier = ">=0.2.3" },
+    { name = "langchain-openai", specifier = ">=1.1.6" },
-    { name = "langgraph", specifier = ">=0.2.38" },
+    { name = "langgraph", specifier = ">=1.0.5" },
-    { name = "langgraph-checkpoint-sqlite", specifier = ">=2.0.0" },
+    { name = "langgraph-checkpoint-sqlite", specifier = ">=3.0.1" },
     { name = "loguru", specifier = ">=0.7.2" },
     { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.11.1" },
     { name = "podcast-creator", specifier = ">=0.7.0" },
@@ -2466,7 +2466,7 @@ requires-dist = [
     { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.5.5" },
     { name = "surreal-commands", specifier = ">=1.2.0" },
     { name = "surrealdb", specifier = ">=1.0.4" },
-    { name = "tiktoken", specifier = ">=0.8.0" },
+    { name = "tiktoken", specifier = ">=0.12.0" },
     { name = "tomli", specifier = ">=2.0.2" },
     { name = "types-requests", marker = "extra == 'dev'", specifier = ">=2.32.0.20241016" },
     { name = "uvicorn", specifier = ">=0.24.0" },