From b1bd522c5c1ba6994eb24c60d06010f4b50f06d2 Mon Sep 17 00:00:00 2001 From: LUIS NOVO Date: Mon, 5 Jan 2026 08:22:41 -0300 Subject: [PATCH] feat: improve dev commands, update all langchain dependencies to their latest major versions --- Makefile | 12 +- README.dev.md | 408 +++++++++++++++++++++++++++++++ README.md | 13 - open_notebook/domain/base.py | 17 +- open_notebook/domain/notebook.py | 7 +- open_notebook/podcasts/models.py | 5 +- pyproject.toml | 28 +-- uv.lock | 34 +-- 8 files changed, 464 insertions(+), 60 deletions(-) create mode 100644 README.dev.md diff --git a/Makefile b/Makefile index 69a5a96..b71b4da 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .PHONY: run frontend check ruff database lint api start-all stop-all status clean-cache worker worker-start worker-stop worker-restart .PHONY: docker-buildx-prepare docker-buildx-clean docker-buildx-reset -.PHONY: docker-push docker-push-latest docker-release tag export-docs +.PHONY: docker-push docker-push-latest docker-release docker-build-local tag export-docs # Get version from pyproject.toml VERSION := $(shell grep -m1 version pyproject.toml | cut -d'"' -f2) @@ -45,6 +45,16 @@ docker-buildx-reset: docker-buildx-clean docker-buildx-prepare # === Docker Build Targets === +# Build production image for local platform only (no push) +docker-build-local: + @echo "๐Ÿ”จ Building production image locally ($(shell uname -m))..." + docker build \ + -t $(DOCKERHUB_IMAGE):$(VERSION) \ + -t $(DOCKERHUB_IMAGE):local \ + . + @echo "โœ… Built $(DOCKERHUB_IMAGE):$(VERSION) and $(DOCKERHUB_IMAGE):local" + @echo "Run with: docker run -p 5055:5055 -p 3000:3000 $(DOCKERHUB_IMAGE):local" + # Build and push version tags ONLY (no latest) for both regular and single images docker-push: docker-buildx-prepare @echo "๐Ÿ“ค Building and pushing version $(VERSION) to both registries..." diff --git a/README.dev.md b/README.dev.md new file mode 100644 index 0000000..1c7d27e --- /dev/null +++ b/README.dev.md @@ -0,0 +1,408 @@ +# Developer Guide + +This guide is for developers working on Open Notebook. For end-user documentation, see [README.md](README.md) and [docs/](docs/). + +## Quick Start for Development + +```bash +# 1. Clone and setup +git clone https://github.com/lfnovo/open-notebook.git +cd open-notebook + +# 2. Copy environment files +cp .env.example .env +cp .env.example docker.env + +# 3. Install dependencies +uv sync + +# 4. Start all services (recommended for development) +make start-all +``` + +## Development Workflows + +### When to Use What? + +| Workflow | Use Case | Speed | Production Parity | +|----------|----------|-------|-------------------| +| **Local Services** (`make start-all`) | Day-to-day development, fastest iteration | โšกโšกโšก Fast | Medium | +| **Docker Compose** (`make dev`) | Testing containerized setup | โšกโšก Medium | High | +| **Local Docker Build** (`make docker-build-local`) | Testing Dockerfile changes | โšก Slow | Very High | +| **Multi-platform Build** (`make docker-push`) | Publishing releases | ๐ŸŒ Very Slow | Exact | + +--- + +## 1. Local Development (Recommended) + +**Best for:** Daily development, hot reload, debugging + +### Setup + +```bash +# Start database +make database + +# Start all services (DB + API + Worker + Frontend) +make start-all +``` + +### What This Does + +1. Starts SurrealDB in Docker (port 8000) +2. Starts FastAPI backend (port 5055) +3. Starts background worker (surreal-commands) +4. Starts Next.js frontend (port 3000) + +### Individual Services + +```bash +# Just the database +make database + +# Just the API +make api + +# Just the frontend +make frontend + +# Just the worker +make worker +``` + +### Checking Status + +```bash +# See what's running +make status + +# Stop everything +make stop-all +``` + +### Advantages +- โœ… Fastest iteration (hot reload) +- โœ… Easy debugging (direct process access) +- โœ… Low resource usage +- โœ… Direct log access + +### Disadvantages +- โŒ Doesn't test Docker build +- โŒ Environment may differ from production +- โŒ Requires local Python/Node setup + +--- + +## 2. Docker Compose Development + +**Best for:** Testing containerized setup, CI/CD verification + +```bash +# Start with dev profile +make dev + +# Or full stack +make full +``` + +### Configuration Files + +- `docker-compose.dev.yml` - Development setup +- `docker-compose.full.yml` - Full stack setup +- `docker-compose.yml` - Base configuration + +### Advantages +- โœ… Closer to production environment +- โœ… Isolated dependencies +- โœ… Easy to share exact environment + +### Disadvantages +- โŒ Slower rebuilds +- โŒ More complex debugging +- โŒ Higher resource usage + +--- + +## 3. Testing Production Docker Images + +**Best for:** Verifying Dockerfile changes before publishing + +### Build Locally + +```bash +# Build production image for your platform only +make docker-build-local +``` + +This creates two tags: +- `lfnovo/open_notebook:` (from pyproject.toml) +- `lfnovo/open_notebook:local` + +### Run Locally + +```bash +docker run -p 5055:5055 -p 3000:3000 lfnovo/open_notebook:local +``` + +### When to Use +- โœ… Before pushing to registry +- โœ… Testing Dockerfile changes +- โœ… Debugging production-specific issues +- โœ… Verifying build process + +--- + +## 4. Publishing Docker Images + +### Workflow + +```bash +# 1. Test locally first +make docker-build-local + +# 2. If successful, push version tag (no latest update) +make docker-push + +# 3. Test the pushed version in staging/production + +# 4. When ready, promote to latest +make docker-push-latest +``` + +### Available Commands + +| Command | What It Does | Updates Latest? | +|---------|--------------|-----------------| +| `make docker-build-local` | Build for current platform only | No registry push | +| `make docker-push` | Push version tags to registries | โŒ No | +| `make docker-push-latest` | Push version + update v1-latest | โœ… Yes | +| `make docker-release` | Full release (same as docker-push-latest) | โœ… Yes | + +### Publishing Details + +- **Platforms:** `linux/amd64`, `linux/arm64` +- **Registries:** Docker Hub + GitHub Container Registry +- **Image Variants:** Regular + Single-container (`-single`) +- **Version Source:** `pyproject.toml` + +### Creating Git Tags + +```bash +# Create and push git tag matching pyproject.toml version +make tag +``` + +--- + +## Code Quality + +```bash +# Run linter with auto-fix +make ruff + +# Run type checking +make lint + +# Run tests +uv run pytest tests/ + +# Clean cache directories +make clean-cache +``` + +--- + +## Common Development Tasks + +### Adding a New Feature + +1. Create feature branch +2. Develop using `make start-all` +3. Write tests +4. Run `make ruff` and `make lint` +5. Test with `make docker-build-local` +6. Create PR + +### Fixing a Bug + +1. Reproduce locally with `make start-all` +2. Add test case demonstrating bug +3. Fix the bug +4. Verify test passes +5. Check with `make docker-build-local` + +### Updating Dependencies + +```bash +# Add Python dependency +uv add package-name + +# Update dependencies +uv sync + +# Frontend dependencies +cd frontend && npm install package-name +``` + +### Database Migrations + +Database migrations run **automatically** when the API starts. + +1. Create migration file: `migrations/XXX_description.surql` +2. Write SurrealQL schema changes +3. (Optional) Create rollback: `migrations/XXX_description_down.surql` +4. Restart API - migration runs on startup + +--- + +## Troubleshooting + +### Services Won't Start + +```bash +# Check status +make status + +# Check database +docker compose ps surrealdb + +# View logs +docker compose logs surrealdb + +# Restart everything +make stop-all +make start-all +``` + +### Port Already in Use + +```bash +# Find process using port +lsof -i :5055 +lsof -i :3000 +lsof -i :8000 + +# Kill stuck processes +make stop-all +``` + +### Database Connection Issues + +```bash +# Verify SurrealDB is running +docker compose ps surrealdb + +# Check connection settings in .env +cat .env | grep SURREAL +``` + +### Docker Build Fails + +```bash +# Clean Docker cache +docker builder prune + +# Reset buildx +make docker-buildx-reset + +# Try local build first +make docker-build-local +``` + +--- + +## Project Structure + +``` +open-notebook/ +โ”œโ”€โ”€ api/ # FastAPI backend +โ”œโ”€โ”€ frontend/ # Next.js React frontend +โ”œโ”€โ”€ open_notebook/ # Python core library +โ”‚ โ”œโ”€โ”€ domain/ # Domain models +โ”‚ โ”œโ”€โ”€ graphs/ # LangGraph workflows +โ”‚ โ”œโ”€โ”€ ai/ # AI provider integration +โ”‚ โ””โ”€โ”€ database/ # SurrealDB operations +โ”œโ”€โ”€ migrations/ # Database migrations +โ”œโ”€โ”€ tests/ # Test suite +โ”œโ”€โ”€ docs/ # User documentation +โ””โ”€โ”€ Makefile # Development commands +``` + +See component-specific CLAUDE.md files for detailed architecture: +- [frontend/CLAUDE.md](frontend/CLAUDE.md) +- [api/CLAUDE.md](api/CLAUDE.md) +- [open_notebook/CLAUDE.md](open_notebook/CLAUDE.md) + +--- + +## Environment Variables + +### Required for Local Development + +```bash +# .env file +SURREAL_URL=ws://localhost:8000 +SURREAL_USER=root +SURREAL_PASS=root +SURREAL_DB=open_notebook +SURREAL_NS=production + +# AI Provider (at least one required) +OPENAI_API_KEY=sk-... +# OR +ANTHROPIC_API_KEY=sk-ant-... +# OR configure other providers (see docs/5-CONFIGURATION/) +``` + +See [docs/5-CONFIGURATION/](docs/5-CONFIGURATION/) for complete configuration guide. + +--- + +## Performance Tips + +### Speed Up Local Development + +1. **Use `make start-all`** instead of Docker for daily work +2. **Keep SurrealDB running** between sessions (`make database`) +3. **Use `make docker-build-local`** only when testing Dockerfile changes +4. **Skip multi-platform builds** until ready to publish + +### Reduce Resource Usage + +```bash +# Stop unused services +make stop-all + +# Clean up Docker +docker system prune -a + +# Clean Python cache +make clean-cache +``` + +--- + +## TODO: Sections to Add + +- [ ] Frontend development guide (hot reload, component structure) +- [ ] API development guide (adding endpoints, services) +- [ ] LangGraph workflow development +- [ ] Testing strategy and coverage +- [ ] Debugging tips (VSCode/PyCharm setup) +- [ ] CI/CD pipeline overview +- [ ] Release process checklist +- [ ] Common error messages and solutions + +--- + +## Resources + +- **Documentation:** https://open-notebook.ai +- **Discord:** https://discord.gg/37XJPXfz2w +- **Issues:** https://github.com/lfnovo/open-notebook/issues +- **Contributing:** [CONTRIBUTING.md](CONTRIBUTING.md) +- **Maintainer Guide:** [MAINTAINER_GUIDE.md](MAINTAINER_GUIDE.md) + +--- + +**Last Updated:** January 2025 diff --git a/README.md b/README.md index 17154d6..3d79ef1 100644 --- a/README.md +++ b/README.md @@ -195,19 +195,6 @@ Thanks to the [Esperanto](https://github.com/lfnovo/esperanto) library, we suppo ## Podcast Feature -### Audio 1 - - -### Audio 2 - - - [![Check out our podcast sample](https://img.youtube.com/vi/D-760MlGwaI/0.jpg)](https://www.youtube.com/watch?v=D-760MlGwaI) ## ๐Ÿ“š Documentation diff --git a/open_notebook/domain/base.py b/open_notebook/domain/base.py index cf9dfdb..7684489 100644 --- a/open_notebook/domain/base.py +++ b/open_notebook/domain/base.py @@ -2,7 +2,7 @@ from datetime import datetime from typing import Any, ClassVar, Dict, List, Optional, Type, TypeVar, Union, cast from loguru import logger -from pydantic import BaseModel, ValidationError, field_validator, model_validator +from pydantic import BaseModel, ConfigDict, ValidationError, field_validator, model_validator from open_notebook.database.repository import ( ensure_record_id, @@ -211,19 +211,20 @@ class ObjectModel(BaseModel): class RecordModel(BaseModel): + model_config = ConfigDict( + validate_assignment=True, + arbitrary_types_allowed=True, + extra="allow", + from_attributes=True, + defer_build=True, + ) + record_id: ClassVar[str] auto_save: ClassVar[bool] = ( False # Default to False, can be overridden in subclasses ) _instances: ClassVar[Dict[str, "RecordModel"]] = {} # Store instances by record_id - class Config: - validate_assignment = True - arbitrary_types_allowed = True - extra = "allow" - from_attributes = True - defer_build = True - def __new__(cls, **kwargs): # If an instance already exists for this record_id, return it if cls.record_id in cls._instances: diff --git a/open_notebook/domain/notebook.py b/open_notebook/domain/notebook.py index fd03d5b..3507509 100644 --- a/open_notebook/domain/notebook.py +++ b/open_notebook/domain/notebook.py @@ -2,7 +2,7 @@ import asyncio from typing import Any, ClassVar, Dict, List, Literal, Optional, Tuple, Union from loguru import logger -from pydantic import BaseModel, Field, field_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator from surreal_commands import submit_command from surrealdb import RecordID @@ -142,6 +142,8 @@ class SourceInsight(ObjectModel): class Source(ObjectModel): + model_config = ConfigDict(arbitrary_types_allowed=True) + table_name: ClassVar[str] = "source" asset: Optional[Asset] = None title: Optional[str] = None @@ -151,9 +153,6 @@ class Source(ObjectModel): default=None, description="Link to surreal-commands processing job" ) - class Config: - arbitrary_types_allowed = True - @field_validator("command", mode="before") @classmethod def parse_command(cls, value): diff --git a/open_notebook/podcasts/models.py b/open_notebook/podcasts/models.py index ad5ef44..5545f94 100644 --- a/open_notebook/podcasts/models.py +++ b/open_notebook/podcasts/models.py @@ -1,6 +1,6 @@ from typing import Any, ClassVar, Dict, List, Optional, Union -from pydantic import Field, field_validator +from pydantic import ConfigDict, Field, field_validator from surrealdb import RecordID from open_notebook.database.repository import ensure_record_id, repo_query @@ -114,8 +114,7 @@ class PodcastEpisode(ObjectModel): default=None, description="Link to surreal-commands job" ) - class Config: - arbitrary_types_allowed = True + model_config = ConfigDict(arbitrary_types_allowed=True) async def get_job_status(self) -> Optional[str]: """Get the status of the associated command""" diff --git a/pyproject.toml b/pyproject.toml index 30b0cdd..1d8c0f7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,25 +17,25 @@ dependencies = [ "uvicorn>=0.24.0", "pydantic>=2.9.2", "loguru>=0.7.2", - "langchain>=0.3.3", - "langgraph>=0.2.38", - "tiktoken>=0.8.0", - "langgraph-checkpoint-sqlite>=2.0.0", - "langchain-community>=0.3.3", - "langchain-openai>=0.2.3", - "langchain-anthropic>=0.2.3", - "langchain-ollama>=0.2.0", - "langchain-google-genai>=2.1.10", - "langchain-groq>=0.2.1", - "langchain_mistralai>=0.2.1", - "langchain_deepseek>=0.1.3", - "langchain-google-vertexai>=2.0.28", + "langchain>=1.2.0", + "langgraph>=1.0.5", + "tiktoken>=0.12.0", + "langgraph-checkpoint-sqlite>=3.0.1", + "langchain-community>=0.4.1", + "langchain-openai>=1.1.6", + "langchain-anthropic>=1.3.0", + "langchain-ollama>=1.0.1", + "langchain-google-genai>=4.1.2", + "langchain-groq>=1.1.1", + "langchain_mistralai>=1.1.1", + "langchain_deepseek>=1.0.0", + "langchain-google-vertexai>=3.2.0", "tomli>=2.0.2", "python-dotenv>=1.0.1", "httpx[socks]>=0.27.0", "content-core>=1.0.2", "ai-prompter>=0.3", - "esperanto>=2.8.3", + "esperanto>=2.13", "surrealdb>=1.0.4", "podcast-creator>=0.7.0", "surreal-commands>=1.2.0", diff --git a/uv.lock b/uv.lock index 26fe971..86e3d7d 100644 --- a/uv.lock +++ b/uv.lock @@ -671,15 +671,15 @@ wheels = [ [[package]] name = "esperanto" -version = "2.12.1" +version = "2.13.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "httpx" }, { name = "pydantic" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/7f/7d/856ecb7ab6b05fa212ef4cc9186be5008373aeced2dc1ebc99013c96fa3c/esperanto-2.12.1.tar.gz", hash = "sha256:177f001363d7710a7bdf747adc2c3f5e00aebd6b759c3ac6642e2c8df8a4e1cf", size = 811783, upload-time = "2025-12-16T00:53:46.798Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b5/8c/9f655703422fc895f4c327316a8b4c824ec334374a8f6e2dea61f2512362/esperanto-2.13.0.tar.gz", hash = "sha256:78df58492700d4cdfe9dd715313c48e5e4b816ecb87dc22a56d03610431c640e", size = 742118, upload-time = "2026-01-04T22:14:24.996Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/65/48/5d8c6bc2b5db29f3b8f8be7c0964f8d78bd40b3586133f71b773530f67af/esperanto-2.12.1-py3-none-any.whl", hash = "sha256:341c239d3c7a14d556a3b4225c6e1b5f66d635e34ec8026117fc18f4f608b3c3", size = 152050, upload-time = "2025-12-16T00:53:48.023Z" }, + { url = "https://files.pythonhosted.org/packages/8c/44/e1c9aa604f252a351d67398c886bb9b1cd572881289864d3e0d45212e61d/esperanto-2.13.0-py3-none-any.whl", hash = "sha256:5b9d12eb3d03f63acb7a77c0c17ed90b32b761ded1f122bc44bb4e8b6625cec0", size = 152019, upload-time = "2026-01-04T22:14:24.031Z" }, ] [[package]] @@ -2439,23 +2439,23 @@ dev = [ requires-dist = [ { name = "ai-prompter", specifier = ">=0.3" }, { name = "content-core", specifier = ">=1.0.2" }, - { name = "esperanto", specifier = ">=2.8.3" }, + { name = "esperanto", specifier = ">=2.13" }, { name = "fastapi", specifier = ">=0.104.0" }, { name = "httpx", extras = ["socks"], specifier = ">=0.27.0" }, { name = "ipykernel", marker = "extra == 'dev'", specifier = ">=6.29.5" }, { name = "ipywidgets", marker = "extra == 'dev'", specifier = ">=8.1.5" }, - { name = "langchain", specifier = ">=0.3.3" }, - { name = "langchain-anthropic", specifier = ">=0.2.3" }, - { name = "langchain-community", specifier = ">=0.3.3" }, - { name = "langchain-deepseek", specifier = ">=0.1.3" }, - { name = "langchain-google-genai", specifier = ">=2.1.10" }, - { name = "langchain-google-vertexai", specifier = ">=2.0.28" }, - { name = "langchain-groq", specifier = ">=0.2.1" }, - { name = "langchain-mistralai", specifier = ">=0.2.1" }, - { name = "langchain-ollama", specifier = ">=0.2.0" }, - { name = "langchain-openai", specifier = ">=0.2.3" }, - { name = "langgraph", specifier = ">=0.2.38" }, - { name = "langgraph-checkpoint-sqlite", specifier = ">=2.0.0" }, + { name = "langchain", specifier = ">=1.2.0" }, + { name = "langchain-anthropic", specifier = ">=1.3.0" }, + { name = "langchain-community", specifier = ">=0.4.1" }, + { name = "langchain-deepseek", specifier = ">=1.0.0" }, + { name = "langchain-google-genai", specifier = ">=4.1.2" }, + { name = "langchain-google-vertexai", specifier = ">=3.2.0" }, + { name = "langchain-groq", specifier = ">=1.1.1" }, + { name = "langchain-mistralai", specifier = ">=1.1.1" }, + { name = "langchain-ollama", specifier = ">=1.0.1" }, + { name = "langchain-openai", specifier = ">=1.1.6" }, + { name = "langgraph", specifier = ">=1.0.5" }, + { name = "langgraph-checkpoint-sqlite", specifier = ">=3.0.1" }, { name = "loguru", specifier = ">=0.7.2" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.11.1" }, { name = "podcast-creator", specifier = ">=0.7.0" }, @@ -2466,7 +2466,7 @@ requires-dist = [ { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.5.5" }, { name = "surreal-commands", specifier = ">=1.2.0" }, { name = "surrealdb", specifier = ">=1.0.4" }, - { name = "tiktoken", specifier = ">=0.8.0" }, + { name = "tiktoken", specifier = ">=0.12.0" }, { name = "tomli", specifier = ">=2.0.2" }, { name = "types-requests", marker = "extra == 'dev'", specifier = ">=2.32.0.20241016" }, { name = "uvicorn", specifier = ">=0.24.0" },