fix environment variable error and enable docker build automation (#94)

* chore: fix database import error

* remove unused file and improve env example

* docker build automation
This commit is contained in:
Luis Novo 2025-07-17 09:54:28 -03:00 committed by GitHub
parent d7b0fff954
commit 3b2ced54e2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 359 additions and 341 deletions

View file

@ -65,14 +65,7 @@ SURREAL_PASSWORD="root"
SURREAL_NAMESPACE="open_notebook"
SURREAL_DATABASE="staging"
# Old format (backward compatible) - will be converted automatically
# SURREAL_ADDRESS="localhost"
# SURREAL_PORT=8000
# SURREAL_USER="root"
# SURREAL_PASS="root"
# SURREAL_NAMESPACE="open_notebook"
# SURREAL_DATABASE="staging"
# OPEN_NOTEBOOK_PASSWORD=
# FIRECRAWL - Get a key at https://firecrawl.dev/
FIRECRAWL_API_KEY=

168
.github/workflows/build-and-release.yml vendored Normal file
View file

@ -0,0 +1,168 @@
name: Build and Release
on:
workflow_dispatch:
inputs:
build_type:
description: 'Build type to create'
required: true
default: 'both'
type: choice
options:
- both
- regular
- single
push_latest:
description: 'Also push latest tags'
required: false
default: true
type: boolean
release:
types: [published]
env:
REGISTRY: docker.io
IMAGE_NAME: lfnovo/open_notebook
jobs:
extract-version:
runs-on: ubuntu-latest
outputs:
version: ${{ steps.version.outputs.version }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Extract version from pyproject.toml
id: version
run: |
VERSION=$(grep -m1 '^version = ' pyproject.toml | cut -d'"' -f2)
echo "version=$VERSION" >> $GITHUB_OUTPUT
echo "Extracted version: $VERSION"
build-regular:
needs: extract-version
runs-on: ubuntu-latest
if: github.event.inputs.build_type == 'regular' || github.event.inputs.build_type == 'both' || github.event_name == 'release'
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Cache Docker layers
uses: actions/cache@v3
with:
path: /tmp/.buildx-cache
key: ${{ runner.os }}-buildx-regular-${{ github.sha }}
restore-keys: |
${{ runner.os }}-buildx-regular-
- name: Build and push regular image
uses: docker/build-push-action@v5
with:
context: .
file: ./Dockerfile
platforms: linux/amd64,linux/arm64
push: true
tags: |
${{ env.IMAGE_NAME }}:${{ needs.extract-version.outputs.version }}
${{ github.event.inputs.push_latest == 'true' && format('{0}:latest', env.IMAGE_NAME) || '' }}
cache-from: type=local,src=/tmp/.buildx-cache
cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
- name: Move cache
run: |
rm -rf /tmp/.buildx-cache
mv /tmp/.buildx-cache-new /tmp/.buildx-cache
build-single:
needs: extract-version
runs-on: ubuntu-latest
if: github.event.inputs.build_type == 'single' || github.event.inputs.build_type == 'both' || github.event_name == 'release'
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Cache Docker layers
uses: actions/cache@v3
with:
path: /tmp/.buildx-cache-single
key: ${{ runner.os }}-buildx-single-${{ github.sha }}
restore-keys: |
${{ runner.os }}-buildx-single-
- name: Build and push single-container image
uses: docker/build-push-action@v5
with:
context: .
file: ./Dockerfile.single
platforms: linux/amd64,linux/arm64
push: true
tags: |
${{ env.IMAGE_NAME }}:${{ needs.extract-version.outputs.version }}-single
${{ github.event.inputs.push_latest == 'true' && format('{0}:latest-single', env.IMAGE_NAME) || '' }}
cache-from: type=local,src=/tmp/.buildx-cache-single
cache-to: type=local,dest=/tmp/.buildx-cache-single-new,mode=max
- name: Move cache
run: |
rm -rf /tmp/.buildx-cache-single
mv /tmp/.buildx-cache-single-new /tmp/.buildx-cache-single
summary:
needs: [extract-version, build-regular, build-single]
runs-on: ubuntu-latest
if: always()
steps:
- name: Build Summary
run: |
echo "## Build Summary" >> $GITHUB_STEP_SUMMARY
echo "**Version:** ${{ needs.extract-version.outputs.version }}" >> $GITHUB_STEP_SUMMARY
echo "**Build Type:** ${{ github.event.inputs.build_type || 'both' }}" >> $GITHUB_STEP_SUMMARY
echo "**Push Latest:** ${{ github.event.inputs.push_latest || 'true' }}" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "### Images Built:" >> $GITHUB_STEP_SUMMARY
if [[ "${{ needs.build-regular.result }}" == "success" ]]; then
echo "✅ **Regular:** \`${{ env.IMAGE_NAME }}:${{ needs.extract-version.outputs.version }}\`" >> $GITHUB_STEP_SUMMARY
if [[ "${{ github.event.inputs.push_latest }}" == "true" ]]; then
echo "✅ **Regular Latest:** \`${{ env.IMAGE_NAME }}:latest\`" >> $GITHUB_STEP_SUMMARY
fi
elif [[ "${{ needs.build-regular.result }}" == "skipped" ]]; then
echo "⏭️ **Regular:** Skipped" >> $GITHUB_STEP_SUMMARY
else
echo "❌ **Regular:** Failed" >> $GITHUB_STEP_SUMMARY
fi
if [[ "${{ needs.build-single.result }}" == "success" ]]; then
echo "✅ **Single:** \`${{ env.IMAGE_NAME }}:${{ needs.extract-version.outputs.version }}-single\`" >> $GITHUB_STEP_SUMMARY
if [[ "${{ github.event.inputs.push_latest }}" == "true" ]]; then
echo "✅ **Single Latest:** \`${{ env.IMAGE_NAME }}:latest-single\`" >> $GITHUB_STEP_SUMMARY
fi
elif [[ "${{ needs.build-single.result }}" == "skipped" ]]; then
echo "⏭️ **Single:** Skipped" >> $GITHUB_STEP_SUMMARY
else
echo "❌ **Single:** Failed" >> $GITHUB_STEP_SUMMARY
fi
echo "" >> $GITHUB_STEP_SUMMARY
echo "### Platforms:" >> $GITHUB_STEP_SUMMARY
echo "- linux/amd64" >> $GITHUB_STEP_SUMMARY
echo "- linux/arm64" >> $GITHUB_STEP_SUMMARY

186
.github/workflows/build-dev.yml vendored Normal file
View file

@ -0,0 +1,186 @@
name: Development Build
on:
pull_request:
branches: [ main ]
push:
branches: [ main ]
paths-ignore:
- '**.md'
- 'docs/**'
- 'notebooks/**'
- '.github/workflows/claude*.yml'
workflow_dispatch:
inputs:
dockerfile:
description: 'Dockerfile to test'
required: true
default: 'both'
type: choice
options:
- both
- regular
- single
platform:
description: 'Platform to build'
required: true
default: 'linux/amd64'
type: choice
options:
- linux/amd64
- linux/arm64
- linux/amd64,linux/arm64
env:
REGISTRY: docker.io
IMAGE_NAME: lfnovo/open_notebook
jobs:
extract-version:
runs-on: ubuntu-latest
outputs:
version: ${{ steps.version.outputs.version }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Extract version from pyproject.toml
id: version
run: |
VERSION=$(grep -m1 '^version = ' pyproject.toml | cut -d'"' -f2)
echo "version=$VERSION" >> $GITHUB_OUTPUT
echo "Extracted version: $VERSION"
lint-and-check:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.12'
- name: Install uv
uses: astral-sh/setup-uv@v3
- name: Install dependencies
run: uv sync --dev
- name: Run ruff
run: uv run ruff check . --output-format=github
- name: Run mypy
run: uv run python -m mypy .
test-build-regular:
needs: extract-version
runs-on: ubuntu-latest
if: github.event.inputs.dockerfile == 'regular' || github.event.inputs.dockerfile == 'both' || github.event_name != 'workflow_dispatch'
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Cache Docker layers
uses: actions/cache@v3
with:
path: /tmp/.buildx-cache-dev
key: ${{ runner.os }}-buildx-dev-regular-${{ github.sha }}
restore-keys: |
${{ runner.os }}-buildx-dev-regular-
- name: Build regular image (test only)
uses: docker/build-push-action@v5
with:
context: .
file: ./Dockerfile
platforms: ${{ github.event.inputs.platform || 'linux/amd64' }}
push: false
tags: ${{ env.IMAGE_NAME }}:${{ needs.extract-version.outputs.version }}-dev-regular
cache-from: type=local,src=/tmp/.buildx-cache-dev
cache-to: type=local,dest=/tmp/.buildx-cache-dev-new,mode=max
- name: Move cache
run: |
rm -rf /tmp/.buildx-cache-dev
mv /tmp/.buildx-cache-dev-new /tmp/.buildx-cache-dev
test-build-single:
needs: extract-version
runs-on: ubuntu-latest
if: github.event.inputs.dockerfile == 'single' || github.event.inputs.dockerfile == 'both' || github.event_name != 'workflow_dispatch'
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Cache Docker layers
uses: actions/cache@v3
with:
path: /tmp/.buildx-cache-dev-single
key: ${{ runner.os }}-buildx-dev-single-${{ github.sha }}
restore-keys: |
${{ runner.os }}-buildx-dev-single-
- name: Build single-container image (test only)
uses: docker/build-push-action@v5
with:
context: .
file: ./Dockerfile.single
platforms: ${{ github.event.inputs.platform || 'linux/amd64' }}
push: false
tags: ${{ env.IMAGE_NAME }}:${{ needs.extract-version.outputs.version }}-dev-single
cache-from: type=local,src=/tmp/.buildx-cache-dev-single
cache-to: type=local,dest=/tmp/.buildx-cache-dev-single-new,mode=max
- name: Move cache
run: |
rm -rf /tmp/.buildx-cache-dev-single
mv /tmp/.buildx-cache-dev-single-new /tmp/.buildx-cache-dev-single
summary:
needs: [extract-version, lint-and-check, test-build-regular, test-build-single]
runs-on: ubuntu-latest
if: always()
steps:
- name: Development Build Summary
run: |
echo "## Development Build Summary" >> $GITHUB_STEP_SUMMARY
echo "**Version:** ${{ needs.extract-version.outputs.version }}" >> $GITHUB_STEP_SUMMARY
echo "**Platform:** ${{ github.event.inputs.platform || 'linux/amd64' }}" >> $GITHUB_STEP_SUMMARY
echo "**Dockerfile:** ${{ github.event.inputs.dockerfile || 'both' }}" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "### Results:" >> $GITHUB_STEP_SUMMARY
if [[ "${{ needs.lint-and-check.result }}" == "success" ]]; then
echo "✅ **Lint & Type Check:** Passed" >> $GITHUB_STEP_SUMMARY
else
echo "❌ **Lint & Type Check:** Failed" >> $GITHUB_STEP_SUMMARY
fi
if [[ "${{ needs.test-build-regular.result }}" == "success" ]]; then
echo "✅ **Regular Dockerfile:** Build successful" >> $GITHUB_STEP_SUMMARY
elif [[ "${{ needs.test-build-regular.result }}" == "skipped" ]]; then
echo "⏭️ **Regular Dockerfile:** Skipped" >> $GITHUB_STEP_SUMMARY
else
echo "❌ **Regular Dockerfile:** Build failed" >> $GITHUB_STEP_SUMMARY
fi
if [[ "${{ needs.test-build-single.result }}" == "success" ]]; then
echo "✅ **Single Dockerfile:** Build successful" >> $GITHUB_STEP_SUMMARY
elif [[ "${{ needs.test-build-single.result }}" == "skipped" ]]; then
echo "⏭️ **Single Dockerfile:** Skipped" >> $GITHUB_STEP_SUMMARY
else
echo "❌ **Single Dockerfile:** Build failed" >> $GITHUB_STEP_SUMMARY
fi
echo "" >> $GITHUB_STEP_SUMMARY
echo "### Notes:" >> $GITHUB_STEP_SUMMARY
echo "- This is a development build (no images pushed to registry)" >> $GITHUB_STEP_SUMMARY
echo "- For production releases, use the 'Build and Release' workflow" >> $GITHUB_STEP_SUMMARY

View file

@ -1,178 +0,0 @@
import os
from contextlib import asynccontextmanager
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional, TypeVar, Union
from loguru import logger
from surrealdb import AsyncSurreal, RecordID # type: ignore
T = TypeVar("T", Dict[str, Any], List[Dict[str, Any]])
def get_database_url():
"""Get database URL with backward compatibility"""
surreal_url = os.getenv("SURREAL_URL")
if surreal_url:
return surreal_url
# Fallback to old format - WebSocket URL format
address = os.getenv("SURREAL_ADDRESS", "localhost")
port = os.getenv("SURREAL_PORT", "8000")
return f"ws://{address}/rpc:{port}"
def get_database_password():
"""Get password with backward compatibility"""
return os.getenv("SURREAL_PASSWORD") or os.getenv("SURREAL_PASS")
def parse_record_ids(obj: Any) -> Any:
"""Recursively parse and convert RecordIDs into strings."""
if isinstance(obj, dict):
return {k: parse_record_ids(v) for k, v in obj.items()}
elif isinstance(obj, list):
return [parse_record_ids(item) for item in obj]
elif isinstance(obj, RecordID):
return str(obj)
return obj
def ensure_record_id(value: Union[str, RecordID]) -> RecordID:
"""Ensure a value is a RecordID."""
if isinstance(value, RecordID):
return value
return RecordID.parse(value)
@asynccontextmanager
async def db_connection():
db = AsyncSurreal(get_database_url())
await db.signin(
{
"username": os.environ["SURREAL_USER"],
"password": get_database_password(),
}
)
await db.use(os.environ["SURREAL_NAMESPACE"], os.environ["SURREAL_DATABASE"])
try:
yield db
finally:
await db.close()
async def repo_query(
query_str: str, vars: Optional[Dict[str, Any]] = None
) -> List[Dict[str, Any]]:
"""Execute a SurrealQL query and return the results"""
async with db_connection() as connection:
try:
result = parse_record_ids(await connection.query(query_str, vars))
if isinstance(result, str):
raise RuntimeError(result)
return result
except Exception as e:
logger.error(f"Query: {query_str[:200]} vars: {vars}")
logger.exception(e)
raise
async def repo_create(table: str, data: Dict[str, Any]) -> Dict[str, Any]:
"""Create a new record in the specified table"""
# Remove 'id' attribute if it exists in data
data.pop("id", None)
data["created"] = datetime.now(timezone.utc)
data["updated"] = datetime.now(timezone.utc)
try:
async with db_connection() as connection:
return parse_record_ids(await connection.insert(table, data))
except Exception as e:
logger.exception(e)
raise RuntimeError("Failed to create record")
async def repo_relate(
source: str, relationship: str, target: str, data: Optional[Dict[str, Any]] = None
) -> List[Dict[str, Any]]:
"""Create a relationship between two records with optional data"""
if data is None:
data = {}
query = f"RELATE {source}->{relationship}->{target} CONTENT $data;"
# logger.debug(f"Relate query: {query}")
return await repo_query(
query,
{
"data": data,
},
)
async def repo_upsert(
table: str, id: Optional[str], data: Dict[str, Any], add_timestamp: bool = False
) -> List[Dict[str, Any]]:
"""Create or update a record in the specified table"""
data.pop("id", None)
if add_timestamp:
data["updated"] = datetime.now(timezone.utc)
query = f"UPSERT {id if id else table} MERGE $data;"
return await repo_query(query, {"data": data})
async def repo_update(
table: str, id: str, data: Dict[str, Any]
) -> List[Dict[str, Any]]:
"""Update an existing record by table and id"""
# If id already contains the table name, use it as is
try:
if isinstance(id, RecordID) or (":" in id and id.startswith(f"{table}:")):
record_id = id
else:
record_id = f"{table}:{id}"
data["updated"] = datetime.now(timezone.utc)
query = f"UPDATE {record_id} MERGE $data;"
# logger.debug(f"Update query: {query}")
result = await repo_query(query, {"data": data})
# if isinstance(result, list):
# return [_return_data(item) for item in result]
return [parse_record_ids(result)]
except Exception as e:
raise RuntimeError(f"Failed to update record: {str(e)}")
async def repo_get_news_by_jota_id(jota_id: str) -> Dict[str, Any]:
try:
results = await repo_query(
"SELECT * omit embedding FROM news where jota_id=$jota_id",
{"jota_id": jota_id},
)
return parse_record_ids(results)
except Exception as e:
logger.exception(e)
raise RuntimeError(f"Failed to fetch record: {str(e)}")
async def repo_delete(record_id: Union[str, RecordID]):
"""Delete a record by record id"""
try:
async with db_connection() as connection:
return await connection.delete(record_id)
except Exception as e:
logger.exception(e)
raise RuntimeError(f"Failed to delete record: {str(e)}")
async def repo_insert(
table: str, data: List[Dict[str, Any]], ignore_duplicates: bool = False
) -> List[Dict[str, Any]]:
"""Create a new record in the specified table"""
try:
async with db_connection() as connection:
return parse_record_ids(await connection.insert(table, data))
except Exception as e:
if ignore_duplicates and "already contains" in str(e):
return []
logger.exception(e)
raise RuntimeError("Failed to create record")

View file

@ -49,11 +49,13 @@ async def db_connection():
db = AsyncSurreal(get_database_url())
await db.signin(
{
"username": os.environ["SURREAL_USER"],
"username": os.environ.get("SURREAL_USER"),
"password": get_database_password(),
}
)
await db.use(os.environ["SURREAL_NAMESPACE"], os.environ["SURREAL_DATABASE"])
await db.use(
os.environ.get("SURREAL_NAMESPACE"), os.environ.get("SURREAL_DATABASE")
)
try:
yield db
finally:

View file

@ -1,63 +0,0 @@
import os
from contextlib import contextmanager
from typing import Any, Dict, Optional
from loguru import logger
from sblpy.connection import SurrealSyncConnection
@contextmanager
def db_connection():
connection = SurrealSyncConnection(
host=os.environ["SURREAL_ADDRESS"],
port=int(os.environ["SURREAL_PORT"]),
user=os.environ["SURREAL_USER"],
password=os.environ["SURREAL_PASS"],
namespace=os.environ["SURREAL_NAMESPACE"],
database=os.environ["SURREAL_DATABASE"],
max_size=2.2**20,
encrypted=False, # Set to True if using SSL
)
try:
yield connection
finally:
connection.socket.close()
def repo_query(query_str: str, vars: Optional[Dict[str, Any]] = None):
with db_connection() as connection:
try:
result = connection.query(query_str, vars)
return result
except Exception as e:
logger.critical(f"Query: {query_str}")
logger.exception(e)
raise
def repo_create(table: str, data: Dict[str, Any]):
query = f"CREATE {table} CONTENT {data};"
return repo_query(query)
def repo_upsert(table: str, data: Dict[str, Any]):
query = f"UPSERT {table} CONTENT {data};"
return repo_query(query)
def repo_update(id: str, data: Dict[str, Any]):
query = "UPDATE $id CONTENT $data;"
vars = {"id": id, "data": data}
return repo_query(query, vars)
def repo_delete(id: str):
query = "DELETE $id;"
vars = {"id": id}
return repo_query(query, vars)
def repo_relate(source: str, relationship: str, target: str, data: Optional[Dict] = {}):
query = f"RELATE {source}->{relationship}->{target} CONTENT $content;"
result = repo_query(query, {"content": data})
return result

View file

@ -1,90 +0,0 @@
#!/bin/bash
echo "=== Testing Surreal Commands Integration ==="
echo ""
# Base URL
BASE_URL="http://localhost:5055/api"
# 1. Test text processing command
echo "1. Testing text processing command (uppercase)..."
curl -X POST "$BASE_URL/commands/jobs" \
-H "Content-Type: application/json" \
-d '{
"command": "process_text",
"app": "open_notebook",
"input": {
"text": "Hello, this is a test message!",
"operation": "uppercase"
}
}' | jq .
echo ""
echo "2. Testing text processing with delay (3 seconds)..."
curl -X POST "$BASE_URL/commands/jobs" \
-H "Content-Type: application/json" \
-d '{
"command": "process_text",
"app": "open_notebook",
"input": {
"text": "Testing async behavior with delay",
"operation": "reverse",
"delay_seconds": 3
}
}' | jq .
echo ""
echo "3. Testing data analysis command..."
curl -X POST "$BASE_URL/commands/jobs" \
-H "Content-Type: application/json" \
-d '{
"command": "analyze_data",
"app": "open_notebook",
"input": {
"numbers": [10, 20, 30, 40, 50],
"analysis_type": "basic"
}
}' | jq .
echo ""
echo "4. Testing error scenario (empty numbers array)..."
curl -X POST "$BASE_URL/commands/jobs" \
-H "Content-Type: application/json" \
-d '{
"command": "analyze_data",
"app": "open_notebook",
"input": {
"numbers": [],
"analysis_type": "basic"
}
}' | jq .
echo ""
echo "5. Testing word count operation..."
curl -X POST "$BASE_URL/commands/jobs" \
-H "Content-Type: application/json" \
-d '{
"command": "process_text",
"app": "open_notebook",
"input": {
"text": "This is a sample text with multiple words to count",
"operation": "word_count"
}
}' | jq .
echo ""
echo "Please save the job_ids from above to check status!"
echo ""
echo "6. To check job status (replace JOB_ID with actual ID):"
echo "curl \"$BASE_URL/commands/jobs/{JOB_ID}\" | jq ."
echo ""
echo "7. To list all jobs:"
echo "curl \"$BASE_URL/commands/jobs\" | jq ."
echo ""
echo "=== Test Commands Complete ==="
echo ""
echo "Manual status check example:"
echo "Replace JOB_ID with one of the job IDs returned above:"
echo "curl \"$BASE_URL/commands/jobs/JOB_ID\" | jq ."