fix environment variable error and enable docker build automation (#94)
* chore: fix database import error * remove unused file and improve env example * docker build automation
This commit is contained in:
parent
d7b0fff954
commit
3b2ced54e2
7 changed files with 359 additions and 341 deletions
|
|
@ -65,14 +65,7 @@ SURREAL_PASSWORD="root"
|
||||||
SURREAL_NAMESPACE="open_notebook"
|
SURREAL_NAMESPACE="open_notebook"
|
||||||
SURREAL_DATABASE="staging"
|
SURREAL_DATABASE="staging"
|
||||||
|
|
||||||
# Old format (backward compatible) - will be converted automatically
|
# OPEN_NOTEBOOK_PASSWORD=
|
||||||
# SURREAL_ADDRESS="localhost"
|
|
||||||
# SURREAL_PORT=8000
|
|
||||||
# SURREAL_USER="root"
|
|
||||||
# SURREAL_PASS="root"
|
|
||||||
# SURREAL_NAMESPACE="open_notebook"
|
|
||||||
# SURREAL_DATABASE="staging"
|
|
||||||
|
|
||||||
|
|
||||||
# FIRECRAWL - Get a key at https://firecrawl.dev/
|
# FIRECRAWL - Get a key at https://firecrawl.dev/
|
||||||
FIRECRAWL_API_KEY=
|
FIRECRAWL_API_KEY=
|
||||||
|
|
|
||||||
168
.github/workflows/build-and-release.yml
vendored
Normal file
168
.github/workflows/build-and-release.yml
vendored
Normal file
|
|
@ -0,0 +1,168 @@
|
||||||
|
name: Build and Release
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
build_type:
|
||||||
|
description: 'Build type to create'
|
||||||
|
required: true
|
||||||
|
default: 'both'
|
||||||
|
type: choice
|
||||||
|
options:
|
||||||
|
- both
|
||||||
|
- regular
|
||||||
|
- single
|
||||||
|
push_latest:
|
||||||
|
description: 'Also push latest tags'
|
||||||
|
required: false
|
||||||
|
default: true
|
||||||
|
type: boolean
|
||||||
|
release:
|
||||||
|
types: [published]
|
||||||
|
|
||||||
|
env:
|
||||||
|
REGISTRY: docker.io
|
||||||
|
IMAGE_NAME: lfnovo/open_notebook
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
extract-version:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
outputs:
|
||||||
|
version: ${{ steps.version.outputs.version }}
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Extract version from pyproject.toml
|
||||||
|
id: version
|
||||||
|
run: |
|
||||||
|
VERSION=$(grep -m1 '^version = ' pyproject.toml | cut -d'"' -f2)
|
||||||
|
echo "version=$VERSION" >> $GITHUB_OUTPUT
|
||||||
|
echo "Extracted version: $VERSION"
|
||||||
|
|
||||||
|
build-regular:
|
||||||
|
needs: extract-version
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
if: github.event.inputs.build_type == 'regular' || github.event.inputs.build_type == 'both' || github.event_name == 'release'
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Set up Docker Buildx
|
||||||
|
uses: docker/setup-buildx-action@v3
|
||||||
|
|
||||||
|
- name: Login to Docker Hub
|
||||||
|
uses: docker/login-action@v3
|
||||||
|
with:
|
||||||
|
username: ${{ secrets.DOCKER_USERNAME }}
|
||||||
|
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||||
|
|
||||||
|
- name: Cache Docker layers
|
||||||
|
uses: actions/cache@v3
|
||||||
|
with:
|
||||||
|
path: /tmp/.buildx-cache
|
||||||
|
key: ${{ runner.os }}-buildx-regular-${{ github.sha }}
|
||||||
|
restore-keys: |
|
||||||
|
${{ runner.os }}-buildx-regular-
|
||||||
|
|
||||||
|
- name: Build and push regular image
|
||||||
|
uses: docker/build-push-action@v5
|
||||||
|
with:
|
||||||
|
context: .
|
||||||
|
file: ./Dockerfile
|
||||||
|
platforms: linux/amd64,linux/arm64
|
||||||
|
push: true
|
||||||
|
tags: |
|
||||||
|
${{ env.IMAGE_NAME }}:${{ needs.extract-version.outputs.version }}
|
||||||
|
${{ github.event.inputs.push_latest == 'true' && format('{0}:latest', env.IMAGE_NAME) || '' }}
|
||||||
|
cache-from: type=local,src=/tmp/.buildx-cache
|
||||||
|
cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
|
||||||
|
|
||||||
|
- name: Move cache
|
||||||
|
run: |
|
||||||
|
rm -rf /tmp/.buildx-cache
|
||||||
|
mv /tmp/.buildx-cache-new /tmp/.buildx-cache
|
||||||
|
|
||||||
|
build-single:
|
||||||
|
needs: extract-version
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
if: github.event.inputs.build_type == 'single' || github.event.inputs.build_type == 'both' || github.event_name == 'release'
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Set up Docker Buildx
|
||||||
|
uses: docker/setup-buildx-action@v3
|
||||||
|
|
||||||
|
- name: Login to Docker Hub
|
||||||
|
uses: docker/login-action@v3
|
||||||
|
with:
|
||||||
|
username: ${{ secrets.DOCKER_USERNAME }}
|
||||||
|
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||||
|
|
||||||
|
- name: Cache Docker layers
|
||||||
|
uses: actions/cache@v3
|
||||||
|
with:
|
||||||
|
path: /tmp/.buildx-cache-single
|
||||||
|
key: ${{ runner.os }}-buildx-single-${{ github.sha }}
|
||||||
|
restore-keys: |
|
||||||
|
${{ runner.os }}-buildx-single-
|
||||||
|
|
||||||
|
- name: Build and push single-container image
|
||||||
|
uses: docker/build-push-action@v5
|
||||||
|
with:
|
||||||
|
context: .
|
||||||
|
file: ./Dockerfile.single
|
||||||
|
platforms: linux/amd64,linux/arm64
|
||||||
|
push: true
|
||||||
|
tags: |
|
||||||
|
${{ env.IMAGE_NAME }}:${{ needs.extract-version.outputs.version }}-single
|
||||||
|
${{ github.event.inputs.push_latest == 'true' && format('{0}:latest-single', env.IMAGE_NAME) || '' }}
|
||||||
|
cache-from: type=local,src=/tmp/.buildx-cache-single
|
||||||
|
cache-to: type=local,dest=/tmp/.buildx-cache-single-new,mode=max
|
||||||
|
|
||||||
|
- name: Move cache
|
||||||
|
run: |
|
||||||
|
rm -rf /tmp/.buildx-cache-single
|
||||||
|
mv /tmp/.buildx-cache-single-new /tmp/.buildx-cache-single
|
||||||
|
|
||||||
|
summary:
|
||||||
|
needs: [extract-version, build-regular, build-single]
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
if: always()
|
||||||
|
steps:
|
||||||
|
- name: Build Summary
|
||||||
|
run: |
|
||||||
|
echo "## Build Summary" >> $GITHUB_STEP_SUMMARY
|
||||||
|
echo "**Version:** ${{ needs.extract-version.outputs.version }}" >> $GITHUB_STEP_SUMMARY
|
||||||
|
echo "**Build Type:** ${{ github.event.inputs.build_type || 'both' }}" >> $GITHUB_STEP_SUMMARY
|
||||||
|
echo "**Push Latest:** ${{ github.event.inputs.push_latest || 'true' }}" >> $GITHUB_STEP_SUMMARY
|
||||||
|
echo "" >> $GITHUB_STEP_SUMMARY
|
||||||
|
echo "### Images Built:" >> $GITHUB_STEP_SUMMARY
|
||||||
|
|
||||||
|
if [[ "${{ needs.build-regular.result }}" == "success" ]]; then
|
||||||
|
echo "✅ **Regular:** \`${{ env.IMAGE_NAME }}:${{ needs.extract-version.outputs.version }}\`" >> $GITHUB_STEP_SUMMARY
|
||||||
|
if [[ "${{ github.event.inputs.push_latest }}" == "true" ]]; then
|
||||||
|
echo "✅ **Regular Latest:** \`${{ env.IMAGE_NAME }}:latest\`" >> $GITHUB_STEP_SUMMARY
|
||||||
|
fi
|
||||||
|
elif [[ "${{ needs.build-regular.result }}" == "skipped" ]]; then
|
||||||
|
echo "⏭️ **Regular:** Skipped" >> $GITHUB_STEP_SUMMARY
|
||||||
|
else
|
||||||
|
echo "❌ **Regular:** Failed" >> $GITHUB_STEP_SUMMARY
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "${{ needs.build-single.result }}" == "success" ]]; then
|
||||||
|
echo "✅ **Single:** \`${{ env.IMAGE_NAME }}:${{ needs.extract-version.outputs.version }}-single\`" >> $GITHUB_STEP_SUMMARY
|
||||||
|
if [[ "${{ github.event.inputs.push_latest }}" == "true" ]]; then
|
||||||
|
echo "✅ **Single Latest:** \`${{ env.IMAGE_NAME }}:latest-single\`" >> $GITHUB_STEP_SUMMARY
|
||||||
|
fi
|
||||||
|
elif [[ "${{ needs.build-single.result }}" == "skipped" ]]; then
|
||||||
|
echo "⏭️ **Single:** Skipped" >> $GITHUB_STEP_SUMMARY
|
||||||
|
else
|
||||||
|
echo "❌ **Single:** Failed" >> $GITHUB_STEP_SUMMARY
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "" >> $GITHUB_STEP_SUMMARY
|
||||||
|
echo "### Platforms:" >> $GITHUB_STEP_SUMMARY
|
||||||
|
echo "- linux/amd64" >> $GITHUB_STEP_SUMMARY
|
||||||
|
echo "- linux/arm64" >> $GITHUB_STEP_SUMMARY
|
||||||
186
.github/workflows/build-dev.yml
vendored
Normal file
186
.github/workflows/build-dev.yml
vendored
Normal file
|
|
@ -0,0 +1,186 @@
|
||||||
|
name: Development Build
|
||||||
|
|
||||||
|
on:
|
||||||
|
pull_request:
|
||||||
|
branches: [ main ]
|
||||||
|
push:
|
||||||
|
branches: [ main ]
|
||||||
|
paths-ignore:
|
||||||
|
- '**.md'
|
||||||
|
- 'docs/**'
|
||||||
|
- 'notebooks/**'
|
||||||
|
- '.github/workflows/claude*.yml'
|
||||||
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
dockerfile:
|
||||||
|
description: 'Dockerfile to test'
|
||||||
|
required: true
|
||||||
|
default: 'both'
|
||||||
|
type: choice
|
||||||
|
options:
|
||||||
|
- both
|
||||||
|
- regular
|
||||||
|
- single
|
||||||
|
platform:
|
||||||
|
description: 'Platform to build'
|
||||||
|
required: true
|
||||||
|
default: 'linux/amd64'
|
||||||
|
type: choice
|
||||||
|
options:
|
||||||
|
- linux/amd64
|
||||||
|
- linux/arm64
|
||||||
|
- linux/amd64,linux/arm64
|
||||||
|
|
||||||
|
env:
|
||||||
|
REGISTRY: docker.io
|
||||||
|
IMAGE_NAME: lfnovo/open_notebook
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
extract-version:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
outputs:
|
||||||
|
version: ${{ steps.version.outputs.version }}
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Extract version from pyproject.toml
|
||||||
|
id: version
|
||||||
|
run: |
|
||||||
|
VERSION=$(grep -m1 '^version = ' pyproject.toml | cut -d'"' -f2)
|
||||||
|
echo "version=$VERSION" >> $GITHUB_OUTPUT
|
||||||
|
echo "Extracted version: $VERSION"
|
||||||
|
|
||||||
|
lint-and-check:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Set up Python
|
||||||
|
uses: actions/setup-python@v4
|
||||||
|
with:
|
||||||
|
python-version: '3.12'
|
||||||
|
|
||||||
|
- name: Install uv
|
||||||
|
uses: astral-sh/setup-uv@v3
|
||||||
|
|
||||||
|
- name: Install dependencies
|
||||||
|
run: uv sync --dev
|
||||||
|
|
||||||
|
- name: Run ruff
|
||||||
|
run: uv run ruff check . --output-format=github
|
||||||
|
|
||||||
|
- name: Run mypy
|
||||||
|
run: uv run python -m mypy .
|
||||||
|
|
||||||
|
test-build-regular:
|
||||||
|
needs: extract-version
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
if: github.event.inputs.dockerfile == 'regular' || github.event.inputs.dockerfile == 'both' || github.event_name != 'workflow_dispatch'
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Set up Docker Buildx
|
||||||
|
uses: docker/setup-buildx-action@v3
|
||||||
|
|
||||||
|
- name: Cache Docker layers
|
||||||
|
uses: actions/cache@v3
|
||||||
|
with:
|
||||||
|
path: /tmp/.buildx-cache-dev
|
||||||
|
key: ${{ runner.os }}-buildx-dev-regular-${{ github.sha }}
|
||||||
|
restore-keys: |
|
||||||
|
${{ runner.os }}-buildx-dev-regular-
|
||||||
|
|
||||||
|
- name: Build regular image (test only)
|
||||||
|
uses: docker/build-push-action@v5
|
||||||
|
with:
|
||||||
|
context: .
|
||||||
|
file: ./Dockerfile
|
||||||
|
platforms: ${{ github.event.inputs.platform || 'linux/amd64' }}
|
||||||
|
push: false
|
||||||
|
tags: ${{ env.IMAGE_NAME }}:${{ needs.extract-version.outputs.version }}-dev-regular
|
||||||
|
cache-from: type=local,src=/tmp/.buildx-cache-dev
|
||||||
|
cache-to: type=local,dest=/tmp/.buildx-cache-dev-new,mode=max
|
||||||
|
|
||||||
|
- name: Move cache
|
||||||
|
run: |
|
||||||
|
rm -rf /tmp/.buildx-cache-dev
|
||||||
|
mv /tmp/.buildx-cache-dev-new /tmp/.buildx-cache-dev
|
||||||
|
|
||||||
|
test-build-single:
|
||||||
|
needs: extract-version
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
if: github.event.inputs.dockerfile == 'single' || github.event.inputs.dockerfile == 'both' || github.event_name != 'workflow_dispatch'
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Set up Docker Buildx
|
||||||
|
uses: docker/setup-buildx-action@v3
|
||||||
|
|
||||||
|
- name: Cache Docker layers
|
||||||
|
uses: actions/cache@v3
|
||||||
|
with:
|
||||||
|
path: /tmp/.buildx-cache-dev-single
|
||||||
|
key: ${{ runner.os }}-buildx-dev-single-${{ github.sha }}
|
||||||
|
restore-keys: |
|
||||||
|
${{ runner.os }}-buildx-dev-single-
|
||||||
|
|
||||||
|
- name: Build single-container image (test only)
|
||||||
|
uses: docker/build-push-action@v5
|
||||||
|
with:
|
||||||
|
context: .
|
||||||
|
file: ./Dockerfile.single
|
||||||
|
platforms: ${{ github.event.inputs.platform || 'linux/amd64' }}
|
||||||
|
push: false
|
||||||
|
tags: ${{ env.IMAGE_NAME }}:${{ needs.extract-version.outputs.version }}-dev-single
|
||||||
|
cache-from: type=local,src=/tmp/.buildx-cache-dev-single
|
||||||
|
cache-to: type=local,dest=/tmp/.buildx-cache-dev-single-new,mode=max
|
||||||
|
|
||||||
|
- name: Move cache
|
||||||
|
run: |
|
||||||
|
rm -rf /tmp/.buildx-cache-dev-single
|
||||||
|
mv /tmp/.buildx-cache-dev-single-new /tmp/.buildx-cache-dev-single
|
||||||
|
|
||||||
|
summary:
|
||||||
|
needs: [extract-version, lint-and-check, test-build-regular, test-build-single]
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
if: always()
|
||||||
|
steps:
|
||||||
|
- name: Development Build Summary
|
||||||
|
run: |
|
||||||
|
echo "## Development Build Summary" >> $GITHUB_STEP_SUMMARY
|
||||||
|
echo "**Version:** ${{ needs.extract-version.outputs.version }}" >> $GITHUB_STEP_SUMMARY
|
||||||
|
echo "**Platform:** ${{ github.event.inputs.platform || 'linux/amd64' }}" >> $GITHUB_STEP_SUMMARY
|
||||||
|
echo "**Dockerfile:** ${{ github.event.inputs.dockerfile || 'both' }}" >> $GITHUB_STEP_SUMMARY
|
||||||
|
echo "" >> $GITHUB_STEP_SUMMARY
|
||||||
|
echo "### Results:" >> $GITHUB_STEP_SUMMARY
|
||||||
|
|
||||||
|
if [[ "${{ needs.lint-and-check.result }}" == "success" ]]; then
|
||||||
|
echo "✅ **Lint & Type Check:** Passed" >> $GITHUB_STEP_SUMMARY
|
||||||
|
else
|
||||||
|
echo "❌ **Lint & Type Check:** Failed" >> $GITHUB_STEP_SUMMARY
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "${{ needs.test-build-regular.result }}" == "success" ]]; then
|
||||||
|
echo "✅ **Regular Dockerfile:** Build successful" >> $GITHUB_STEP_SUMMARY
|
||||||
|
elif [[ "${{ needs.test-build-regular.result }}" == "skipped" ]]; then
|
||||||
|
echo "⏭️ **Regular Dockerfile:** Skipped" >> $GITHUB_STEP_SUMMARY
|
||||||
|
else
|
||||||
|
echo "❌ **Regular Dockerfile:** Build failed" >> $GITHUB_STEP_SUMMARY
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "${{ needs.test-build-single.result }}" == "success" ]]; then
|
||||||
|
echo "✅ **Single Dockerfile:** Build successful" >> $GITHUB_STEP_SUMMARY
|
||||||
|
elif [[ "${{ needs.test-build-single.result }}" == "skipped" ]]; then
|
||||||
|
echo "⏭️ **Single Dockerfile:** Skipped" >> $GITHUB_STEP_SUMMARY
|
||||||
|
else
|
||||||
|
echo "❌ **Single Dockerfile:** Build failed" >> $GITHUB_STEP_SUMMARY
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "" >> $GITHUB_STEP_SUMMARY
|
||||||
|
echo "### Notes:" >> $GITHUB_STEP_SUMMARY
|
||||||
|
echo "- This is a development build (no images pushed to registry)" >> $GITHUB_STEP_SUMMARY
|
||||||
|
echo "- For production releases, use the 'Build and Release' workflow" >> $GITHUB_STEP_SUMMARY
|
||||||
|
|
@ -1,178 +0,0 @@
|
||||||
import os
|
|
||||||
from contextlib import asynccontextmanager
|
|
||||||
from datetime import datetime, timezone
|
|
||||||
from typing import Any, Dict, List, Optional, TypeVar, Union
|
|
||||||
|
|
||||||
from loguru import logger
|
|
||||||
from surrealdb import AsyncSurreal, RecordID # type: ignore
|
|
||||||
|
|
||||||
T = TypeVar("T", Dict[str, Any], List[Dict[str, Any]])
|
|
||||||
|
|
||||||
|
|
||||||
def get_database_url():
|
|
||||||
"""Get database URL with backward compatibility"""
|
|
||||||
surreal_url = os.getenv("SURREAL_URL")
|
|
||||||
if surreal_url:
|
|
||||||
return surreal_url
|
|
||||||
|
|
||||||
# Fallback to old format - WebSocket URL format
|
|
||||||
address = os.getenv("SURREAL_ADDRESS", "localhost")
|
|
||||||
port = os.getenv("SURREAL_PORT", "8000")
|
|
||||||
return f"ws://{address}/rpc:{port}"
|
|
||||||
|
|
||||||
|
|
||||||
def get_database_password():
|
|
||||||
"""Get password with backward compatibility"""
|
|
||||||
return os.getenv("SURREAL_PASSWORD") or os.getenv("SURREAL_PASS")
|
|
||||||
|
|
||||||
|
|
||||||
def parse_record_ids(obj: Any) -> Any:
|
|
||||||
"""Recursively parse and convert RecordIDs into strings."""
|
|
||||||
if isinstance(obj, dict):
|
|
||||||
return {k: parse_record_ids(v) for k, v in obj.items()}
|
|
||||||
elif isinstance(obj, list):
|
|
||||||
return [parse_record_ids(item) for item in obj]
|
|
||||||
elif isinstance(obj, RecordID):
|
|
||||||
return str(obj)
|
|
||||||
return obj
|
|
||||||
|
|
||||||
|
|
||||||
def ensure_record_id(value: Union[str, RecordID]) -> RecordID:
|
|
||||||
"""Ensure a value is a RecordID."""
|
|
||||||
if isinstance(value, RecordID):
|
|
||||||
return value
|
|
||||||
return RecordID.parse(value)
|
|
||||||
|
|
||||||
|
|
||||||
@asynccontextmanager
|
|
||||||
async def db_connection():
|
|
||||||
db = AsyncSurreal(get_database_url())
|
|
||||||
await db.signin(
|
|
||||||
{
|
|
||||||
"username": os.environ["SURREAL_USER"],
|
|
||||||
"password": get_database_password(),
|
|
||||||
}
|
|
||||||
)
|
|
||||||
await db.use(os.environ["SURREAL_NAMESPACE"], os.environ["SURREAL_DATABASE"])
|
|
||||||
try:
|
|
||||||
yield db
|
|
||||||
finally:
|
|
||||||
await db.close()
|
|
||||||
|
|
||||||
|
|
||||||
async def repo_query(
|
|
||||||
query_str: str, vars: Optional[Dict[str, Any]] = None
|
|
||||||
) -> List[Dict[str, Any]]:
|
|
||||||
"""Execute a SurrealQL query and return the results"""
|
|
||||||
|
|
||||||
async with db_connection() as connection:
|
|
||||||
try:
|
|
||||||
result = parse_record_ids(await connection.query(query_str, vars))
|
|
||||||
if isinstance(result, str):
|
|
||||||
raise RuntimeError(result)
|
|
||||||
return result
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Query: {query_str[:200]} vars: {vars}")
|
|
||||||
logger.exception(e)
|
|
||||||
raise
|
|
||||||
|
|
||||||
|
|
||||||
async def repo_create(table: str, data: Dict[str, Any]) -> Dict[str, Any]:
|
|
||||||
"""Create a new record in the specified table"""
|
|
||||||
# Remove 'id' attribute if it exists in data
|
|
||||||
data.pop("id", None)
|
|
||||||
data["created"] = datetime.now(timezone.utc)
|
|
||||||
data["updated"] = datetime.now(timezone.utc)
|
|
||||||
try:
|
|
||||||
async with db_connection() as connection:
|
|
||||||
return parse_record_ids(await connection.insert(table, data))
|
|
||||||
except Exception as e:
|
|
||||||
logger.exception(e)
|
|
||||||
raise RuntimeError("Failed to create record")
|
|
||||||
|
|
||||||
|
|
||||||
async def repo_relate(
|
|
||||||
source: str, relationship: str, target: str, data: Optional[Dict[str, Any]] = None
|
|
||||||
) -> List[Dict[str, Any]]:
|
|
||||||
"""Create a relationship between two records with optional data"""
|
|
||||||
if data is None:
|
|
||||||
data = {}
|
|
||||||
query = f"RELATE {source}->{relationship}->{target} CONTENT $data;"
|
|
||||||
# logger.debug(f"Relate query: {query}")
|
|
||||||
|
|
||||||
return await repo_query(
|
|
||||||
query,
|
|
||||||
{
|
|
||||||
"data": data,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
async def repo_upsert(
|
|
||||||
table: str, id: Optional[str], data: Dict[str, Any], add_timestamp: bool = False
|
|
||||||
) -> List[Dict[str, Any]]:
|
|
||||||
"""Create or update a record in the specified table"""
|
|
||||||
data.pop("id", None)
|
|
||||||
if add_timestamp:
|
|
||||||
data["updated"] = datetime.now(timezone.utc)
|
|
||||||
query = f"UPSERT {id if id else table} MERGE $data;"
|
|
||||||
return await repo_query(query, {"data": data})
|
|
||||||
|
|
||||||
|
|
||||||
async def repo_update(
|
|
||||||
table: str, id: str, data: Dict[str, Any]
|
|
||||||
) -> List[Dict[str, Any]]:
|
|
||||||
"""Update an existing record by table and id"""
|
|
||||||
# If id already contains the table name, use it as is
|
|
||||||
try:
|
|
||||||
if isinstance(id, RecordID) or (":" in id and id.startswith(f"{table}:")):
|
|
||||||
record_id = id
|
|
||||||
else:
|
|
||||||
record_id = f"{table}:{id}"
|
|
||||||
|
|
||||||
data["updated"] = datetime.now(timezone.utc)
|
|
||||||
query = f"UPDATE {record_id} MERGE $data;"
|
|
||||||
# logger.debug(f"Update query: {query}")
|
|
||||||
result = await repo_query(query, {"data": data})
|
|
||||||
# if isinstance(result, list):
|
|
||||||
# return [_return_data(item) for item in result]
|
|
||||||
return [parse_record_ids(result)]
|
|
||||||
except Exception as e:
|
|
||||||
raise RuntimeError(f"Failed to update record: {str(e)}")
|
|
||||||
|
|
||||||
|
|
||||||
async def repo_get_news_by_jota_id(jota_id: str) -> Dict[str, Any]:
|
|
||||||
try:
|
|
||||||
results = await repo_query(
|
|
||||||
"SELECT * omit embedding FROM news where jota_id=$jota_id",
|
|
||||||
{"jota_id": jota_id},
|
|
||||||
)
|
|
||||||
return parse_record_ids(results)
|
|
||||||
except Exception as e:
|
|
||||||
logger.exception(e)
|
|
||||||
raise RuntimeError(f"Failed to fetch record: {str(e)}")
|
|
||||||
|
|
||||||
|
|
||||||
async def repo_delete(record_id: Union[str, RecordID]):
|
|
||||||
"""Delete a record by record id"""
|
|
||||||
|
|
||||||
try:
|
|
||||||
async with db_connection() as connection:
|
|
||||||
return await connection.delete(record_id)
|
|
||||||
except Exception as e:
|
|
||||||
logger.exception(e)
|
|
||||||
raise RuntimeError(f"Failed to delete record: {str(e)}")
|
|
||||||
|
|
||||||
|
|
||||||
async def repo_insert(
|
|
||||||
table: str, data: List[Dict[str, Any]], ignore_duplicates: bool = False
|
|
||||||
) -> List[Dict[str, Any]]:
|
|
||||||
"""Create a new record in the specified table"""
|
|
||||||
try:
|
|
||||||
async with db_connection() as connection:
|
|
||||||
return parse_record_ids(await connection.insert(table, data))
|
|
||||||
except Exception as e:
|
|
||||||
if ignore_duplicates and "already contains" in str(e):
|
|
||||||
return []
|
|
||||||
logger.exception(e)
|
|
||||||
raise RuntimeError("Failed to create record")
|
|
||||||
|
|
@ -49,11 +49,13 @@ async def db_connection():
|
||||||
db = AsyncSurreal(get_database_url())
|
db = AsyncSurreal(get_database_url())
|
||||||
await db.signin(
|
await db.signin(
|
||||||
{
|
{
|
||||||
"username": os.environ["SURREAL_USER"],
|
"username": os.environ.get("SURREAL_USER"),
|
||||||
"password": get_database_password(),
|
"password": get_database_password(),
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
await db.use(os.environ["SURREAL_NAMESPACE"], os.environ["SURREAL_DATABASE"])
|
await db.use(
|
||||||
|
os.environ.get("SURREAL_NAMESPACE"), os.environ.get("SURREAL_DATABASE")
|
||||||
|
)
|
||||||
try:
|
try:
|
||||||
yield db
|
yield db
|
||||||
finally:
|
finally:
|
||||||
|
|
|
||||||
|
|
@ -1,63 +0,0 @@
|
||||||
import os
|
|
||||||
from contextlib import contextmanager
|
|
||||||
from typing import Any, Dict, Optional
|
|
||||||
|
|
||||||
from loguru import logger
|
|
||||||
from sblpy.connection import SurrealSyncConnection
|
|
||||||
|
|
||||||
|
|
||||||
@contextmanager
|
|
||||||
def db_connection():
|
|
||||||
connection = SurrealSyncConnection(
|
|
||||||
host=os.environ["SURREAL_ADDRESS"],
|
|
||||||
port=int(os.environ["SURREAL_PORT"]),
|
|
||||||
user=os.environ["SURREAL_USER"],
|
|
||||||
password=os.environ["SURREAL_PASS"],
|
|
||||||
namespace=os.environ["SURREAL_NAMESPACE"],
|
|
||||||
database=os.environ["SURREAL_DATABASE"],
|
|
||||||
max_size=2.2**20,
|
|
||||||
encrypted=False, # Set to True if using SSL
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
yield connection
|
|
||||||
finally:
|
|
||||||
connection.socket.close()
|
|
||||||
|
|
||||||
|
|
||||||
def repo_query(query_str: str, vars: Optional[Dict[str, Any]] = None):
|
|
||||||
with db_connection() as connection:
|
|
||||||
try:
|
|
||||||
result = connection.query(query_str, vars)
|
|
||||||
return result
|
|
||||||
except Exception as e:
|
|
||||||
logger.critical(f"Query: {query_str}")
|
|
||||||
logger.exception(e)
|
|
||||||
raise
|
|
||||||
|
|
||||||
|
|
||||||
def repo_create(table: str, data: Dict[str, Any]):
|
|
||||||
query = f"CREATE {table} CONTENT {data};"
|
|
||||||
return repo_query(query)
|
|
||||||
|
|
||||||
|
|
||||||
def repo_upsert(table: str, data: Dict[str, Any]):
|
|
||||||
query = f"UPSERT {table} CONTENT {data};"
|
|
||||||
return repo_query(query)
|
|
||||||
|
|
||||||
|
|
||||||
def repo_update(id: str, data: Dict[str, Any]):
|
|
||||||
query = "UPDATE $id CONTENT $data;"
|
|
||||||
vars = {"id": id, "data": data}
|
|
||||||
return repo_query(query, vars)
|
|
||||||
|
|
||||||
|
|
||||||
def repo_delete(id: str):
|
|
||||||
query = "DELETE $id;"
|
|
||||||
vars = {"id": id}
|
|
||||||
return repo_query(query, vars)
|
|
||||||
|
|
||||||
|
|
||||||
def repo_relate(source: str, relationship: str, target: str, data: Optional[Dict] = {}):
|
|
||||||
query = f"RELATE {source}->{relationship}->{target} CONTENT $content;"
|
|
||||||
result = repo_query(query, {"content": data})
|
|
||||||
return result
|
|
||||||
|
|
@ -1,90 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
echo "=== Testing Surreal Commands Integration ==="
|
|
||||||
echo ""
|
|
||||||
|
|
||||||
# Base URL
|
|
||||||
BASE_URL="http://localhost:5055/api"
|
|
||||||
|
|
||||||
# 1. Test text processing command
|
|
||||||
echo "1. Testing text processing command (uppercase)..."
|
|
||||||
curl -X POST "$BASE_URL/commands/jobs" \
|
|
||||||
-H "Content-Type: application/json" \
|
|
||||||
-d '{
|
|
||||||
"command": "process_text",
|
|
||||||
"app": "open_notebook",
|
|
||||||
"input": {
|
|
||||||
"text": "Hello, this is a test message!",
|
|
||||||
"operation": "uppercase"
|
|
||||||
}
|
|
||||||
}' | jq .
|
|
||||||
|
|
||||||
echo ""
|
|
||||||
echo "2. Testing text processing with delay (3 seconds)..."
|
|
||||||
curl -X POST "$BASE_URL/commands/jobs" \
|
|
||||||
-H "Content-Type: application/json" \
|
|
||||||
-d '{
|
|
||||||
"command": "process_text",
|
|
||||||
"app": "open_notebook",
|
|
||||||
"input": {
|
|
||||||
"text": "Testing async behavior with delay",
|
|
||||||
"operation": "reverse",
|
|
||||||
"delay_seconds": 3
|
|
||||||
}
|
|
||||||
}' | jq .
|
|
||||||
|
|
||||||
echo ""
|
|
||||||
echo "3. Testing data analysis command..."
|
|
||||||
curl -X POST "$BASE_URL/commands/jobs" \
|
|
||||||
-H "Content-Type: application/json" \
|
|
||||||
-d '{
|
|
||||||
"command": "analyze_data",
|
|
||||||
"app": "open_notebook",
|
|
||||||
"input": {
|
|
||||||
"numbers": [10, 20, 30, 40, 50],
|
|
||||||
"analysis_type": "basic"
|
|
||||||
}
|
|
||||||
}' | jq .
|
|
||||||
|
|
||||||
echo ""
|
|
||||||
echo "4. Testing error scenario (empty numbers array)..."
|
|
||||||
curl -X POST "$BASE_URL/commands/jobs" \
|
|
||||||
-H "Content-Type: application/json" \
|
|
||||||
-d '{
|
|
||||||
"command": "analyze_data",
|
|
||||||
"app": "open_notebook",
|
|
||||||
"input": {
|
|
||||||
"numbers": [],
|
|
||||||
"analysis_type": "basic"
|
|
||||||
}
|
|
||||||
}' | jq .
|
|
||||||
|
|
||||||
echo ""
|
|
||||||
echo "5. Testing word count operation..."
|
|
||||||
curl -X POST "$BASE_URL/commands/jobs" \
|
|
||||||
-H "Content-Type: application/json" \
|
|
||||||
-d '{
|
|
||||||
"command": "process_text",
|
|
||||||
"app": "open_notebook",
|
|
||||||
"input": {
|
|
||||||
"text": "This is a sample text with multiple words to count",
|
|
||||||
"operation": "word_count"
|
|
||||||
}
|
|
||||||
}' | jq .
|
|
||||||
|
|
||||||
echo ""
|
|
||||||
echo "Please save the job_ids from above to check status!"
|
|
||||||
echo ""
|
|
||||||
echo "6. To check job status (replace JOB_ID with actual ID):"
|
|
||||||
echo "curl \"$BASE_URL/commands/jobs/{JOB_ID}\" | jq ."
|
|
||||||
|
|
||||||
echo ""
|
|
||||||
echo "7. To list all jobs:"
|
|
||||||
echo "curl \"$BASE_URL/commands/jobs\" | jq ."
|
|
||||||
|
|
||||||
echo ""
|
|
||||||
echo "=== Test Commands Complete ==="
|
|
||||||
echo ""
|
|
||||||
echo "Manual status check example:"
|
|
||||||
echo "Replace JOB_ID with one of the job IDs returned above:"
|
|
||||||
echo "curl \"$BASE_URL/commands/jobs/JOB_ID\" | jq ."
|
|
||||||
Loading…
Reference in a new issue