From 45a99831a90a27bd7ce6c494ddc33f984b888248 Mon Sep 17 00:00:00 2001 From: Luis Novo Date: Tue, 25 Nov 2025 16:59:26 -0300 Subject: [PATCH] Hide sources notes (#273) * fix: add missing overflow wrapper to notebooks list page Adds flex-1 overflow-y-auto wrapper to enable proper scrolling when notebook list exceeds viewport height. Matches the layout pattern used by all other dashboard pages. Co-Authored-By: Claude * fix: reorder transformation routes to prevent dynamic route interception Moved static routes (/transformations/execute and /transformations/default-prompt) before dynamic routes (/transformations/{transformation_id}) to ensure FastAPI matches them correctly. Previously, requests to static routes were incorrectly captured by the dynamic route handler. Fixes #250 Co-Authored-By: Claude * chore: bump to 1.2.1 * hide source and notes panel - fixes #193 * feat: improve layout for mobile views * bump version to 1.2.2 * fix: address PR review feedback for collapsible columns - Remove unused CollapseButton component from CollapsibleColumn.tsx - Rename useCollapseButton to createCollapseButton (not a React hook) - Move dialogs outside Card in SourcesColumn.tsx for consistency - Add useMemo for collapseButton in both columns to prevent re-renders * feat: support multiple sources * fix: prevent ChatColumn double mounting on desktop Add useIsDesktop hook to conditionally render mobile view only on mobile screens. Previously, the mobile ChatColumn was hidden via CSS on desktop but still mounted, causing duplicate hooks initialization and redundant network requests. --------- Co-authored-by: Claude --- .github/README.md | 191 ----- commands/source_commands.py | 19 +- .../app/(dashboard)/notebooks/[id]/page.tsx | 124 ++- .../notebooks/components/NotesColumn.tsx | 231 +++--- .../notebooks/components/SourcesColumn.tsx | 144 ++-- .../notebooks/CollapsibleColumn.tsx | 93 +++ .../components/sources/AddSourceDialog.tsx | 273 ++++++- .../sources/steps/SourceTypeStep.tsx | 204 ++++- frontend/src/lib/hooks/use-media-query.ts | 32 + .../src/lib/stores/notebook-columns-store.ts | 27 + open_notebook/database/repository.py | 19 +- open_notebook/domain/base.py | 3 + pyproject.toml | 2 +- uv.lock | 753 +++++++++--------- 14 files changed, 1285 insertions(+), 830 deletions(-) delete mode 100644 .github/README.md create mode 100644 frontend/src/components/notebooks/CollapsibleColumn.tsx create mode 100644 frontend/src/lib/hooks/use-media-query.ts create mode 100644 frontend/src/lib/stores/notebook-columns-store.ts diff --git a/.github/README.md b/.github/README.md deleted file mode 100644 index 66f0a27..0000000 --- a/.github/README.md +++ /dev/null @@ -1,191 +0,0 @@ -# GitHub Workflow & Templates - -This folder contains GitHub-specific configuration for Open Notebook's contribution workflow. - -## 📋 Issue Templates - -We have three issue templates to guide contributors: - -### 🐛 Bug Report (`ISSUE_TEMPLATE/bug_report.yml`) - -For reporting bugs or unexpected behavior when the app is running but misbehaving. - -**Key Features:** -- Structured format for bug details -- Environment and version information -- Checkbox for contributors who want to fix the issue -- Automatic `bug` and `needs-triage` labels - -**When to Use:** App is installed and running, but something doesn't work as expected. - -### ✨ Feature Request (`ISSUE_TEMPLATE/feature_request.yml`) - -For suggesting new features or improvements. - -**Key Features:** -- Description of the feature -- Explanation of why it would be helpful -- Space for proposed solution -- Checkbox for contributors who want to implement it -- Automatic `enhancement` and `needs-triage` labels - -**When to Use:** You have an idea for a new feature or improvement. - -### 🔧 Installation Issue (`ISSUE_TEMPLATE/installation_issue.yml`) - -For problems with installation or setup. - -**Key Features:** -- Deployment type selection -- Environment details -- Error message collection -- Automatic `installation` label - -**When to Use:** Having trouble getting Open Notebook running. - -## 🔄 Pull Request Template - -The PR template (`pull_request_template.md`) ensures contributors provide all necessary information: - -**Sections:** -- Description and related issue -- Type of change -- Testing details -- Design alignment with project principles -- Comprehensive checklist for code quality, testing, documentation -- Screenshots for UI changes - -**Key Checkpoints:** -- References an approved issue -- Aligns with [design principles](../DESIGN_PRINCIPLES.md) -- Includes tests and documentation -- Follows code style guidelines - -## 🚀 GitHub Actions - -Our CI/CD workflows in `.github/workflows/`: - -### `build-and-release.yml` -- Builds Docker images on releases -- Publishes to Docker Hub and GitHub Container Registry -- Supports multi-platform builds (amd64, arm64) - -### `build-dev.yml` -- Builds dev images on pushes to main -- Tags with commit SHA for testing - -### `claude-code-review.yml` -- Automated code review using Claude Code -- Runs on pull requests -- Provides AI-powered suggestions - -## 📖 How the Contribution Flow Works - -``` -┌─────────────────────────────────────────────────────────┐ -│ 1. Contributor identifies a bug or has a feature idea │ -└────────────────────┬────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────┐ -│ 2. Creates an issue using appropriate template │ -│ - Describes the problem/feature │ -│ - Checks "I am a developer..." if willing to work │ -└────────────────────┬────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────┐ -│ 3. Maintainer reviews issue (within 48 hours) │ -│ - Assesses alignment with design principles │ -│ - Labels appropriately │ -│ - Asks for clarification if needed │ -└────────────────────┬────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────┐ -│ 4. If approved: Contributor proposes solution approach │ -│ - Discusses implementation strategy │ -│ - Maintainer provides feedback │ -└────────────────────┬────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────┐ -│ 5. Issue is assigned to contributor │ -│ - Contributor forks repo │ -│ - Creates feature branch from main │ -└────────────────────┬────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────┐ -│ 6. Contributor develops solution │ -│ - Reads DESIGN_PRINCIPLES.md │ -│ - Reads docs/development/architecture.md │ -│ - Writes code, tests, documentation │ -└────────────────────┬────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────┐ -│ 7. Creates Pull Request │ -│ - Uses PR template │ -│ - References issue number │ -│ - Fills out all checklist items │ -└────────────────────┬────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────┐ -│ 8. Maintainer reviews PR │ -│ - Checks code quality │ -│ - Verifies tests pass │ -│ - Ensures alignment with architecture │ -│ - Provides feedback or approves │ -└────────────────────┬────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────┐ -│ 9. If approved: PR is merged! 🎉 │ -│ - Contributor is thanked │ -│ - Issue is closed │ -│ - Changes included in next release │ -└─────────────────────────────────────────────────────────┘ -``` - -## 🎯 Why This Process? - -### Prevents Wasted Effort -- Contributors don't spend time on code that won't be merged -- Ensures alignment with project vision before coding starts - -### Maintains Quality -- All code reviewed against design principles -- Consistent architecture across contributions -- Proper testing and documentation - -### Respects Everyone's Time -- Clear expectations upfront -- Structured feedback process -- Efficient review process - -### Protects Project Vision -- Maintainers can guide direction -- Features align with long-term goals -- Technical debt is minimized - -## 📚 Related Documentation - -For contributors: -- [CONTRIBUTING.md](../CONTRIBUTING.md) - Contribution guidelines -- [DESIGN_PRINCIPLES.md](../DESIGN_PRINCIPLES.md) - Project vision and principles -- [docs/development/architecture.md](../docs/development/architecture.md) - Technical architecture - -For maintainers: -- [MAINTAINER_GUIDE.md](../MAINTAINER_GUIDE.md) - How to review and manage contributions - -## 🤝 Questions? - -- Join our [Discord](https://discord.gg/37XJPXfz2w) -- Open a [Discussion](https://github.com/lfnovo/open-notebook/discussions) -- Read the [FAQ](../docs/troubleshooting/faq.md) - ---- - -**Thank you for contributing to Open Notebook!** Your contributions help make this the best open-source research tool available. diff --git a/commands/source_commands.py b/commands/source_commands.py index 0f862b4..5385921 100644 --- a/commands/source_commands.py +++ b/commands/source_commands.py @@ -44,7 +44,17 @@ class SourceProcessingOutput(CommandOutput): error_message: Optional[str] = None -@command("process_source", app="open_notebook") +@command( + "process_source", + app="open_notebook", + retry={ + "max_attempts": 5, + "wait_strategy": "exponential_jitter", + "wait_min": 1, + "wait_max": 30, + "retry_on": [RuntimeError], + }, +) async def process_source_command( input_data: SourceProcessingInput, ) -> SourceProcessingOutput: @@ -124,10 +134,15 @@ async def process_source_command( processing_time=processing_time, ) + except RuntimeError as e: + # Transaction conflicts should be retried by surreal-commands + logger.warning(f"Transaction conflict, will retry: {e}") + raise + except Exception as e: + # Other errors are permanent failures processing_time = time.time() - start_time logger.error(f"Source processing failed: {e}") - logger.exception(e) return SourceProcessingOutput( success=False, diff --git a/frontend/src/app/(dashboard)/notebooks/[id]/page.tsx b/frontend/src/app/(dashboard)/notebooks/[id]/page.tsx index d336e35..7f598e2 100644 --- a/frontend/src/app/(dashboard)/notebooks/[id]/page.tsx +++ b/frontend/src/app/(dashboard)/notebooks/[id]/page.tsx @@ -11,6 +11,11 @@ import { useNotebook } from '@/lib/hooks/use-notebooks' import { useSources } from '@/lib/hooks/use-sources' import { useNotes } from '@/lib/hooks/use-notes' import { LoadingSpinner } from '@/components/common/LoadingSpinner' +import { useNotebookColumnsStore } from '@/lib/stores/notebook-columns-store' +import { useIsDesktop } from '@/lib/hooks/use-media-query' +import { cn } from '@/lib/utils' +import { Tabs, TabsList, TabsTrigger } from '@/components/ui/tabs' +import { FileText, StickyNote, MessageSquare } from 'lucide-react' export type ContextMode = 'off' | 'insights' | 'full' @@ -29,6 +34,15 @@ export default function NotebookPage() { const { data: sources, isLoading: sourcesLoading, refetch: refetchSources } = useSources(notebookId) const { data: notes, isLoading: notesLoading } = useNotes(notebookId) + // Get collapse states for dynamic layout + const { sourcesCollapsed, notesCollapsed } = useNotebookColumnsStore() + + // Detect desktop to avoid double-mounting ChatColumn + const isDesktop = useIsDesktop() + + // Mobile tab state (Sources, Notes, or Chat) + const [mobileActiveTab, setMobileActiveTab] = useState<'sources' | 'notes' | 'chat'>('chat') + // Context selection state const [contextSelections, setContextSelections] = useState({ sources: {}, @@ -105,32 +119,98 @@ export default function NotebookPage() { -
-
-
-
- handleContextModeChange(sourceId, mode, 'source')} - /> +
+ {/* Mobile: Tabbed interface - only render on mobile to avoid double-mounting */} + {!isDesktop && ( + <> +
+ setMobileActiveTab(value as 'sources' | 'notes' | 'chat')}> + + + + Sources + + + + Notes + + + + Chat + + +
-
- handleContextModeChange(noteId, mode, 'note')} - /> + + {/* Mobile: Show only active tab */} +
+ {mobileActiveTab === 'sources' && ( + handleContextModeChange(sourceId, mode, 'source')} + /> + )} + {mobileActiveTab === 'notes' && ( + handleContextModeChange(noteId, mode, 'note')} + /> + )} + {mobileActiveTab === 'chat' && ( + + )}
+ + )} + + {/* Desktop: Collapsible columns layout */} +
+ {/* Sources Column */} +
+ handleContextModeChange(sourceId, mode, 'source')} + />
-
+ {/* Notes Column */} +
+ handleContextModeChange(noteId, mode, 'note')} + /> +
+ + {/* Chat Column - always expanded, takes remaining space */} +
createCollapseButton(toggleNotes, 'Notes'), + [toggleNotes] + ) + const handleDeleteClick = (noteId: string) => { setNoteToDelete(noteId) setDeleteDialogOpen(true) @@ -62,113 +71,123 @@ export function NotesColumn({ return ( <> - - -
- Notes - -
- - - - {isLoading ? ( -
- -
- ) : !notes || notes.length === 0 ? ( - - ) : ( -
- {notes.map((note) => ( -
setEditingNote(note)} + + + +
+ Notes +
+ - - - { - e.stopPropagation() - handleDeleteClick(note.id) - }} - className="text-red-600 focus:text-red-600" - > - - Delete Note - - - -
-
- - {note.title && ( -

{note.title}

- )} - - {note.content && ( -

- {note.content} -

- )} -
- ))} + + Write Note + + {collapseButton} +
- )} - - + + + + {isLoading ? ( +
+ +
+ ) : !notes || notes.length === 0 ? ( + + ) : ( +
+ {notes.map((note) => ( +
setEditingNote(note)} + > +
+
+ {note.note_type === 'ai' ? ( + + ) : ( + + )} + + {note.note_type === 'ai' ? 'AI Generated' : 'Human'} + +
+ +
+ + {formatDistanceToNow(new Date(note.updated), { addSuffix: true })} + + + {/* Context toggle - only show if handler provided */} + {onContextModeChange && contextSelections?.[note.id] && ( +
event.stopPropagation()}> + onContextModeChange(note.id, mode)} + /> +
+ )} + + {/* Ellipsis menu for delete action */} + + + + + + { + e.stopPropagation() + handleDeleteClick(note.id) + }} + className="text-red-600 focus:text-red-600" + > + + Delete Note + + + +
+
+ + {note.title && ( +

{note.title}

+ )} + + {note.content && ( +

+ {note.content} +

+ )} +
+ ))} +
+ )} + + + createCollapseButton(toggleSources, 'Sources'), + [toggleSources] + ) const handleDeleteClick = (sourceId: string) => { setSourceToDelete(sourceId) @@ -102,67 +111,80 @@ export function SourcesColumn({ const handleSourceClick = (sourceId: string) => { openModal('source', sourceId) } - return ( - - -
- Sources - - - - - - { setDropdownOpen(false); setAddDialogOpen(true); }}> - - Add New Source - - { setDropdownOpen(false); setAddExistingDialogOpen(true); }}> - - Add Existing Source - - - -
- - - {isLoading ? ( -
- -
- ) : !sources || sources.length === 0 ? ( - - ) : ( -
- {sources.map((source) => ( - onContextModeChange(source.id, mode) - : undefined - } + return ( + <> + + + +
+ Sources +
+ + + + + + { setDropdownOpen(false); setAddDialogOpen(true); }}> + + Add New Source + + { setDropdownOpen(false); setAddExistingDialogOpen(true); }}> + + Add Existing Source + + + + {collapseButton} +
+
+ + + + {isLoading ? ( +
+ +
+ ) : !sources || sources.length === 0 ? ( + - ))} -
- )} - - + ) : ( +
+ {sources.map((source) => ( + onContextModeChange(source.id, mode) + : undefined + } + /> + ))} +
+ )} + + + + - + ) } diff --git a/frontend/src/components/notebooks/CollapsibleColumn.tsx b/frontend/src/components/notebooks/CollapsibleColumn.tsx new file mode 100644 index 0000000..3dc3b21 --- /dev/null +++ b/frontend/src/components/notebooks/CollapsibleColumn.tsx @@ -0,0 +1,93 @@ +'use client' + +import { ReactNode } from 'react' +import { Button } from '@/components/ui/button' +import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from '@/components/ui/tooltip' +import { ChevronLeft, LucideIcon } from 'lucide-react' +import { cn } from '@/lib/utils' + +interface CollapsibleColumnProps { + isCollapsed: boolean + onToggle: () => void + collapsedIcon: LucideIcon + collapsedLabel: string + children: ReactNode +} + +export function CollapsibleColumn({ + isCollapsed, + onToggle, + collapsedIcon: CollapsedIcon, + collapsedLabel, + children, +}: CollapsibleColumnProps) { + if (isCollapsed) { + return ( + + + + + + +

Expand {collapsedLabel}

+ + + + ) + } + + return ( +
+ {children} +
+ ) +} + +// Factory function to create a collapse button for card headers +export function createCollapseButton(onToggle: () => void, label: string) { + return ( +
+ + + + + + +

Collapse {label}

+ + + +
+ ) +} diff --git a/frontend/src/components/sources/AddSourceDialog.tsx b/frontend/src/components/sources/AddSourceDialog.tsx index e4fbee6..4db8d51 100644 --- a/frontend/src/components/sources/AddSourceDialog.tsx +++ b/frontend/src/components/sources/AddSourceDialog.tsx @@ -1,10 +1,11 @@ 'use client' -import { useState, useRef, useEffect } from 'react' +import { useState, useRef, useEffect, useMemo } from 'react' import { useForm } from 'react-hook-form' import { zodResolver } from '@hookform/resolvers/zod' import { z } from 'zod' -import { LoaderIcon } from 'lucide-react' +import { LoaderIcon, CheckCircleIcon, XCircleIcon } from 'lucide-react' +import { toast } from 'sonner' import { Dialog, DialogContent, @@ -14,7 +15,7 @@ import { } from '@/components/ui/dialog' import { Button } from '@/components/ui/button' import { WizardContainer, WizardStep } from '@/components/ui/wizard-container' -import { SourceTypeStep } from './steps/SourceTypeStep' +import { SourceTypeStep, parseAndValidateUrls } from './steps/SourceTypeStep' import { NotebooksStep } from './steps/NotebooksStep' import { ProcessingStep } from './steps/ProcessingStep' import { useNotebooks } from '@/lib/hooks/use-notebooks' @@ -23,6 +24,8 @@ import { useCreateSource } from '@/lib/hooks/use-sources' import { useSettings } from '@/lib/hooks/use-settings' import { CreateSourceRequest } from '@/lib/types/api' +const MAX_BATCH_SIZE = 50 + const createSourceSchema = z.object({ type: z.enum(['link', 'upload', 'text']), title: z.string().optional(), @@ -80,6 +83,13 @@ interface ProcessingState { progress?: number } +interface BatchProgress { + total: number + completed: number + failed: number + currentItem?: string +} + export function AddSourceDialog({ open, onOpenChange, @@ -94,6 +104,10 @@ export function AddSourceDialog({ ) const [selectedTransformations, setSelectedTransformations] = useState([]) + // Batch-specific state + const [urlValidationErrors, setUrlValidationErrors] = useState<{ url: string; line: number }[]>([]) + const [batchProgress, setBatchProgress] = useState(null) + // Cleanup timeouts to prevent memory leaks const timeoutRef = useRef(null) @@ -158,12 +172,51 @@ export function AddSourceDialog({ const watchedFile = watch('file') const watchedTitle = watch('title') + // Batch mode detection + const { isBatchMode, itemCount, parsedUrls, parsedFiles } = useMemo(() => { + let urlCount = 0 + let fileCount = 0 + let parsedUrls: string[] = [] + let parsedFiles: File[] = [] + + if (selectedType === 'link' && watchedUrl) { + const { valid } = parseAndValidateUrls(watchedUrl) + parsedUrls = valid + urlCount = valid.length + } + + if (selectedType === 'upload' && watchedFile) { + const fileList = watchedFile as FileList + if (fileList?.length) { + parsedFiles = Array.from(fileList) + fileCount = parsedFiles.length + } + } + + const isBatchMode = urlCount > 1 || fileCount > 1 + const itemCount = selectedType === 'link' ? urlCount : fileCount + + return { isBatchMode, itemCount, parsedUrls, parsedFiles } + }, [selectedType, watchedUrl, watchedFile]) + + // Check for batch size limit + const isOverLimit = itemCount > MAX_BATCH_SIZE + // Step validation - now reactive with watched values const isStepValid = (step: number): boolean => { switch (step) { case 1: if (!selectedType) return false + // Check batch size limit + if (isOverLimit) return false + // Check for URL validation errors + if (urlValidationErrors.length > 0) return false + if (selectedType === 'link') { + // In batch mode, check that we have at least one valid URL + if (isBatchMode) { + return parsedUrls.length > 0 + } return !!watchedUrl && watchedUrl.trim() !== '' } if (selectedType === 'text') { @@ -172,7 +225,7 @@ export function AddSourceDialog({ } if (selectedType === 'upload') { if (watchedFile instanceof FileList) { - return watchedFile.length > 0 + return watchedFile.length > 0 && watchedFile.length <= MAX_BATCH_SIZE } return !!watchedFile } @@ -189,11 +242,27 @@ export function AddSourceDialog({ const handleNextStep = (e?: React.MouseEvent) => { e?.preventDefault() e?.stopPropagation() + + // Validate URLs when leaving step 1 in link mode + if (currentStep === 1 && selectedType === 'link' && watchedUrl) { + const { invalid } = parseAndValidateUrls(watchedUrl) + if (invalid.length > 0) { + setUrlValidationErrors(invalid) + return + } + setUrlValidationErrors([]) + } + if (currentStep < 3 && isStepValid(currentStep)) { setCurrentStep(currentStep + 1) } } + // Clear URL validation errors when user edits + const handleClearUrlErrors = () => { + setUrlValidationErrors([]) + } + const handlePrevStep = (e?: React.MouseEvent) => { e?.preventDefault() e?.stopPropagation() @@ -223,43 +292,127 @@ export function AddSourceDialog({ setSelectedTransformations(updated) } + // Single source submission + const submitSingleSource = async (data: CreateSourceFormData): Promise => { + const createRequest: CreateSourceRequest = { + type: data.type, + notebooks: selectedNotebooks, + url: data.type === 'link' ? data.url : undefined, + content: data.type === 'text' ? data.content : undefined, + title: data.title, + transformations: selectedTransformations, + embed: data.embed, + delete_source: false, + async_processing: true, + } + + if (data.type === 'upload' && data.file) { + const file = data.file instanceof FileList ? data.file[0] : data.file + const requestWithFile = createRequest as CreateSourceRequest & { file?: File } + requestWithFile.file = file + } + + await createSource.mutateAsync(createRequest) + } + + // Batch submission + const submitBatch = async (data: CreateSourceFormData): Promise<{ success: number; failed: number }> => { + const results = { success: 0, failed: 0 } + const items: { type: 'url' | 'file'; value: string | File }[] = [] + + // Collect items to process + if (data.type === 'link' && parsedUrls.length > 0) { + parsedUrls.forEach(url => items.push({ type: 'url', value: url })) + } else if (data.type === 'upload' && parsedFiles.length > 0) { + parsedFiles.forEach(file => items.push({ type: 'file', value: file })) + } + + setBatchProgress({ + total: items.length, + completed: 0, + failed: 0, + }) + + // Process each item sequentially + for (let i = 0; i < items.length; i++) { + const item = items[i] + const itemLabel = item.type === 'url' + ? (item.value as string).substring(0, 50) + '...' + : (item.value as File).name + + setBatchProgress(prev => prev ? { + ...prev, + currentItem: itemLabel, + } : null) + + try { + const createRequest: CreateSourceRequest = { + type: item.type === 'url' ? 'link' : 'upload', + notebooks: selectedNotebooks, + url: item.type === 'url' ? item.value as string : undefined, + transformations: selectedTransformations, + embed: data.embed, + delete_source: false, + async_processing: true, + } + + if (item.type === 'file') { + const requestWithFile = createRequest as CreateSourceRequest & { file?: File } + requestWithFile.file = item.value as File + } + + await createSource.mutateAsync(createRequest) + results.success++ + } catch (error) { + console.error(`Error creating source for ${itemLabel}:`, error) + results.failed++ + } + + setBatchProgress(prev => prev ? { + ...prev, + completed: results.success, + failed: results.failed, + } : null) + } + + return results + } + // Form submission const onSubmit = async (data: CreateSourceFormData) => { try { setProcessing(true) - setProcessingStatus({ message: 'Submitting source for processing...' }) - const createRequest: CreateSourceRequest = { - type: data.type, - notebooks: selectedNotebooks, - url: data.type === 'link' ? data.url : undefined, - content: data.type === 'text' ? data.content : undefined, - title: data.title, - transformations: selectedTransformations, - embed: data.embed, - delete_source: false, - async_processing: true, // Always use async processing for frontend submissions + if (isBatchMode) { + // Batch submission + setProcessingStatus({ message: `Processing ${itemCount} sources...` }) + const results = await submitBatch(data) + + // Show summary toast + if (results.failed === 0) { + toast.success(`${results.success} source${results.success !== 1 ? 's' : ''} created successfully`) + } else if (results.success === 0) { + toast.error(`Failed to create all ${results.failed} sources`) + } else { + toast.warning(`${results.success} succeeded, ${results.failed} failed`) + } + + handleClose() + } else { + // Single source submission + setProcessingStatus({ message: 'Submitting source for processing...' }) + await submitSingleSource(data) + handleClose() } - - - if (data.type === 'upload' && data.file) { - const file = data.file instanceof FileList ? data.file[0] : data.file - const requestWithFile = createRequest as CreateSourceRequest & { file?: File } - requestWithFile.file = file - } - - await createSource.mutateAsync(createRequest) - - // Close immediately - the toast will show the success message - handleClose() } catch (error) { console.error('Error creating source:', error) - setProcessingStatus({ + setProcessingStatus({ message: 'Error creating source. Please try again.', }) timeoutRef.current = setTimeout(() => { setProcessing(false) setProcessingStatus(null) + setBatchProgress(null) }, 3000) } } @@ -277,6 +430,8 @@ export function AddSourceDialog({ setProcessing(false) setProcessingStatus(null) setSelectedNotebooks(defaultNotebookId ? [defaultNotebookId] : []) + setUrlValidationErrors([]) + setBatchProgress(null) // Reset to default transformations if (transformations.length > 0) { @@ -293,16 +448,25 @@ export function AddSourceDialog({ // Processing view if (processing) { + const progressPercent = batchProgress + ? Math.round(((batchProgress.completed + batchProgress.failed) / batchProgress.total) * 100) + : undefined + return ( - Processing Source + + {batchProgress ? 'Processing Batch' : 'Processing Source'} + - Your source is being processed. This may take a few moments. + {batchProgress + ? `Processing ${batchProgress.total} sources. This may take a few moments.` + : 'Your source is being processed. This may take a few moments.' + } - +
@@ -310,11 +474,48 @@ export function AddSourceDialog({ {processingStatus?.message || 'Processing...'}
- - {processingStatus?.progress && ( + + {/* Batch progress */} + {batchProgress && ( + <> +
+
+
+ +
+
+ + + {batchProgress.completed} completed + + {batchProgress.failed > 0 && ( + + + {batchProgress.failed} failed + + )} +
+ + {batchProgress.completed + batchProgress.failed} / {batchProgress.total} + +
+ + {batchProgress.currentItem && ( +

+ Current: {batchProgress.currentItem} +

+ )} + + )} + + {/* Single source progress */} + {!batchProgress && processingStatus?.progress && (
-
@@ -351,6 +552,8 @@ export function AddSourceDialog({ register={register} // @ts-expect-error - Type inference issue with zod schema errors={errors} + urlValidationErrors={urlValidationErrors} + onClearUrlErrors={handleClearUrlErrors} /> )} diff --git a/frontend/src/components/sources/steps/SourceTypeStep.tsx b/frontend/src/components/sources/steps/SourceTypeStep.tsx index b049e9f..c3c609d 100644 --- a/frontend/src/components/sources/steps/SourceTypeStep.tsx +++ b/frontend/src/components/sources/steps/SourceTypeStep.tsx @@ -1,5 +1,6 @@ "use client" +import { useMemo } from "react" import { Control, FieldErrors, UseFormRegister, useWatch } from "react-hook-form" import { FileIcon, LinkIcon, FileTextIcon } from "lucide-react" import { FormSection } from "@/components/ui/form-section" @@ -7,6 +8,7 @@ import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs" import { Input } from "@/components/ui/input" import { Textarea } from "@/components/ui/textarea" import { Label } from "@/components/ui/label" +import { Badge } from "@/components/ui/badge" import { Controller } from "react-hook-form" interface CreateSourceFormData { @@ -21,6 +23,45 @@ interface CreateSourceFormData { async_processing: boolean } +// Helper functions for batch URL parsing +function parseUrls(text: string): string[] { + return text + .split('\n') + .map(line => line.trim()) + .filter(line => line.length > 0) +} + +function validateUrl(url: string): boolean { + try { + new URL(url) + return true + } catch { + return false + } +} + +export function parseAndValidateUrls(text: string): { + valid: string[] + invalid: { url: string; line: number }[] +} { + const lines = text.split('\n') + const valid: string[] = [] + const invalid: { url: string; line: number }[] = [] + + lines.forEach((line, index) => { + const trimmed = line.trim() + if (trimmed.length === 0) return // skip empty lines + + if (validateUrl(trimmed)) { + valid.push(trimmed) + } else { + invalid.push({ url: trimmed, line: index + 1 }) + } + }) + + return { valid, invalid } +} + const SOURCE_TYPES = [ { value: 'link' as const, @@ -46,11 +87,41 @@ interface SourceTypeStepProps { control: Control register: UseFormRegister errors: FieldErrors + urlValidationErrors?: { url: string; line: number }[] + onClearUrlErrors?: () => void } -export function SourceTypeStep({ control, register, errors }: SourceTypeStepProps) { - // Watch the selected type to make title conditional +const MAX_BATCH_SIZE = 50 + +export function SourceTypeStep({ control, register, errors, urlValidationErrors, onClearUrlErrors }: SourceTypeStepProps) { + // Watch the selected type and inputs to detect batch mode const selectedType = useWatch({ control, name: 'type' }) + const urlInput = useWatch({ control, name: 'url' }) + const fileInput = useWatch({ control, name: 'file' }) + + // Batch mode detection + const { isBatchMode, itemCount, urlCount, fileCount } = useMemo(() => { + let urlCount = 0 + let fileCount = 0 + + if (selectedType === 'link' && urlInput) { + const urls = parseUrls(urlInput) + urlCount = urls.length + } + + if (selectedType === 'upload' && fileInput) { + const fileList = fileInput as FileList + fileCount = fileList?.length || 0 + } + + const isBatchMode = urlCount > 1 || fileCount > 1 + const itemCount = selectedType === 'link' ? urlCount : fileCount + + return { isBatchMode, itemCount, urlCount, fileCount } + }, [selectedType, urlInput, fileInput]) + + // Check for batch size limit + const isOverLimit = itemCount > MAX_BATCH_SIZE return (
- - + + {urlCount > 0 && ( + + {urlCount} URL{urlCount !== 1 ? 's' : ''} + {isOverLimit && ` (max ${MAX_BATCH_SIZE})`} + + )} +
+