diff --git a/.github/README.md b/.github/README.md deleted file mode 100644 index 66f0a27..0000000 --- a/.github/README.md +++ /dev/null @@ -1,191 +0,0 @@ -# GitHub Workflow & Templates - -This folder contains GitHub-specific configuration for Open Notebook's contribution workflow. - -## 📋 Issue Templates - -We have three issue templates to guide contributors: - -### 🐛 Bug Report (`ISSUE_TEMPLATE/bug_report.yml`) - -For reporting bugs or unexpected behavior when the app is running but misbehaving. - -**Key Features:** -- Structured format for bug details -- Environment and version information -- Checkbox for contributors who want to fix the issue -- Automatic `bug` and `needs-triage` labels - -**When to Use:** App is installed and running, but something doesn't work as expected. - -### ✨ Feature Request (`ISSUE_TEMPLATE/feature_request.yml`) - -For suggesting new features or improvements. - -**Key Features:** -- Description of the feature -- Explanation of why it would be helpful -- Space for proposed solution -- Checkbox for contributors who want to implement it -- Automatic `enhancement` and `needs-triage` labels - -**When to Use:** You have an idea for a new feature or improvement. - -### 🔧 Installation Issue (`ISSUE_TEMPLATE/installation_issue.yml`) - -For problems with installation or setup. - -**Key Features:** -- Deployment type selection -- Environment details -- Error message collection -- Automatic `installation` label - -**When to Use:** Having trouble getting Open Notebook running. - -## 🔄 Pull Request Template - -The PR template (`pull_request_template.md`) ensures contributors provide all necessary information: - -**Sections:** -- Description and related issue -- Type of change -- Testing details -- Design alignment with project principles -- Comprehensive checklist for code quality, testing, documentation -- Screenshots for UI changes - -**Key Checkpoints:** -- References an approved issue -- Aligns with [design principles](../DESIGN_PRINCIPLES.md) -- Includes tests and documentation -- Follows code style guidelines - -## 🚀 GitHub Actions - -Our CI/CD workflows in `.github/workflows/`: - -### `build-and-release.yml` -- Builds Docker images on releases -- Publishes to Docker Hub and GitHub Container Registry -- Supports multi-platform builds (amd64, arm64) - -### `build-dev.yml` -- Builds dev images on pushes to main -- Tags with commit SHA for testing - -### `claude-code-review.yml` -- Automated code review using Claude Code -- Runs on pull requests -- Provides AI-powered suggestions - -## 📖 How the Contribution Flow Works - -``` -┌─────────────────────────────────────────────────────────┐ -│ 1. Contributor identifies a bug or has a feature idea │ -└────────────────────┬────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────┐ -│ 2. Creates an issue using appropriate template │ -│ - Describes the problem/feature │ -│ - Checks "I am a developer..." if willing to work │ -└────────────────────┬────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────┐ -│ 3. Maintainer reviews issue (within 48 hours) │ -│ - Assesses alignment with design principles │ -│ - Labels appropriately │ -│ - Asks for clarification if needed │ -└────────────────────┬────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────┐ -│ 4. If approved: Contributor proposes solution approach │ -│ - Discusses implementation strategy │ -│ - Maintainer provides feedback │ -└────────────────────┬────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────┐ -│ 5. Issue is assigned to contributor │ -│ - Contributor forks repo │ -│ - Creates feature branch from main │ -└────────────────────┬────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────┐ -│ 6. Contributor develops solution │ -│ - Reads DESIGN_PRINCIPLES.md │ -│ - Reads docs/development/architecture.md │ -│ - Writes code, tests, documentation │ -└────────────────────┬────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────┐ -│ 7. Creates Pull Request │ -│ - Uses PR template │ -│ - References issue number │ -│ - Fills out all checklist items │ -└────────────────────┬────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────┐ -│ 8. Maintainer reviews PR │ -│ - Checks code quality │ -│ - Verifies tests pass │ -│ - Ensures alignment with architecture │ -│ - Provides feedback or approves │ -└────────────────────┬────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────┐ -│ 9. If approved: PR is merged! 🎉 │ -│ - Contributor is thanked │ -│ - Issue is closed │ -│ - Changes included in next release │ -└─────────────────────────────────────────────────────────┘ -``` - -## 🎯 Why This Process? - -### Prevents Wasted Effort -- Contributors don't spend time on code that won't be merged -- Ensures alignment with project vision before coding starts - -### Maintains Quality -- All code reviewed against design principles -- Consistent architecture across contributions -- Proper testing and documentation - -### Respects Everyone's Time -- Clear expectations upfront -- Structured feedback process -- Efficient review process - -### Protects Project Vision -- Maintainers can guide direction -- Features align with long-term goals -- Technical debt is minimized - -## 📚 Related Documentation - -For contributors: -- [CONTRIBUTING.md](../CONTRIBUTING.md) - Contribution guidelines -- [DESIGN_PRINCIPLES.md](../DESIGN_PRINCIPLES.md) - Project vision and principles -- [docs/development/architecture.md](../docs/development/architecture.md) - Technical architecture - -For maintainers: -- [MAINTAINER_GUIDE.md](../MAINTAINER_GUIDE.md) - How to review and manage contributions - -## 🤝 Questions? - -- Join our [Discord](https://discord.gg/37XJPXfz2w) -- Open a [Discussion](https://github.com/lfnovo/open-notebook/discussions) -- Read the [FAQ](../docs/troubleshooting/faq.md) - ---- - -**Thank you for contributing to Open Notebook!** Your contributions help make this the best open-source research tool available. diff --git a/commands/source_commands.py b/commands/source_commands.py index 0f862b4..5385921 100644 --- a/commands/source_commands.py +++ b/commands/source_commands.py @@ -44,7 +44,17 @@ class SourceProcessingOutput(CommandOutput): error_message: Optional[str] = None -@command("process_source", app="open_notebook") +@command( + "process_source", + app="open_notebook", + retry={ + "max_attempts": 5, + "wait_strategy": "exponential_jitter", + "wait_min": 1, + "wait_max": 30, + "retry_on": [RuntimeError], + }, +) async def process_source_command( input_data: SourceProcessingInput, ) -> SourceProcessingOutput: @@ -124,10 +134,15 @@ async def process_source_command( processing_time=processing_time, ) + except RuntimeError as e: + # Transaction conflicts should be retried by surreal-commands + logger.warning(f"Transaction conflict, will retry: {e}") + raise + except Exception as e: + # Other errors are permanent failures processing_time = time.time() - start_time logger.error(f"Source processing failed: {e}") - logger.exception(e) return SourceProcessingOutput( success=False, diff --git a/frontend/src/app/(dashboard)/notebooks/[id]/page.tsx b/frontend/src/app/(dashboard)/notebooks/[id]/page.tsx index d336e35..7f598e2 100644 --- a/frontend/src/app/(dashboard)/notebooks/[id]/page.tsx +++ b/frontend/src/app/(dashboard)/notebooks/[id]/page.tsx @@ -11,6 +11,11 @@ import { useNotebook } from '@/lib/hooks/use-notebooks' import { useSources } from '@/lib/hooks/use-sources' import { useNotes } from '@/lib/hooks/use-notes' import { LoadingSpinner } from '@/components/common/LoadingSpinner' +import { useNotebookColumnsStore } from '@/lib/stores/notebook-columns-store' +import { useIsDesktop } from '@/lib/hooks/use-media-query' +import { cn } from '@/lib/utils' +import { Tabs, TabsList, TabsTrigger } from '@/components/ui/tabs' +import { FileText, StickyNote, MessageSquare } from 'lucide-react' export type ContextMode = 'off' | 'insights' | 'full' @@ -29,6 +34,15 @@ export default function NotebookPage() { const { data: sources, isLoading: sourcesLoading, refetch: refetchSources } = useSources(notebookId) const { data: notes, isLoading: notesLoading } = useNotes(notebookId) + // Get collapse states for dynamic layout + const { sourcesCollapsed, notesCollapsed } = useNotebookColumnsStore() + + // Detect desktop to avoid double-mounting ChatColumn + const isDesktop = useIsDesktop() + + // Mobile tab state (Sources, Notes, or Chat) + const [mobileActiveTab, setMobileActiveTab] = useState<'sources' | 'notes' | 'chat'>('chat') + // Context selection state const [contextSelections, setContextSelections] = useState({ sources: {}, @@ -105,32 +119,98 @@ export default function NotebookPage() { -
-
-
-
- handleContextModeChange(sourceId, mode, 'source')} - /> +
+ {/* Mobile: Tabbed interface - only render on mobile to avoid double-mounting */} + {!isDesktop && ( + <> +
+ setMobileActiveTab(value as 'sources' | 'notes' | 'chat')}> + + + + Sources + + + + Notes + + + + Chat + + +
-
- handleContextModeChange(noteId, mode, 'note')} - /> + + {/* Mobile: Show only active tab */} +
+ {mobileActiveTab === 'sources' && ( + handleContextModeChange(sourceId, mode, 'source')} + /> + )} + {mobileActiveTab === 'notes' && ( + handleContextModeChange(noteId, mode, 'note')} + /> + )} + {mobileActiveTab === 'chat' && ( + + )}
+ + )} + + {/* Desktop: Collapsible columns layout */} +
+ {/* Sources Column */} +
+ handleContextModeChange(sourceId, mode, 'source')} + />
-
+ {/* Notes Column */} +
+ handleContextModeChange(noteId, mode, 'note')} + /> +
+ + {/* Chat Column - always expanded, takes remaining space */} +
createCollapseButton(toggleNotes, 'Notes'), + [toggleNotes] + ) + const handleDeleteClick = (noteId: string) => { setNoteToDelete(noteId) setDeleteDialogOpen(true) @@ -62,113 +71,123 @@ export function NotesColumn({ return ( <> - - -
- Notes - -
- - - - {isLoading ? ( -
- -
- ) : !notes || notes.length === 0 ? ( - - ) : ( -
- {notes.map((note) => ( -
setEditingNote(note)} + + + +
+ Notes +
+ - - - { - e.stopPropagation() - handleDeleteClick(note.id) - }} - className="text-red-600 focus:text-red-600" - > - - Delete Note - - - -
-
- - {note.title && ( -

{note.title}

- )} - - {note.content && ( -

- {note.content} -

- )} -
- ))} + + Write Note + + {collapseButton} +
- )} - - + + + + {isLoading ? ( +
+ +
+ ) : !notes || notes.length === 0 ? ( + + ) : ( +
+ {notes.map((note) => ( +
setEditingNote(note)} + > +
+
+ {note.note_type === 'ai' ? ( + + ) : ( + + )} + + {note.note_type === 'ai' ? 'AI Generated' : 'Human'} + +
+ +
+ + {formatDistanceToNow(new Date(note.updated), { addSuffix: true })} + + + {/* Context toggle - only show if handler provided */} + {onContextModeChange && contextSelections?.[note.id] && ( +
event.stopPropagation()}> + onContextModeChange(note.id, mode)} + /> +
+ )} + + {/* Ellipsis menu for delete action */} + + + + + + { + e.stopPropagation() + handleDeleteClick(note.id) + }} + className="text-red-600 focus:text-red-600" + > + + Delete Note + + + +
+
+ + {note.title && ( +

{note.title}

+ )} + + {note.content && ( +

+ {note.content} +

+ )} +
+ ))} +
+ )} + + + createCollapseButton(toggleSources, 'Sources'), + [toggleSources] + ) const handleDeleteClick = (sourceId: string) => { setSourceToDelete(sourceId) @@ -102,67 +111,80 @@ export function SourcesColumn({ const handleSourceClick = (sourceId: string) => { openModal('source', sourceId) } - return ( - - -
- Sources - - - - - - { setDropdownOpen(false); setAddDialogOpen(true); }}> - - Add New Source - - { setDropdownOpen(false); setAddExistingDialogOpen(true); }}> - - Add Existing Source - - - -
- - - {isLoading ? ( -
- -
- ) : !sources || sources.length === 0 ? ( - - ) : ( -
- {sources.map((source) => ( - onContextModeChange(source.id, mode) - : undefined - } + return ( + <> + + + +
+ Sources +
+ + + + + + { setDropdownOpen(false); setAddDialogOpen(true); }}> + + Add New Source + + { setDropdownOpen(false); setAddExistingDialogOpen(true); }}> + + Add Existing Source + + + + {collapseButton} +
+
+ + + + {isLoading ? ( +
+ +
+ ) : !sources || sources.length === 0 ? ( + - ))} -
- )} - - + ) : ( +
+ {sources.map((source) => ( + onContextModeChange(source.id, mode) + : undefined + } + /> + ))} +
+ )} + + + + - + ) } diff --git a/frontend/src/components/notebooks/CollapsibleColumn.tsx b/frontend/src/components/notebooks/CollapsibleColumn.tsx new file mode 100644 index 0000000..3dc3b21 --- /dev/null +++ b/frontend/src/components/notebooks/CollapsibleColumn.tsx @@ -0,0 +1,93 @@ +'use client' + +import { ReactNode } from 'react' +import { Button } from '@/components/ui/button' +import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from '@/components/ui/tooltip' +import { ChevronLeft, LucideIcon } from 'lucide-react' +import { cn } from '@/lib/utils' + +interface CollapsibleColumnProps { + isCollapsed: boolean + onToggle: () => void + collapsedIcon: LucideIcon + collapsedLabel: string + children: ReactNode +} + +export function CollapsibleColumn({ + isCollapsed, + onToggle, + collapsedIcon: CollapsedIcon, + collapsedLabel, + children, +}: CollapsibleColumnProps) { + if (isCollapsed) { + return ( + + + + + + +

Expand {collapsedLabel}

+ + + + ) + } + + return ( +
+ {children} +
+ ) +} + +// Factory function to create a collapse button for card headers +export function createCollapseButton(onToggle: () => void, label: string) { + return ( +
+ + + + + + +

Collapse {label}

+ + + +
+ ) +} diff --git a/frontend/src/components/sources/AddSourceDialog.tsx b/frontend/src/components/sources/AddSourceDialog.tsx index e4fbee6..4db8d51 100644 --- a/frontend/src/components/sources/AddSourceDialog.tsx +++ b/frontend/src/components/sources/AddSourceDialog.tsx @@ -1,10 +1,11 @@ 'use client' -import { useState, useRef, useEffect } from 'react' +import { useState, useRef, useEffect, useMemo } from 'react' import { useForm } from 'react-hook-form' import { zodResolver } from '@hookform/resolvers/zod' import { z } from 'zod' -import { LoaderIcon } from 'lucide-react' +import { LoaderIcon, CheckCircleIcon, XCircleIcon } from 'lucide-react' +import { toast } from 'sonner' import { Dialog, DialogContent, @@ -14,7 +15,7 @@ import { } from '@/components/ui/dialog' import { Button } from '@/components/ui/button' import { WizardContainer, WizardStep } from '@/components/ui/wizard-container' -import { SourceTypeStep } from './steps/SourceTypeStep' +import { SourceTypeStep, parseAndValidateUrls } from './steps/SourceTypeStep' import { NotebooksStep } from './steps/NotebooksStep' import { ProcessingStep } from './steps/ProcessingStep' import { useNotebooks } from '@/lib/hooks/use-notebooks' @@ -23,6 +24,8 @@ import { useCreateSource } from '@/lib/hooks/use-sources' import { useSettings } from '@/lib/hooks/use-settings' import { CreateSourceRequest } from '@/lib/types/api' +const MAX_BATCH_SIZE = 50 + const createSourceSchema = z.object({ type: z.enum(['link', 'upload', 'text']), title: z.string().optional(), @@ -80,6 +83,13 @@ interface ProcessingState { progress?: number } +interface BatchProgress { + total: number + completed: number + failed: number + currentItem?: string +} + export function AddSourceDialog({ open, onOpenChange, @@ -94,6 +104,10 @@ export function AddSourceDialog({ ) const [selectedTransformations, setSelectedTransformations] = useState([]) + // Batch-specific state + const [urlValidationErrors, setUrlValidationErrors] = useState<{ url: string; line: number }[]>([]) + const [batchProgress, setBatchProgress] = useState(null) + // Cleanup timeouts to prevent memory leaks const timeoutRef = useRef(null) @@ -158,12 +172,51 @@ export function AddSourceDialog({ const watchedFile = watch('file') const watchedTitle = watch('title') + // Batch mode detection + const { isBatchMode, itemCount, parsedUrls, parsedFiles } = useMemo(() => { + let urlCount = 0 + let fileCount = 0 + let parsedUrls: string[] = [] + let parsedFiles: File[] = [] + + if (selectedType === 'link' && watchedUrl) { + const { valid } = parseAndValidateUrls(watchedUrl) + parsedUrls = valid + urlCount = valid.length + } + + if (selectedType === 'upload' && watchedFile) { + const fileList = watchedFile as FileList + if (fileList?.length) { + parsedFiles = Array.from(fileList) + fileCount = parsedFiles.length + } + } + + const isBatchMode = urlCount > 1 || fileCount > 1 + const itemCount = selectedType === 'link' ? urlCount : fileCount + + return { isBatchMode, itemCount, parsedUrls, parsedFiles } + }, [selectedType, watchedUrl, watchedFile]) + + // Check for batch size limit + const isOverLimit = itemCount > MAX_BATCH_SIZE + // Step validation - now reactive with watched values const isStepValid = (step: number): boolean => { switch (step) { case 1: if (!selectedType) return false + // Check batch size limit + if (isOverLimit) return false + // Check for URL validation errors + if (urlValidationErrors.length > 0) return false + if (selectedType === 'link') { + // In batch mode, check that we have at least one valid URL + if (isBatchMode) { + return parsedUrls.length > 0 + } return !!watchedUrl && watchedUrl.trim() !== '' } if (selectedType === 'text') { @@ -172,7 +225,7 @@ export function AddSourceDialog({ } if (selectedType === 'upload') { if (watchedFile instanceof FileList) { - return watchedFile.length > 0 + return watchedFile.length > 0 && watchedFile.length <= MAX_BATCH_SIZE } return !!watchedFile } @@ -189,11 +242,27 @@ export function AddSourceDialog({ const handleNextStep = (e?: React.MouseEvent) => { e?.preventDefault() e?.stopPropagation() + + // Validate URLs when leaving step 1 in link mode + if (currentStep === 1 && selectedType === 'link' && watchedUrl) { + const { invalid } = parseAndValidateUrls(watchedUrl) + if (invalid.length > 0) { + setUrlValidationErrors(invalid) + return + } + setUrlValidationErrors([]) + } + if (currentStep < 3 && isStepValid(currentStep)) { setCurrentStep(currentStep + 1) } } + // Clear URL validation errors when user edits + const handleClearUrlErrors = () => { + setUrlValidationErrors([]) + } + const handlePrevStep = (e?: React.MouseEvent) => { e?.preventDefault() e?.stopPropagation() @@ -223,43 +292,127 @@ export function AddSourceDialog({ setSelectedTransformations(updated) } + // Single source submission + const submitSingleSource = async (data: CreateSourceFormData): Promise => { + const createRequest: CreateSourceRequest = { + type: data.type, + notebooks: selectedNotebooks, + url: data.type === 'link' ? data.url : undefined, + content: data.type === 'text' ? data.content : undefined, + title: data.title, + transformations: selectedTransformations, + embed: data.embed, + delete_source: false, + async_processing: true, + } + + if (data.type === 'upload' && data.file) { + const file = data.file instanceof FileList ? data.file[0] : data.file + const requestWithFile = createRequest as CreateSourceRequest & { file?: File } + requestWithFile.file = file + } + + await createSource.mutateAsync(createRequest) + } + + // Batch submission + const submitBatch = async (data: CreateSourceFormData): Promise<{ success: number; failed: number }> => { + const results = { success: 0, failed: 0 } + const items: { type: 'url' | 'file'; value: string | File }[] = [] + + // Collect items to process + if (data.type === 'link' && parsedUrls.length > 0) { + parsedUrls.forEach(url => items.push({ type: 'url', value: url })) + } else if (data.type === 'upload' && parsedFiles.length > 0) { + parsedFiles.forEach(file => items.push({ type: 'file', value: file })) + } + + setBatchProgress({ + total: items.length, + completed: 0, + failed: 0, + }) + + // Process each item sequentially + for (let i = 0; i < items.length; i++) { + const item = items[i] + const itemLabel = item.type === 'url' + ? (item.value as string).substring(0, 50) + '...' + : (item.value as File).name + + setBatchProgress(prev => prev ? { + ...prev, + currentItem: itemLabel, + } : null) + + try { + const createRequest: CreateSourceRequest = { + type: item.type === 'url' ? 'link' : 'upload', + notebooks: selectedNotebooks, + url: item.type === 'url' ? item.value as string : undefined, + transformations: selectedTransformations, + embed: data.embed, + delete_source: false, + async_processing: true, + } + + if (item.type === 'file') { + const requestWithFile = createRequest as CreateSourceRequest & { file?: File } + requestWithFile.file = item.value as File + } + + await createSource.mutateAsync(createRequest) + results.success++ + } catch (error) { + console.error(`Error creating source for ${itemLabel}:`, error) + results.failed++ + } + + setBatchProgress(prev => prev ? { + ...prev, + completed: results.success, + failed: results.failed, + } : null) + } + + return results + } + // Form submission const onSubmit = async (data: CreateSourceFormData) => { try { setProcessing(true) - setProcessingStatus({ message: 'Submitting source for processing...' }) - const createRequest: CreateSourceRequest = { - type: data.type, - notebooks: selectedNotebooks, - url: data.type === 'link' ? data.url : undefined, - content: data.type === 'text' ? data.content : undefined, - title: data.title, - transformations: selectedTransformations, - embed: data.embed, - delete_source: false, - async_processing: true, // Always use async processing for frontend submissions + if (isBatchMode) { + // Batch submission + setProcessingStatus({ message: `Processing ${itemCount} sources...` }) + const results = await submitBatch(data) + + // Show summary toast + if (results.failed === 0) { + toast.success(`${results.success} source${results.success !== 1 ? 's' : ''} created successfully`) + } else if (results.success === 0) { + toast.error(`Failed to create all ${results.failed} sources`) + } else { + toast.warning(`${results.success} succeeded, ${results.failed} failed`) + } + + handleClose() + } else { + // Single source submission + setProcessingStatus({ message: 'Submitting source for processing...' }) + await submitSingleSource(data) + handleClose() } - - - if (data.type === 'upload' && data.file) { - const file = data.file instanceof FileList ? data.file[0] : data.file - const requestWithFile = createRequest as CreateSourceRequest & { file?: File } - requestWithFile.file = file - } - - await createSource.mutateAsync(createRequest) - - // Close immediately - the toast will show the success message - handleClose() } catch (error) { console.error('Error creating source:', error) - setProcessingStatus({ + setProcessingStatus({ message: 'Error creating source. Please try again.', }) timeoutRef.current = setTimeout(() => { setProcessing(false) setProcessingStatus(null) + setBatchProgress(null) }, 3000) } } @@ -277,6 +430,8 @@ export function AddSourceDialog({ setProcessing(false) setProcessingStatus(null) setSelectedNotebooks(defaultNotebookId ? [defaultNotebookId] : []) + setUrlValidationErrors([]) + setBatchProgress(null) // Reset to default transformations if (transformations.length > 0) { @@ -293,16 +448,25 @@ export function AddSourceDialog({ // Processing view if (processing) { + const progressPercent = batchProgress + ? Math.round(((batchProgress.completed + batchProgress.failed) / batchProgress.total) * 100) + : undefined + return ( - Processing Source + + {batchProgress ? 'Processing Batch' : 'Processing Source'} + - Your source is being processed. This may take a few moments. + {batchProgress + ? `Processing ${batchProgress.total} sources. This may take a few moments.` + : 'Your source is being processed. This may take a few moments.' + } - +
@@ -310,11 +474,48 @@ export function AddSourceDialog({ {processingStatus?.message || 'Processing...'}
- - {processingStatus?.progress && ( + + {/* Batch progress */} + {batchProgress && ( + <> +
+
+
+ +
+
+ + + {batchProgress.completed} completed + + {batchProgress.failed > 0 && ( + + + {batchProgress.failed} failed + + )} +
+ + {batchProgress.completed + batchProgress.failed} / {batchProgress.total} + +
+ + {batchProgress.currentItem && ( +

+ Current: {batchProgress.currentItem} +

+ )} + + )} + + {/* Single source progress */} + {!batchProgress && processingStatus?.progress && (
-
@@ -351,6 +552,8 @@ export function AddSourceDialog({ register={register} // @ts-expect-error - Type inference issue with zod schema errors={errors} + urlValidationErrors={urlValidationErrors} + onClearUrlErrors={handleClearUrlErrors} /> )} diff --git a/frontend/src/components/sources/steps/SourceTypeStep.tsx b/frontend/src/components/sources/steps/SourceTypeStep.tsx index b049e9f..c3c609d 100644 --- a/frontend/src/components/sources/steps/SourceTypeStep.tsx +++ b/frontend/src/components/sources/steps/SourceTypeStep.tsx @@ -1,5 +1,6 @@ "use client" +import { useMemo } from "react" import { Control, FieldErrors, UseFormRegister, useWatch } from "react-hook-form" import { FileIcon, LinkIcon, FileTextIcon } from "lucide-react" import { FormSection } from "@/components/ui/form-section" @@ -7,6 +8,7 @@ import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs" import { Input } from "@/components/ui/input" import { Textarea } from "@/components/ui/textarea" import { Label } from "@/components/ui/label" +import { Badge } from "@/components/ui/badge" import { Controller } from "react-hook-form" interface CreateSourceFormData { @@ -21,6 +23,45 @@ interface CreateSourceFormData { async_processing: boolean } +// Helper functions for batch URL parsing +function parseUrls(text: string): string[] { + return text + .split('\n') + .map(line => line.trim()) + .filter(line => line.length > 0) +} + +function validateUrl(url: string): boolean { + try { + new URL(url) + return true + } catch { + return false + } +} + +export function parseAndValidateUrls(text: string): { + valid: string[] + invalid: { url: string; line: number }[] +} { + const lines = text.split('\n') + const valid: string[] = [] + const invalid: { url: string; line: number }[] = [] + + lines.forEach((line, index) => { + const trimmed = line.trim() + if (trimmed.length === 0) return // skip empty lines + + if (validateUrl(trimmed)) { + valid.push(trimmed) + } else { + invalid.push({ url: trimmed, line: index + 1 }) + } + }) + + return { valid, invalid } +} + const SOURCE_TYPES = [ { value: 'link' as const, @@ -46,11 +87,41 @@ interface SourceTypeStepProps { control: Control register: UseFormRegister errors: FieldErrors + urlValidationErrors?: { url: string; line: number }[] + onClearUrlErrors?: () => void } -export function SourceTypeStep({ control, register, errors }: SourceTypeStepProps) { - // Watch the selected type to make title conditional +const MAX_BATCH_SIZE = 50 + +export function SourceTypeStep({ control, register, errors, urlValidationErrors, onClearUrlErrors }: SourceTypeStepProps) { + // Watch the selected type and inputs to detect batch mode const selectedType = useWatch({ control, name: 'type' }) + const urlInput = useWatch({ control, name: 'url' }) + const fileInput = useWatch({ control, name: 'file' }) + + // Batch mode detection + const { isBatchMode, itemCount, urlCount, fileCount } = useMemo(() => { + let urlCount = 0 + let fileCount = 0 + + if (selectedType === 'link' && urlInput) { + const urls = parseUrls(urlInput) + urlCount = urls.length + } + + if (selectedType === 'upload' && fileInput) { + const fileList = fileInput as FileList + fileCount = fileList?.length || 0 + } + + const isBatchMode = urlCount > 1 || fileCount > 1 + const itemCount = selectedType === 'link' ? urlCount : fileCount + + return { isBatchMode, itemCount, urlCount, fileCount } + }, [selectedType, urlInput, fileInput]) + + // Check for batch size limit + const isOverLimit = itemCount > MAX_BATCH_SIZE return (
- - + + {urlCount > 0 && ( + + {urlCount} URL{urlCount !== 1 ? 's' : ''} + {isOverLimit && ` (max ${MAX_BATCH_SIZE})`} + + )} +
+