From 3d1b3292217b35d4de37a948c86b9f909e64e6e0 Mon Sep 17 00:00:00 2001 From: 777genius Date: Fri, 22 May 2026 00:15:57 +0300 Subject: [PATCH] feat(discovery): bound recent project scans --- ...xSessionFileRecentProjectsSourceAdapter.ts | 250 ++++++++++- src/main/services/discovery/ProjectScanner.ts | 421 +++++++++++++++--- .../services/discovery/SubprojectRegistry.ts | 34 ++ src/main/utils/jsonl.ts | 6 +- src/main/utils/metadataExtraction.ts | 56 ++- ...ionFileRecentProjectsSourceAdapter.test.ts | 308 ++++++++++++- .../ProjectScanner.scanDedup.safe-e2e.test.ts | 413 ++++++++++++++++- 7 files changed, 1388 insertions(+), 100 deletions(-) diff --git a/src/features/recent-projects/main/adapters/output/sources/CodexSessionFileRecentProjectsSourceAdapter.ts b/src/features/recent-projects/main/adapters/output/sources/CodexSessionFileRecentProjectsSourceAdapter.ts index 1ee20721..2258fc39 100644 --- a/src/features/recent-projects/main/adapters/output/sources/CodexSessionFileRecentProjectsSourceAdapter.ts +++ b/src/features/recent-projects/main/adapters/output/sources/CodexSessionFileRecentProjectsSourceAdapter.ts @@ -4,6 +4,7 @@ import path from 'node:path'; import { resolveProjectFilesystemState } from '@features/recent-projects/main/infrastructure/filesystem/resolveProjectFilesystemState'; import { normalizeIdentityPath } from '@features/recent-projects/main/infrastructure/identity/normalizeIdentityPath'; +import { getAppDataPath } from '@main/utils/pathDecoder'; import { isEphemeralProjectPath } from '@shared/utils/ephemeralProjectPath'; import type { LoggerPort } from '@features/recent-projects/core/application/ports/LoggerPort'; @@ -18,12 +19,21 @@ import type { ServiceContext } from '@main/services'; const CODEX_SESSION_FILE_PARSE_LIMIT = 500; const CODEX_PROJECT_CANDIDATE_LIMIT = 40; const CODEX_SESSION_FILE_SOURCE_TIMEOUT_MS = 8_000; +const CODEX_SESSION_FILE_SOFT_BUDGET_MS = 6_500; +const CODEX_SESSION_FILE_MAX_UNCACHED_READS_PER_RUN = 160; const CODEX_SESSION_FILE_READ_BATCH_SIZE = 24; +const CODEX_SESSION_FILE_READ_TIMEOUT_MS = 700; const CODEX_SESSION_METADATA_READ_LIMIT_BYTES = 128 * 1024; +const CODEX_SESSION_FILE_CACHE_SCHEMA_VERSION = 1; +const CODEX_SESSION_FILE_CACHE_RELATIVE_PATH = path.join( + 'recent-projects', + 'codex-session-files-index.json' +); interface CodexSessionFileEntry { filePath: string; mtimeMs: number; + size: number; } interface CodexSessionEvent { @@ -53,6 +63,50 @@ interface CodexSessionMetadata { branchName?: string; } +interface CodexSessionFileCacheEntry { + filePath: string; + mtimeMs: number; + size: number; + snapshot: CodexSessionProjectSnapshot | null; +} + +interface CodexSessionFileCacheFile { + schemaVersion: number; + entries: Record; +} + +interface CodexSessionSnapshotLoadResult { + snapshots: CodexSessionProjectSnapshot[]; + degraded: boolean; + stats: { + files: number; + cached: number; + uncachedReads: number; + timedOutReads: number; + skippedUncached: number; + durationMs: number; + }; +} + +function emptyCache(): CodexSessionFileCacheFile { + return { + schemaVersion: CODEX_SESSION_FILE_CACHE_SCHEMA_VERSION, + entries: {}, + }; +} + +function isUsableCacheEntry( + entry: CodexSessionFileCacheEntry | undefined, + file: CodexSessionFileEntry +): entry is CodexSessionFileCacheEntry { + return ( + !!entry && + entry.filePath === file.filePath && + entry.mtimeMs === file.mtimeMs && + entry.size === file.size + ); +} + function isInteractiveSource(source: unknown): boolean { return source === 'vscode' || source === 'cli'; } @@ -118,6 +172,45 @@ async function readFirstLine(filePath: string): Promise { } } +async function readFirstLineWithTimeout( + filePath: string, + timeoutMs: number +): Promise<{ firstLine: string | null; timedOut: boolean }> { + if (timeoutMs <= 0) { + return { + firstLine: null, + timedOut: true, + }; + } + + let timer: ReturnType | null = null; + const readPromise = readFirstLine(filePath) + .then((firstLine) => ({ + firstLine, + timedOut: false, + })) + .catch(() => ({ + firstLine: null, + timedOut: false, + })); + const timeoutPromise = new Promise<{ firstLine: null; timedOut: true }>((resolve) => { + timer = setTimeout( + () => + resolve({ + firstLine: null, + timedOut: true, + }), + timeoutMs + ); + }); + + const result = await Promise.race([readPromise, timeoutPromise]); + if (timer) { + clearTimeout(timer); + } + return result; +} + async function listJsonlFiles(root: string, maxDepth: number): Promise { async function walk(directory: string, depth: number): Promise { let entries; @@ -144,6 +237,7 @@ async function listJsonlFiles(root: string, maxDepth: number): Promise { @@ -224,9 +324,11 @@ export class CodexSessionFileRecentProjectsSourceAdapter implements RecentProjec } try { - const snapshots = await this.#listRecentSessionSnapshots(); + const snapshotResult = await this.#listRecentSessionSnapshots(); const candidates = await Promise.all( - snapshots.map((snapshot) => this.#toCandidate(snapshot, activeContext.fsProvider)) + snapshotResult.snapshots.map((snapshot) => + this.#toCandidate(snapshot, activeContext.fsProvider) + ) ); const validCandidates = candidates.filter( @@ -236,11 +338,13 @@ export class CodexSessionFileRecentProjectsSourceAdapter implements RecentProjec this.deps.logger.info('codex session-file recent-projects source loaded', { count: validCandidates.length, codexHome: this.#codexHome, + degraded: snapshotResult.degraded, + ...snapshotResult.stats, }); return { candidates: validCandidates, - degraded: false, + degraded: snapshotResult.degraded, }; } catch (error) { this.deps.logger.warn('codex session-file recent-projects source failed', { @@ -254,15 +358,23 @@ export class CodexSessionFileRecentProjectsSourceAdapter implements RecentProjec } } - async #listRecentSessionSnapshots(): Promise { + async #listRecentSessionSnapshots(): Promise { + const startedAt = Date.now(); + const deadline = startedAt + CODEX_SESSION_FILE_SOFT_BUDGET_MS; const files = [ ...(await listJsonlFiles(path.join(this.#codexHome, 'sessions'), 4)), ...(await listJsonlFiles(path.join(this.#codexHome, 'archived_sessions'), 1)), ].sort((left, right) => right.mtimeMs - left.mtimeMs); const snapshotsByCwd = new Map(); - const candidateFiles = files.slice(0, CODEX_SESSION_FILE_PARSE_LIMIT); + const cache = await this.#readCacheSafe(); + const nextCacheEntries = new Map(); + let degraded = false; + let cached = 0; + let uncachedReads = 0; + let timedOutReads = 0; + let skippedUncached = 0; for ( let offset = 0; @@ -270,20 +382,49 @@ export class CodexSessionFileRecentProjectsSourceAdapter implements RecentProjec offset += CODEX_SESSION_FILE_READ_BATCH_SIZE ) { const batch = candidateFiles.slice(offset, offset + CODEX_SESSION_FILE_READ_BATCH_SIZE); - const firstLines = await Promise.all( - batch.map(async (file) => ({ - file, - firstLine: await readFirstLine(file.filePath), - })) + const metadata = await Promise.all( + batch.map(async (file) => { + const cachedEntry = cache.entries[file.filePath]; + if (isUsableCacheEntry(cachedEntry, file)) { + cached += 1; + nextCacheEntries.set(file.filePath, cachedEntry); + return { snapshot: cachedEntry.snapshot, processed: true }; + } + + if ( + Date.now() >= deadline || + uncachedReads >= CODEX_SESSION_FILE_MAX_UNCACHED_READS_PER_RUN + ) { + degraded = true; + skippedUncached += 1; + return { snapshot: null, processed: false }; + } + + uncachedReads += 1; + const readResult = await readFirstLineWithTimeout( + file.filePath, + Math.min(CODEX_SESSION_FILE_READ_TIMEOUT_MS, deadline - Date.now()) + ); + if (readResult.timedOut) { + degraded = true; + timedOutReads += 1; + return { snapshot: null, processed: false }; + } + + const firstLine = readResult.firstLine; + const snapshot = firstLine ? parseSessionSnapshot(firstLine, file.mtimeMs) : null; + nextCacheEntries.set(file.filePath, { + filePath: file.filePath, + mtimeMs: file.mtimeMs, + size: file.size, + snapshot, + }); + return { snapshot, processed: true }; + }) ); - for (const { file, firstLine } of firstLines) { - if (!firstLine) { - continue; - } - - const snapshot = parseSessionSnapshot(firstLine, file.mtimeMs); - if (!snapshot) { + for (const { snapshot, processed } of metadata) { + if (!processed || !snapshot) { continue; } @@ -298,9 +439,82 @@ export class CodexSessionFileRecentProjectsSourceAdapter implements RecentProjec } } - return Array.from(snapshotsByCwd.values()) + for (const file of candidateFiles) { + const cachedEntry = cache.entries[file.filePath]; + if (isUsableCacheEntry(cachedEntry, file) && !nextCacheEntries.has(file.filePath)) { + nextCacheEntries.set(file.filePath, cachedEntry); + } + } + await this.#writeCacheSafe(nextCacheEntries); + + const snapshots = Array.from(snapshotsByCwd.values()) .sort((left, right) => right.lastActivityAt - left.lastActivityAt) .slice(0, CODEX_PROJECT_CANDIDATE_LIMIT); + const durationMs = Date.now() - startedAt; + if (degraded) { + this.deps.logger.warn('codex session-file recent-projects source partial', { + files: candidateFiles.length, + cached, + uncachedReads, + timedOutReads, + skippedUncached, + candidates: snapshots.length, + durationMs, + }); + } + + return { + snapshots, + degraded, + stats: { + files: candidateFiles.length, + cached, + uncachedReads, + timedOutReads, + skippedUncached, + durationMs, + }, + }; + } + + async #readCacheSafe(): Promise { + try { + const raw = await fs.readFile(this.#cachePath, 'utf8'); + const parsed = JSON.parse(raw) as Partial; + if ( + parsed.schemaVersion !== CODEX_SESSION_FILE_CACHE_SCHEMA_VERSION || + !parsed.entries || + typeof parsed.entries !== 'object' || + Array.isArray(parsed.entries) + ) { + return emptyCache(); + } + return { + schemaVersion: CODEX_SESSION_FILE_CACHE_SCHEMA_VERSION, + entries: parsed.entries, + }; + } catch { + return emptyCache(); + } + } + + async #writeCacheSafe(entries: ReadonlyMap): Promise { + let tempPath: string | null = null; + try { + await fs.mkdir(path.dirname(this.#cachePath), { recursive: true }); + const cacheFile: CodexSessionFileCacheFile = { + schemaVersion: CODEX_SESSION_FILE_CACHE_SCHEMA_VERSION, + entries: Object.fromEntries(entries), + }; + tempPath = `${this.#cachePath}.${process.pid}.${Date.now()}.tmp`; + await fs.writeFile(tempPath, JSON.stringify(cacheFile), 'utf8'); + await fs.rename(tempPath, this.#cachePath); + } catch { + if (tempPath) { + await fs.rm(tempPath, { force: true }).catch(() => undefined); + } + // Cache is an optimization only; never fail recent projects because it is unavailable. + } } async #toCandidate( diff --git a/src/main/services/discovery/ProjectScanner.ts b/src/main/services/discovery/ProjectScanner.ts index 67440140..060f1719 100644 --- a/src/main/services/discovery/ProjectScanner.ts +++ b/src/main/services/discovery/ProjectScanner.ts @@ -42,7 +42,7 @@ import { } from '@main/types'; import { analyzeSessionFileMetadata, - extractCwd, + extractCwdFromKnownJsonlFile, type SessionFileMetadata, } from '@main/utils/jsonl'; import { @@ -106,6 +106,17 @@ export interface ProjectScannerOptions { sessionIndexDir?: string; /** Test hook: set to 0 to persist index files without debounce. */ sessionIndexPersistDelayMs?: number; + /** + * Bounds local stat/read work during project discovery. + * This keeps startup scans from saturating libuv while other services initialize. + */ + scanFileIoConcurrency?: number; + /** + * Upper bound for a full project-directory scan. If exhausted, the previous + * complete scan is reused when available; otherwise completed projects are + * returned without caching the partial result. + */ + scanBudgetMs?: number; } interface ScanInFlight { @@ -113,6 +124,28 @@ interface ScanInFlight { promise: Promise; } +interface RepositoryGroupScanInFlight { + generation: number; + promise: Promise; +} + +interface ProjectScanWithTimeoutResult { + projects: Project[]; + timedOut: boolean; +} + +interface ScanProjectOptions { + shouldCommitSubprojects?: () => boolean; + signal?: AbortSignal; +} + +class ScanProjectAbortedError extends Error { + constructor() { + super('Project scan aborted'); + this.name = 'ScanProjectAbortedError'; + } +} + function splitPathSegments(value: string): string[] { return value.split(/[/\\]+/).filter(Boolean); } @@ -190,8 +223,11 @@ export class ProjectScanner { // Both getProjects() and getRepositoryGroups() call scan() — the cache deduplicates. private scanCache: { projects: Project[]; timestamp: number } | null = null; private scanInFlight: ScanInFlight | null = null; + private repositoryGroupInFlight: RepositoryGroupScanInFlight | null = null; private scanGeneration = 0; private static readonly SCAN_CACHE_TTL_MS = 2000; + private static readonly SCAN_BUDGET_MS = 24_000; + private static readonly MIN_SCAN_BATCH_BUDGET_MS = 1000; /** Cached project list for search — avoids re-scanning disk on every query */ private searchProjectCache: { projects: Project[]; timestamp: number } | null = null; @@ -199,6 +235,9 @@ export class ProjectScanner { // Platform-aware batch sizes to avoid UV thread pool saturation on Windows private static readonly LOCAL_SESSION_BATCH = process.platform === 'win32' ? 16 : 64; private static readonly LOCAL_PROJECT_BATCH = process.platform === 'win32' ? 4 : 12; + private static readonly LOCAL_SCAN_FILE_IO_CONCURRENCY = process.platform === 'win32' ? 8 : 16; + private static readonly EXHAUSTIVE_CWD_SPLIT_FILE_LIMIT = 12; + private static readonly CWD_HINT_SAMPLE_FILE_LIMIT = 1; // Delegated services private readonly fsProvider: FileSystemProvider; @@ -207,6 +246,10 @@ export class ProjectScanner { private readonly sessionSearcher: SessionSearcher; private readonly projectPathResolver: ProjectPathResolver; private readonly sessionMetadataIndex: SessionMetadataIndex | null; + private readonly scanFileIoConcurrency: number; + private readonly scanBudgetMs: number; + private scanFileIoActive = 0; + private readonly scanFileIoQueue: Array<() => void> = []; constructor( projectsDir?: string, @@ -230,6 +273,17 @@ export class ProjectScanner { persistDelayMs: options?.sessionIndexPersistDelayMs, }) : null; + const configuredScanFileIoConcurrency = options?.scanFileIoConcurrency; + this.scanFileIoConcurrency = + typeof configuredScanFileIoConcurrency === 'number' && + Number.isFinite(configuredScanFileIoConcurrency) + ? Math.max(1, Math.floor(configuredScanFileIoConcurrency)) + : ProjectScanner.LOCAL_SCAN_FILE_IO_CONCURRENCY; + const configuredScanBudgetMs = options?.scanBudgetMs; + this.scanBudgetMs = + typeof configuredScanBudgetMs === 'number' && Number.isFinite(configuredScanBudgetMs) + ? Math.max(1, Math.floor(configuredScanBudgetMs)) + : ProjectScanner.SCAN_BUDGET_MS; } // =========================================================================== @@ -277,6 +331,7 @@ export class ProjectScanner { private async performScan(generation: number): Promise { const startedAt = Date.now(); let stage = 'start'; + let previousSubprojectRegistry: ReturnType | null = null; const slowWarnAfterMs = 10_000; const slowWarnTimer = setTimeout(() => { logger.warn( @@ -291,6 +346,7 @@ export class ProjectScanner { } // Clear the subproject registry on full re-scan + previousSubprojectRegistry = subprojectRegistry.snapshot(); subprojectRegistry.clear(); stage = 'readdirProjectsDir'; @@ -308,10 +364,9 @@ export class ProjectScanner { // Process each project directory (may return multiple projects per dir) stage = 'scanProjects'; - const projectArrays = await this.collectFulfilledInBatches( + const { projectArrays, completedAll, slowest } = await this.scanProjectDirsWithBudget( projectDirs, - this.fsProvider.type === 'ssh' ? 8 : ProjectScanner.LOCAL_PROJECT_BATCH, - async (dir) => this.scanProjectWithTimeout(dir.name) + startedAt ); // Flatten and sort by most recent @@ -327,14 +382,28 @@ export class ProjectScanner { const ms = Date.now() - startedAt; if (ms >= 5000) { logger.warn( - `[scan] completed slow ms=${ms} projectDirs=${projectDirs.length} projects=${validProjects.length}` + `[scan] completed slow ms=${ms} projectDirs=${projectDirs.length} projects=${validProjects.length} slowest=${JSON.stringify( + slowest.slice(0, 5) + )}` ); } - if (this.scanGeneration === generation) { + if (!completedAll && this.scanCache) { + if (previousSubprojectRegistry) { + subprojectRegistry.restore(previousSubprojectRegistry); + } + logger.warn( + `[scan] returning cached complete result after budget exhaustion projects=${this.scanCache.projects.length}` + ); + return this.scanCache.projects; + } + if (completedAll && this.scanGeneration === generation) { this.scanCache = { projects: validProjects, timestamp: Date.now() }; } return validProjects; } catch (error) { + if (previousSubprojectRegistry && this.scanCache) { + subprojectRegistry.restore(previousSubprojectRegistry); + } logger.error('Error scanning projects directory:', error); return []; } finally { @@ -374,6 +443,31 @@ export class ProjectScanner { * @returns Promise resolving to RepositoryGroups sorted by most recent activity */ async scanWithWorktreeGrouping(): Promise { + const generation = this.scanGeneration; + const inFlight = this.repositoryGroupInFlight; + if (inFlight) { + if (inFlight.generation === generation) { + return inFlight.promise; + } + await inFlight.promise.catch(() => undefined); + if (this.repositoryGroupInFlight?.promise === inFlight.promise) { + this.repositoryGroupInFlight = null; + } + return this.scanWithWorktreeGrouping(); + } + + const promise = this.performScanWithWorktreeGrouping(); + this.repositoryGroupInFlight = { generation, promise }; + try { + return await promise; + } finally { + if (this.repositoryGroupInFlight?.promise === promise) { + this.repositoryGroupInFlight = null; + } + } + } + + private async performScanWithWorktreeGrouping(): Promise { try { // 1. Scan all projects using existing logic const projects = await this.scan(); @@ -474,34 +568,142 @@ export class ProjectScanner { /** * Scans a single project directory with a timeout guard. - * Returns empty array if the scan exceeds the timeout. + * Timeout is reported separately from an actually empty project directory so + * partial scans cannot be mistaken for complete results. */ - private async scanProjectWithTimeout(encodedName: string): Promise { + private async scanProjectWithTimeout( + encodedName: string, + timeoutMs = ProjectScanner.SCAN_PROJECT_TIMEOUT_MS, + options: { logTimeout?: boolean } = {} + ): Promise { + const abortController = new AbortController(); let timer: ReturnType | null = null; - const timeout = new Promise((resolve) => { + let timedOut = false; + const effectiveTimeoutMs = Math.max(1, Math.floor(timeoutMs)); + const timeout = new Promise((resolve) => { timer = setTimeout(() => { - logger.warn( - `[scanProject] timeout after ${ProjectScanner.SCAN_PROJECT_TIMEOUT_MS}ms project=${encodedName}` - ); - resolve([]); - }, ProjectScanner.SCAN_PROJECT_TIMEOUT_MS); + timedOut = true; + abortController.abort(); + if (options.logTimeout !== false) { + logger.warn(`[scanProject] timeout after ${effectiveTimeoutMs}ms project=${encodedName}`); + } + resolve({ projects: [], timedOut: true }); + }, effectiveTimeoutMs); }); try { - return await Promise.race([this.scanProject(encodedName), timeout]); + return await Promise.race([ + this.scanProject(encodedName, { + shouldCommitSubprojects: () => !timedOut, + signal: abortController.signal, + }).then((projects) => ({ projects, timedOut })), + timeout, + ]); } finally { if (timer) clearTimeout(timer); } } + private async scanProjectDirsWithBudget( + projectDirs: FsDirent[], + scanStartedAt: number + ): Promise<{ + projectArrays: Project[][]; + completedAll: boolean; + slowest: Array<{ project: string; ms: number; returned: number }>; + }> { + const workerCount = Math.max( + 1, + Math.min( + this.fsProvider.type === 'ssh' ? 8 : ProjectScanner.LOCAL_PROJECT_BATCH, + projectDirs.length + ) + ); + const projectArrays: Project[][] = []; + const slowest: Array<{ project: string; ms: number; returned: number }> = []; + let nextIndex = 0; + let startedDirs = 0; + let completedAll = true; + + const claimNextDir = (): { dir: FsDirent; remainingBudgetMs: number } | null => { + if (nextIndex >= projectDirs.length) { + return null; + } + + const elapsedMs = Date.now() - scanStartedAt; + const remainingBudgetMs = this.scanBudgetMs - elapsedMs; + if (remainingBudgetMs < ProjectScanner.MIN_SCAN_BATCH_BUDGET_MS) { + completedAll = false; + return null; + } + + const dir = projectDirs[nextIndex]; + nextIndex += 1; + startedDirs += 1; + return { dir, remainingBudgetMs }; + }; + + const workers = new Array(workerCount).fill(0).map(async () => { + while (true) { + const claimed = claimNextDir(); + if (!claimed) { + return; + } + const { dir, remainingBudgetMs } = claimed; + const perProjectTimeoutMs = Math.min( + ProjectScanner.SCAN_PROJECT_TIMEOUT_MS, + remainingBudgetMs + ); + try { + const projectStartedAt = Date.now(); + const result = await this.scanProjectWithTimeout(dir.name, perProjectTimeoutMs, { + logTimeout: false, + }); + const ms = Date.now() - projectStartedAt; + if (ms >= 500) { + slowest.push({ project: dir.name, ms, returned: result.projects.length }); + } + if (result.timedOut) { + completedAll = false; + } + projectArrays.push(result.projects); + } catch { + completedAll = false; + } + } + }); + + await Promise.all(workers); + + if (startedDirs < projectDirs.length) { + completedAll = false; + } + + slowest.sort((a, b) => b.ms - a.ms); + if (!completedAll) { + logger.warn( + `[scan] budget exhausted ms=${Date.now() - scanStartedAt} scannedDirs=${startedDirs}/${projectDirs.length} returnedProjectArrays=${projectArrays.length} slowest=${JSON.stringify( + slowest.slice(0, 5) + )}` + ); + } + + return { projectArrays, completedAll, slowest }; + } + /** * Scans a single project directory and returns project metadata. * If sessions have different cwd values, splits into multiple projects. */ - private async scanProject(encodedName: string): Promise { + private async scanProject( + encodedName: string, + options: ScanProjectOptions = {} + ): Promise { try { + this.throwIfScanAborted(options.signal); const projectPath = path.join(this.projectsDir, encodedName); const readdirStart = Date.now(); const entries = await this.readdirForProjectDiscovery(projectPath); + this.throwIfScanAborted(options.signal); const readdirMs = Date.now() - readdirStart; // Get session files (.jsonl at root level) @@ -528,46 +730,63 @@ export class ProjectScanner { cwd: string | null; } - // Reading JSONL heads for cwd across hundreds/thousands of sessions can saturate I/O and - // make the renderer appear frozen while waiting for repository groups. - // Prefer correctness for small projects; for large ones, skip cwd splitting and fall back - // to encoded-path decoding / limited path probing. - const MAX_CWD_SPLIT_FILES = 80; + // Reading JSONL heads for cwd can dominate startup when cold I/O is already contended. + // Keep cwd splitting exact for small project dirs. For larger dirs, read only a newest-session + // cwd hint so the project path stays accurate without building a partial subproject split. const shouldSplitByCwd = - this.fsProvider.type !== 'ssh' && sessionFiles.length <= MAX_CWD_SPLIT_FILES; + this.fsProvider.type !== 'ssh' && + sessionFiles.length <= ProjectScanner.EXHAUSTIVE_CWD_SPLIT_FILE_LIMIT; const sessionStatStart = Date.now(); const sessionInfos = await this.collectFulfilledInBatches( sessionFiles, this.fsProvider.type === 'ssh' ? 32 : ProjectScanner.LOCAL_SESSION_BATCH, async (file) => { + this.throwIfScanAborted(options.signal); const filePath = path.join(projectPath, file.name); - const { mtimeMs, birthtimeMs } = await this.resolveFileDetails(file, filePath); - let cwd: string | null = null; - - // Over SSH, avoid reading every file body during project discovery. - if (shouldSplitByCwd) { - try { - cwd = await extractCwd(filePath, this.fsProvider); - } catch { - // Ignore unreadable files - } - } + const { mtimeMs, birthtimeMs } = await this.runScanFileIo( + () => this.resolveFileDetails(file, filePath), + options.signal + ); return { sessionId: extractSessionId(file.name), filePath, mtimeMs, birthtimeMs, - cwd, + cwd: null as string | null, } satisfies SessionInfo; } ); + this.throwIfScanAborted(options.signal); if (sessionInfos.length === 0) { return []; } + if (this.fsProvider.type !== 'ssh') { + const cwdCandidates = shouldSplitByCwd + ? sessionInfos + : [...sessionInfos] + .sort((a, b) => b.mtimeMs - a.mtimeMs) + .slice(0, ProjectScanner.CWD_HINT_SAMPLE_FILE_LIMIT); + await this.collectFulfilledInBatches(cwdCandidates, 2, async (info) => { + try { + info.cwd = await this.runScanFileIo( + () => extractCwdFromKnownJsonlFile(info.filePath, this.fsProvider), + options.signal + ); + } catch (error) { + if (error instanceof ScanProjectAbortedError || options.signal?.aborted) { + throw error; + } + // Ignore unreadable files + } + return null; + }); + this.throwIfScanAborted(options.signal); + } + const sessionStatMs = Date.now() - sessionStatStart; if (sessionFiles.length > 200 || sessionStatMs > 1000) { logger.debug( @@ -607,11 +826,14 @@ export class ProjectScanner { } const sessionPaths = sessionInfos.map((s) => s.filePath); + this.throwIfScanAborted(options.signal); const actualPath = await this.projectPathResolver.resolveProjectPath(encodedName, { cwdHint: firstCwd ?? undefined, sessionPaths, }); + this.throwIfScanAborted(options.signal); const filesystemState = await resolveProjectFilesystemState(this.fsProvider, actualPath); + this.throwIfScanAborted(options.signal); // Derive name from resolved path — more reliable than decodePath for // paths containing dashes (e.g. "test-project" encodes lossily). @@ -632,7 +854,11 @@ export class ProjectScanner { } // Multiple unique cwds: split into subprojects - const projects: Project[] = []; + const pendingProjects: Array<{ + registryCwd: string; + sessionIds: string[]; + project: Omit; + }> = []; // Find the "root" cwd (shortest path, or the one matching the decoded name) const cwdKeys = [...cwdGroups.keys()].filter((k) => !k.startsWith('__decoded__')); @@ -644,16 +870,11 @@ export class ProjectScanner { const rootName = path.basename(rootCwd) || baseName; for (const [cwdKey, sessions] of cwdGroups) { + this.throwIfScanAborted(options.signal); const isDecodedFallback = cwdKey.startsWith('__decoded__'); const actualCwd = isDecodedFallback ? null : cwdKey; - // Register in subproject registry const sessionIds = sessions.map((s) => s.sessionId); - const compositeId = subprojectRegistry.register( - encodedName, - actualCwd ?? decodedFallback, - sessionIds - ); const exportedSessionIds = sessionIds.slice(0, MAX_SESSION_IDS_EXPORTED); // Compute timestamps @@ -677,24 +898,41 @@ export class ProjectScanner { const lastSegment = path.basename(actualCwd); displayName = `${rootName} (${lastSegment})`; } - - projects.push({ - id: compositeId, - path: actualCwd ?? decodedFallback, - name: displayName, - sessions: exportedSessionIds, - totalSessions: sessionIds.length, - createdAt: Math.floor(createdAt), - mostRecentSession: mostRecentSession ? Math.floor(mostRecentSession) : undefined, - filesystemState: await resolveProjectFilesystemState( - this.fsProvider, - actualCwd ?? decodedFallback - ), + const filesystemState = await resolveProjectFilesystemState( + this.fsProvider, + actualCwd ?? decodedFallback + ); + this.throwIfScanAborted(options.signal); + if (options.shouldCommitSubprojects?.() === false) { + return []; + } + pendingProjects.push({ + registryCwd: actualCwd ?? decodedFallback, + sessionIds, + project: { + path: actualCwd ?? decodedFallback, + name: displayName, + sessions: exportedSessionIds, + totalSessions: sessionIds.length, + createdAt: Math.floor(createdAt), + mostRecentSession: mostRecentSession ? Math.floor(mostRecentSession) : undefined, + filesystemState, + }, }); } - return projects; + if (options.shouldCommitSubprojects?.() === false) { + return []; + } + + return pendingProjects.map(({ registryCwd, sessionIds, project }) => ({ + id: subprojectRegistry.register(encodedName, registryCwd, sessionIds), + ...project, + })); } catch (error) { + if (error instanceof ScanProjectAbortedError || options.signal?.aborted) { + return []; + } logger.error(`Error scanning project ${encodedName}:`, error); return []; } @@ -1676,6 +1914,79 @@ export class ProjectScanner { return results; } + private throwIfScanAborted(signal?: AbortSignal): void { + if (signal?.aborted) { + throw new ScanProjectAbortedError(); + } + } + + private async runScanFileIo(operation: () => Promise, signal?: AbortSignal): Promise { + this.throwIfScanAborted(signal); + if (this.fsProvider.type !== 'local') { + const result = await operation(); + this.throwIfScanAborted(signal); + return result; + } + + await this.acquireScanFileIoSlot(signal); + try { + this.throwIfScanAborted(signal); + const result = await operation(); + this.throwIfScanAborted(signal); + return result; + } finally { + this.releaseScanFileIoSlot(); + } + } + + private async acquireScanFileIoSlot(signal?: AbortSignal): Promise { + this.throwIfScanAborted(signal); + if (this.scanFileIoActive < this.scanFileIoConcurrency) { + this.scanFileIoActive += 1; + return; + } + + await new Promise((resolve, reject) => { + let resume: (() => void) | null = null; + const cleanup = () => { + signal?.removeEventListener('abort', onAbort); + }; + const onAbort = () => { + const queuedResume = resume; + if (!queuedResume) { + return; + } + resume = null; + const index = this.scanFileIoQueue.indexOf(queuedResume); + if (index >= 0) { + this.scanFileIoQueue.splice(index, 1); + } + cleanup(); + reject(new ScanProjectAbortedError()); + }; + resume = () => { + if (!resume) { + return; + } + resume = null; + cleanup(); + resolve(); + }; + this.scanFileIoQueue.push(resume); + signal?.addEventListener('abort', onAbort, { once: true }); + }); + this.throwIfScanAborted(signal); + } + + private releaseScanFileIoSlot(): void { + const next = this.scanFileIoQueue.shift(); + if (next) { + next(); + return; + } + this.scanFileIoActive = Math.max(0, this.scanFileIoActive - 1); + } + private getErrorCode(error: unknown): string { if (typeof error === 'object' && error !== null && 'code' in error) { const code = (error as { code?: unknown }).code; diff --git a/src/main/services/discovery/SubprojectRegistry.ts b/src/main/services/discovery/SubprojectRegistry.ts index 823996d7..929d518c 100644 --- a/src/main/services/discovery/SubprojectRegistry.ts +++ b/src/main/services/discovery/SubprojectRegistry.ts @@ -19,6 +19,13 @@ interface SubprojectEntry { sessionIds: Set; } +export interface SubprojectRegistrySnapshotEntry { + id: string; + baseDir: string; + cwd: string; + sessionIds: string[]; +} + class SubprojectRegistryImpl { private readonly entries = new Map(); @@ -92,6 +99,33 @@ class SubprojectRegistryImpl { clear(): void { this.entries.clear(); } + + /** + * Capture a deep copy of the current registry so an interrupted scan can + * restore the previous complete project view. + */ + snapshot(): SubprojectRegistrySnapshotEntry[] { + return [...this.entries.entries()].map(([id, entry]) => ({ + id, + baseDir: entry.baseDir, + cwd: entry.cwd, + sessionIds: [...entry.sessionIds], + })); + } + + /** + * Replace the registry contents with a previously captured snapshot. + */ + restore(snapshot: readonly SubprojectRegistrySnapshotEntry[]): void { + this.entries.clear(); + for (const entry of snapshot) { + this.entries.set(entry.id, { + baseDir: entry.baseDir, + cwd: entry.cwd, + sessionIds: new Set(entry.sessionIds), + }); + } + } } /** Module-level singleton */ diff --git a/src/main/utils/jsonl.ts b/src/main/utils/jsonl.ts index 976f63c6..e61b1d1e 100644 --- a/src/main/utils/jsonl.ts +++ b/src/main/utils/jsonl.ts @@ -41,7 +41,11 @@ const logger = createLogger('Util:jsonl'); const defaultProvider = new LocalFileSystemProvider(); // Re-export for backwards compatibility -export { extractCwd, extractFirstUserMessagePreview } from './metadataExtraction'; +export { + extractCwd, + extractCwdFromKnownJsonlFile, + extractFirstUserMessagePreview, +} from './metadataExtraction'; export { checkMessagesOngoing } from './sessionStateDetection'; // ============================================================================= diff --git a/src/main/utils/metadataExtraction.ts b/src/main/utils/metadataExtraction.ts index 344e565b..4e9c95c4 100644 --- a/src/main/utils/metadataExtraction.ts +++ b/src/main/utils/metadataExtraction.ts @@ -89,27 +89,10 @@ async function extractCwdFromLocalFile(filePath: string): Promise } } -/** - * Extract CWD (current working directory) from the first entry. - * Used to get the actual project path from encoded directory names. - */ -export async function extractCwd( +async function extractCwdFromReadableFile( filePath: string, - fsProvider: FileSystemProvider = defaultProvider + fsProvider: FileSystemProvider ): Promise { - if (!(await fsProvider.exists(filePath))) { - return null; - } - - try { - const stat = await fsProvider.stat(filePath); - if (!stat.isFile()) { - return null; - } - } catch { - return null; - } - if (fsProvider.type === 'local') { try { return await extractCwdFromLocalFile(filePath); @@ -173,6 +156,41 @@ export async function extractCwd( return null; } +/** + * Extract CWD from a JSONL file that the caller already discovered as a file. + * This skips exists/stat preflight for hot project-discovery paths. + */ +export async function extractCwdFromKnownJsonlFile( + filePath: string, + fsProvider: FileSystemProvider = defaultProvider +): Promise { + return extractCwdFromReadableFile(filePath, fsProvider); +} + +/** + * Extract CWD (current working directory) from the first entry. + * Used to get the actual project path from encoded directory names. + */ +export async function extractCwd( + filePath: string, + fsProvider: FileSystemProvider = defaultProvider +): Promise { + if (!(await fsProvider.exists(filePath))) { + return null; + } + + try { + const stat = await fsProvider.stat(filePath); + if (!stat.isFile()) { + return null; + } + } catch { + return null; + } + + return extractCwdFromReadableFile(filePath, fsProvider); +} + /** * Extract a lightweight title preview from the first user message. * For command-style sessions, falls back to a slash-command label. diff --git a/test/features/recent-projects/main/adapters/output/CodexSessionFileRecentProjectsSourceAdapter.test.ts b/test/features/recent-projects/main/adapters/output/CodexSessionFileRecentProjectsSourceAdapter.test.ts index 0e429ea5..691edafe 100644 --- a/test/features/recent-projects/main/adapters/output/CodexSessionFileRecentProjectsSourceAdapter.test.ts +++ b/test/features/recent-projects/main/adapters/output/CodexSessionFileRecentProjectsSourceAdapter.test.ts @@ -2,9 +2,8 @@ import fs from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; - import { CodexSessionFileRecentProjectsSourceAdapter } from '@features/recent-projects/main/adapters/output/sources/CodexSessionFileRecentProjectsSourceAdapter'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import type { LoggerPort } from '@features/recent-projects/core/application/ports/LoggerPort'; import type { RecentProjectIdentityResolver } from '@features/recent-projects/main/infrastructure/identity/RecentProjectIdentityResolver'; @@ -21,6 +20,10 @@ function createLogger(): LoggerPort & { }; } +function getSessionFileCachePath(appDataPath: string): string { + return path.join(appDataPath, 'recent-projects', 'codex-session-files-index.json'); +} + async function writeRollout( filePath: string, payload: { @@ -91,6 +94,7 @@ describe('CodexSessionFileRecentProjectsSourceAdapter', () => { identityResolver, logger, codexHome, + appDataPath: path.join(tempDir, 'app-data'), }); await expect(adapter.list()).resolves.toEqual({ @@ -137,6 +141,7 @@ describe('CodexSessionFileRecentProjectsSourceAdapter', () => { identityResolver, logger, codexHome, + appDataPath: path.join(tempDir, 'app-data'), }); const result = await adapter.list(); @@ -172,6 +177,7 @@ describe('CodexSessionFileRecentProjectsSourceAdapter', () => { identityResolver, logger, codexHome, + appDataPath: path.join(tempDir, 'app-data'), }); const result = await adapter.list(); @@ -213,6 +219,7 @@ describe('CodexSessionFileRecentProjectsSourceAdapter', () => { identityResolver, logger, codexHome, + appDataPath: path.join(tempDir, 'app-data'), }); const result = await adapter.list(); @@ -263,6 +270,7 @@ describe('CodexSessionFileRecentProjectsSourceAdapter', () => { identityResolver, logger, codexHome, + appDataPath: path.join(tempDir, 'app-data'), }); const result = await adapter.list(); @@ -273,6 +281,300 @@ describe('CodexSessionFileRecentProjectsSourceAdapter', () => { ]); }); + it('reuses cached unchanged session metadata without reopening jsonl files', async () => { + const codexHome = path.join(tempDir, '.codex'); + const appDataPath = path.join(tempDir, 'app-data'); + const logger = createLogger(); + const identityResolver = { + resolve: vi.fn().mockResolvedValue(null), + } as unknown as RecentProjectIdentityResolver; + const updatedAt = new Date('2026-04-14T12:00:00.000Z'); + await writeRollout( + path.join(codexHome, 'sessions', '2026', '04', '14', 'rollout-alpha.jsonl'), + { + cwd: '/Users/test/projects/alpha', + branch: 'main', + }, + updatedAt + ); + + const adapter = new CodexSessionFileRecentProjectsSourceAdapter({ + getActiveContext: () => ({ type: 'local', id: 'local-1' }) as never, + getLocalContext: () => ({ type: 'local', id: 'local-1' }) as never, + identityResolver, + logger, + codexHome, + appDataPath, + }); + await expect(adapter.list()).resolves.toEqual({ + candidates: [ + expect.objectContaining({ + primaryPath: '/Users/test/projects/alpha', + branchName: 'main', + }), + ], + degraded: false, + }); + + const openSpy = vi.spyOn(fs, 'open'); + const cachedAdapter = new CodexSessionFileRecentProjectsSourceAdapter({ + getActiveContext: () => ({ type: 'local', id: 'local-1' }) as never, + getLocalContext: () => ({ type: 'local', id: 'local-1' }) as never, + identityResolver, + logger, + codexHome, + appDataPath, + }); + + await expect(cachedAdapter.list()).resolves.toEqual({ + candidates: [ + expect.objectContaining({ + primaryPath: '/Users/test/projects/alpha', + branchName: 'main', + }), + ], + degraded: false, + }); + expect(openSpy).not.toHaveBeenCalled(); + }); + + it('invalidates cached session metadata when the jsonl fingerprint changes', async () => { + const codexHome = path.join(tempDir, '.codex'); + const appDataPath = path.join(tempDir, 'app-data'); + const logger = createLogger(); + const identityResolver = { + resolve: vi.fn().mockResolvedValue(null), + } as unknown as RecentProjectIdentityResolver; + const sessionPath = path.join( + codexHome, + 'sessions', + '2026', + '04', + '14', + 'rollout-active.jsonl' + ); + await writeRollout( + sessionPath, + { + cwd: '/Users/test/projects/alpha', + }, + new Date('2026-04-14T12:00:00.000Z') + ); + + const adapter = new CodexSessionFileRecentProjectsSourceAdapter({ + getActiveContext: () => ({ type: 'local', id: 'local-1' }) as never, + getLocalContext: () => ({ type: 'local', id: 'local-1' }) as never, + identityResolver, + logger, + codexHome, + appDataPath, + }); + await expect(adapter.list()).resolves.toEqual({ + candidates: [ + expect.objectContaining({ + primaryPath: '/Users/test/projects/alpha', + }), + ], + degraded: false, + }); + + await writeRollout( + sessionPath, + { + cwd: '/Users/test/projects/beta', + }, + new Date('2026-04-14T12:01:00.000Z') + ); + + const refreshedAdapter = new CodexSessionFileRecentProjectsSourceAdapter({ + getActiveContext: () => ({ type: 'local', id: 'local-1' }) as never, + getLocalContext: () => ({ type: 'local', id: 'local-1' }) as never, + identityResolver, + logger, + codexHome, + appDataPath, + }); + + await expect(refreshedAdapter.list()).resolves.toEqual({ + candidates: [ + expect.objectContaining({ + primaryPath: '/Users/test/projects/beta', + }), + ], + degraded: false, + }); + }); + + it('does not let a slow jsonl read hold the whole source past its timeout budget', async () => { + const codexHome = path.join(tempDir, '.codex'); + const appDataPath = path.join(tempDir, 'app-data'); + const logger = createLogger(); + const identityResolver = { + resolve: vi.fn().mockResolvedValue(null), + } as unknown as RecentProjectIdentityResolver; + const baseTime = Date.parse('2026-04-14T12:00:00.000Z'); + const slowSessionPath = path.join( + codexHome, + 'sessions', + '2026', + '04', + '14', + 'rollout-slow.jsonl' + ); + await writeRollout( + slowSessionPath, + { + cwd: '/Users/test/projects/slow', + }, + new Date(baseTime) + ); + await writeRollout( + path.join(codexHome, 'sessions', '2026', '04', '14', 'rollout-fast.jsonl'), + { + cwd: '/Users/test/projects/fast', + }, + new Date(baseTime - 1000) + ); + const originalOpen = fs.open.bind(fs); + vi.spyOn(fs, 'open').mockImplementation(async (...args) => { + if (String(args[0]) === slowSessionPath) { + await new Promise((resolve) => setTimeout(resolve, 2500)); + } + return originalOpen(...args); + }); + + const startedAt = Date.now(); + const adapter = new CodexSessionFileRecentProjectsSourceAdapter({ + getActiveContext: () => ({ type: 'local', id: 'local-1' }) as never, + getLocalContext: () => ({ type: 'local', id: 'local-1' }) as never, + identityResolver, + logger, + codexHome, + appDataPath, + }); + const result = await adapter.list(); + + expect(Date.now() - startedAt).toBeLessThan(1600); + expect(result.degraded).toBe(true); + expect(result.candidates.map((candidate) => candidate.primaryPath)).toEqual([ + '/Users/test/projects/fast', + ]); + expect(logger.warn).toHaveBeenCalledWith( + 'codex session-file recent-projects source partial', + expect.objectContaining({ + files: 2, + timedOutReads: 1, + }) + ); + }); + + it('ignores a corrupt session-file cache and rebuilds from session files', async () => { + const codexHome = path.join(tempDir, '.codex'); + const appDataPath = path.join(tempDir, 'app-data'); + const logger = createLogger(); + const identityResolver = { + resolve: vi.fn().mockResolvedValue(null), + } as unknown as RecentProjectIdentityResolver; + await writeRollout( + path.join(codexHome, 'sessions', '2026', '04', '14', 'rollout-alpha.jsonl'), + { + cwd: '/Users/test/projects/alpha', + }, + new Date('2026-04-14T12:00:00.000Z') + ); + const cachePath = getSessionFileCachePath(appDataPath); + await fs.mkdir(path.dirname(cachePath), { recursive: true }); + await fs.writeFile(cachePath, '{not-json', 'utf8'); + + const adapter = new CodexSessionFileRecentProjectsSourceAdapter({ + getActiveContext: () => ({ type: 'local', id: 'local-1' }) as never, + getLocalContext: () => ({ type: 'local', id: 'local-1' }) as never, + identityResolver, + logger, + codexHome, + appDataPath, + }); + + await expect(adapter.list()).resolves.toEqual({ + candidates: [ + expect.objectContaining({ + primaryPath: '/Users/test/projects/alpha', + }), + ], + degraded: false, + }); + }); + + it('returns a degraded partial result under the uncached read cap and completes on the next cached pass', async () => { + const codexHome = path.join(tempDir, '.codex'); + const appDataPath = path.join(tempDir, 'app-data'); + const logger = createLogger(); + const identityResolver = { + resolve: vi.fn().mockResolvedValue(null), + } as unknown as RecentProjectIdentityResolver; + const baseTime = Date.parse('2026-04-14T12:00:00.000Z'); + + await Promise.all( + Array.from({ length: 170 }).map((_, index) => + writeRollout( + path.join(codexHome, 'sessions', '2026', '04', '14', `rollout-alpha-${index}.jsonl`), + { + cwd: '/Users/test/projects/alpha', + branch: 'main', + }, + new Date(baseTime - index * 1000) + ) + ) + ); + await writeRollout( + path.join(codexHome, 'sessions', '2026', '04', '14', 'rollout-beta.jsonl'), + { + cwd: '/Users/test/projects/beta', + branch: 'main', + }, + new Date(baseTime - 200_000) + ); + + const firstAdapter = new CodexSessionFileRecentProjectsSourceAdapter({ + getActiveContext: () => ({ type: 'local', id: 'local-1' }) as never, + getLocalContext: () => ({ type: 'local', id: 'local-1' }) as never, + identityResolver, + logger, + codexHome, + appDataPath, + }); + const firstResult = await firstAdapter.list(); + + expect(firstResult.degraded).toBe(true); + expect(firstResult.candidates.map((candidate) => candidate.primaryPath)).toEqual([ + '/Users/test/projects/alpha', + ]); + expect(logger.warn).toHaveBeenCalledWith( + 'codex session-file recent-projects source partial', + expect.objectContaining({ + files: 171, + uncachedReads: 160, + skippedUncached: 11, + }) + ); + + const secondAdapter = new CodexSessionFileRecentProjectsSourceAdapter({ + getActiveContext: () => ({ type: 'local', id: 'local-1' }) as never, + getLocalContext: () => ({ type: 'local', id: 'local-1' }) as never, + identityResolver, + logger, + codexHome, + appDataPath, + }); + const secondResult = await secondAdapter.list(); + + expect(secondResult.degraded).toBe(false); + expect(secondResult.candidates.map((candidate) => candidate.primaryPath)).toEqual([ + '/Users/test/projects/alpha', + '/Users/test/projects/beta', + ]); + }); + it('skips non-interactive and ephemeral sessions', async () => { const codexHome = path.join(tempDir, '.codex'); const logger = createLogger(); @@ -302,6 +604,7 @@ describe('CodexSessionFileRecentProjectsSourceAdapter', () => { identityResolver, logger, codexHome, + appDataPath: path.join(tempDir, 'app-data'), }); await expect(adapter.list()).resolves.toEqual({ @@ -322,6 +625,7 @@ describe('CodexSessionFileRecentProjectsSourceAdapter', () => { identityResolver, logger, codexHome: path.join(tempDir, 'missing-codex-home'), + appDataPath: path.join(tempDir, 'app-data'), }); await expect(adapter.list()).resolves.toEqual({ diff --git a/test/main/services/discovery/ProjectScanner.scanDedup.safe-e2e.test.ts b/test/main/services/discovery/ProjectScanner.scanDedup.safe-e2e.test.ts index f32c1da3..86049e5d 100644 --- a/test/main/services/discovery/ProjectScanner.scanDedup.safe-e2e.test.ts +++ b/test/main/services/discovery/ProjectScanner.scanDedup.safe-e2e.test.ts @@ -3,10 +3,11 @@ import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; -import { afterEach, describe, expect, it } from 'vitest'; +import { afterEach, describe, expect, it, vi } from 'vitest'; import { ProjectScanner } from '../../../../src/main/services/discovery/ProjectScanner'; import { subprojectRegistry } from '../../../../src/main/services/discovery/SubprojectRegistry'; +import { configManager } from '../../../../src/main/services/infrastructure/ConfigManager'; import type { FileSystemProvider, @@ -18,6 +19,9 @@ import type { interface CountingProvider extends FileSystemProvider { readonly readdirCounts: Map; getMaxConcurrentReaddirs(): number; + getMaxConcurrentStats(): number; + getStatCount(): number; + setProjectReaddirDelay(ms: number): void; releaseBlockedRead(): void; waitForBlockedRead(): Promise; } @@ -32,13 +36,44 @@ function createSessionLine(cwd: string): string { }); } -function createProject(projectsDir: string, encodedName: string, cwd: string): string { +function createProject( + projectsDir: string, + encodedName: string, + cwd: string, + sessionCount = 1 +): string { const projectDir = path.join(projectsDir, encodedName); fs.mkdirSync(projectDir, { recursive: true }); - fs.writeFileSync(path.join(projectDir, 'session-1.jsonl'), `${createSessionLine(cwd)}\n`); + for (let index = 1; index <= sessionCount; index += 1) { + fs.writeFileSync( + path.join(projectDir, `session-${index}.jsonl`), + `${createSessionLine(cwd)}\n` + ); + } return projectDir; } +function createSplitProject( + projectsDir: string, + encodedName: string, + cwds: readonly string[] +): string { + const projectDir = path.join(projectsDir, encodedName); + fs.mkdirSync(projectDir, { recursive: true }); + cwds.forEach((cwd, index) => { + fs.writeFileSync( + path.join(projectDir, `split-session-${index + 1}.jsonl`), + `${createSessionLine(cwd)}\n` + ); + }); + return projectDir; +} + +function buildCompositeId(baseDir: string, cwd: string): string { + const hash = crypto.createHash('sha256').update(cwd).digest('hex').slice(0, 8); + return `${baseDir}::${hash}`; +} + function toStatResult(stats: fs.Stats): FsStatResult { return { size: stats.size, @@ -57,12 +92,47 @@ function toDirent(entry: fs.Dirent): FsDirent { }; } -function createCountingProvider(blockFirstReadPath?: string): CountingProvider { +interface CountingProviderOptions { + blockFirstReadPath?: string; + projectReaddirDelayMs?: number; + statDelayMs?: number; +} + +function delay(ms: number): Promise { + return new Promise((resolve) => { + setTimeout(resolve, ms); + }); +} + +async function waitForCondition( + predicate: () => boolean, + timeoutMs = 1000, + intervalMs = 20 +): Promise { + const startedAt = Date.now(); + while (Date.now() - startedAt < timeoutMs) { + if (predicate()) { + return true; + } + await delay(intervalMs); + } + return predicate(); +} + +function createCountingProvider(options: string | CountingProviderOptions = {}): CountingProvider { + const blockFirstReadPath = typeof options === 'string' ? options : options.blockFirstReadPath; + let projectReaddirDelayMs = + typeof options === 'string' ? 0 : (options.projectReaddirDelayMs ?? 0); + const statDelayMs = typeof options === 'string' ? 0 : (options.statDelayMs ?? 0); const readdirCounts = new Map(); + let rootReaddirPath: string | null = null; let blockedReadStartedResolve: (() => void) | null = null; let releaseBlockedReadResolve: (() => void) | null = null; let activeReaddirs = 0; let maxConcurrentReaddirs = 0; + let activeStats = 0; + let maxConcurrentStats = 0; + let statCount = 0; const blockedReadStarted = blockFirstReadPath ? new Promise((resolve) => { blockedReadStartedResolve = resolve; @@ -79,14 +149,28 @@ function createCountingProvider(blockFirstReadPath?: string): CountingProvider { return fs.promises.readFile(filePath, encoding); }, async stat(filePath: string): Promise { - return toStatResult(await fs.promises.stat(filePath)); + statCount += 1; + activeStats += 1; + maxConcurrentStats = Math.max(maxConcurrentStats, activeStats); + try { + if (statDelayMs > 0) { + await delay(statDelayMs); + } + return toStatResult(await fs.promises.stat(filePath)); + } finally { + activeStats -= 1; + } }, async readdir(dirPath: string): Promise { activeReaddirs += 1; maxConcurrentReaddirs = Math.max(maxConcurrentReaddirs, activeReaddirs); try { + rootReaddirPath ??= dirPath; const count = (readdirCounts.get(dirPath) ?? 0) + 1; readdirCounts.set(dirPath, count); + if (projectReaddirDelayMs > 0 && dirPath !== rootReaddirPath) { + await delay(projectReaddirDelayMs); + } if (dirPath === blockFirstReadPath && count === 1) { blockedReadStartedResolve?.(); await new Promise((resolve) => { @@ -110,6 +194,15 @@ function createCountingProvider(blockFirstReadPath?: string): CountingProvider { getMaxConcurrentReaddirs(): number { return maxConcurrentReaddirs; }, + getMaxConcurrentStats(): number { + return maxConcurrentStats; + }, + getStatCount(): number { + return statCount; + }, + setProjectReaddirDelay(ms: number): void { + projectReaddirDelayMs = Math.max(0, ms); + }, releaseBlockedRead(): void { releaseBlockedReadResolve?.(); }, @@ -124,6 +217,7 @@ describe('ProjectScanner scan dedup safe e2e', () => { afterEach(() => { subprojectRegistry.clear(); + vi.restoreAllMocks(); for (const dir of tempDirs) { fs.rmSync(dir, { recursive: true, force: true }); } @@ -150,6 +244,315 @@ describe('ProjectScanner scan dedup safe e2e', () => { expect(provider.readdirCounts.get(projectDir)).toBe(1); }); + it('shares one in-flight repository-group build including custom path enrichment', async () => { + const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), 'scanner-groups-dedup-')); + tempDirs.push(rootDir); + const projectsDir = path.join(rootDir, 'projects'); + const encodedName = '-Users-test-groups-project'; + createProject(projectsDir, encodedName, '/Users/test/groups-project'); + const customPath = path.join(rootDir, 'manual-project'); + fs.mkdirSync(customPath, { recursive: true }); + const getCustomProjectPathsSpy = vi + .spyOn(configManager, 'getCustomProjectPaths') + .mockReturnValue([customPath]); + const provider = createCountingProvider(); + const scanner = new ProjectScanner(projectsDir, undefined, provider); + + const [firstGroups, secondGroups] = await Promise.all([ + scanner.scanWithWorktreeGrouping(), + scanner.scanWithWorktreeGrouping(), + ]); + + expect(firstGroups).toHaveLength(2); + expect(secondGroups).toEqual(firstGroups); + expect(getCustomProjectPathsSpy).toHaveBeenCalledTimes(1); + }); + + it('does not reuse stale repository groups when custom project paths change', async () => { + const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), 'scanner-groups-custom-fresh-')); + tempDirs.push(rootDir); + const projectsDir = path.join(rootDir, 'projects'); + createProject(projectsDir, '-Users-test-custom-fresh-project', '/Users/test/custom-fresh'); + const firstCustomPath = path.join(rootDir, 'first-manual-project'); + const secondCustomPath = path.join(rootDir, 'second-manual-project'); + fs.mkdirSync(firstCustomPath, { recursive: true }); + fs.mkdirSync(secondCustomPath, { recursive: true }); + vi.spyOn(configManager, 'getCustomProjectPaths') + .mockReturnValueOnce([firstCustomPath]) + .mockReturnValueOnce([secondCustomPath]); + const provider = createCountingProvider(); + const scanner = new ProjectScanner(projectsDir, undefined, provider); + + const firstGroups = await scanner.scanWithWorktreeGrouping(); + const secondGroups = await scanner.scanWithWorktreeGrouping(); + + expect(firstGroups.some((group) => group.worktrees[0]?.path === firstCustomPath)).toBe(true); + expect(secondGroups.some((group) => group.worktrees[0]?.path === secondCustomPath)).toBe(true); + expect(secondGroups.some((group) => group.worktrees[0]?.path === firstCustomPath)).toBe(false); + }); + + it('continues scanning later project directories while one project is slow', async () => { + const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), 'scanner-rolling-projects-')); + tempDirs.push(rootDir); + const projectsDir = path.join(rootDir, 'projects'); + const projectDirs: string[] = []; + for (let projectIndex = 0; projectIndex < 14; projectIndex += 1) { + projectDirs.push( + createProject( + projectsDir, + `-Users-test-rolling-${projectIndex}`, + `/Users/test/rolling-${projectIndex}` + ) + ); + } + const provider = createCountingProvider(projectDirs[0]); + const scanner = new ProjectScanner(projectsDir, undefined, provider); + + const scan = scanner.scan(); + await provider.waitForBlockedRead(); + const laterProjectScanned = await waitForCondition( + () => (provider.readdirCounts.get(projectDirs[13]) ?? 0) > 0 + ); + + provider.releaseBlockedRead(); + const projects = await scan; + + expect(laterProjectScanned).toBe(true); + expect(projects).toHaveLength(14); + }); + + it('uses a bounded cwd hint instead of partial splitting for larger project dirs', async () => { + const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), 'scanner-bounded-cwd-')); + tempDirs.push(rootDir); + const projectsDir = path.join(rootDir, 'projects'); + const encodedName = '-Users-test-bounded-cwd'; + createSplitProject( + projectsDir, + encodedName, + Array.from({ length: 13 }, (_value, index) => `/Users/test/bounded-cwd/app-${index}`) + ); + const provider = createCountingProvider(); + const scanner = new ProjectScanner(projectsDir, undefined, provider); + + const projects = await scanner.scan(); + + expect(projects).toHaveLength(1); + expect(projects[0]).toMatchObject({ + id: encodedName, + totalSessions: 13, + }); + expect(projects[0]?.path).toContain('/Users/test/bounded-cwd/app-'); + expect(subprojectRegistry.isComposite(projects[0]?.id ?? '')).toBe(false); + }); + + it('bounds local scan stat concurrency without restatting files during cwd extraction', async () => { + const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), 'scanner-io-limit-')); + tempDirs.push(rootDir); + const projectsDir = path.join(rootDir, 'projects'); + for (let projectIndex = 0; projectIndex < 6; projectIndex += 1) { + createProject( + projectsDir, + `-Users-test-io-limit-${projectIndex}`, + `/Users/test/io-limit-${projectIndex}`, + 5 + ); + } + const provider = createCountingProvider({ statDelayMs: 10 }); + const scanner = new ProjectScanner(projectsDir, undefined, provider, { + scanFileIoConcurrency: 2, + }); + + const projects = await scanner.scan(); + + expect(projects).toHaveLength(6); + expect(provider.getStatCount()).toBe(30); + expect(provider.getMaxConcurrentStats()).toBeLessThanOrEqual(2); + }); + + it('aborts queued local file I/O after a project scan budget timeout', async () => { + const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), 'scanner-io-abort-')); + tempDirs.push(rootDir); + const projectsDir = path.join(rootDir, 'projects'); + createProject(projectsDir, '-Users-test-io-abort', '/Users/test/io-abort', 8); + const provider = createCountingProvider({ statDelayMs: 250 }); + const scanner = new ProjectScanner(projectsDir, undefined, provider, { + scanBudgetMs: 1300, + scanFileIoConcurrency: 1, + }); + + await expect(scanner.scan()).resolves.toEqual([]); + await delay(2400); + + expect(provider.getStatCount()).toBeGreaterThan(0); + expect(provider.getStatCount()).toBeLessThan(8); + }); + + it('returns an uncached partial scan before the scan budget can hit renderer timeout', async () => { + const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), 'scanner-budget-')); + tempDirs.push(rootDir); + const projectsDir = path.join(rootDir, 'projects'); + const projectCount = 6; + for (let projectIndex = 0; projectIndex < projectCount; projectIndex += 1) { + createProject( + projectsDir, + `-Users-test-budget-${projectIndex}`, + `/Users/test/budget-${projectIndex}` + ); + } + const provider = createCountingProvider({ projectReaddirDelayMs: 250 }); + const scanner = new ProjectScanner(projectsDir, undefined, provider, { + scanBudgetMs: 75, + }); + + const firstStartedAt = Date.now(); + const firstProjects = await scanner.scan(); + const firstMs = Date.now() - firstStartedAt; + + expect(firstMs).toBeLessThan(1000); + expect(firstProjects).toEqual([]); + expect(provider.readdirCounts.get(projectsDir)).toBe(1); + + await delay(300); + await scanner.scan(); + expect(provider.readdirCounts.get(projectsDir)).toBe(2); + }); + + it('does not cache a scan when project-level timeout omits directories', async () => { + const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), 'scanner-project-timeout-')); + tempDirs.push(rootDir); + const projectsDir = path.join(rootDir, 'projects'); + for (let projectIndex = 0; projectIndex < 2; projectIndex += 1) { + createProject( + projectsDir, + `-Users-test-project-timeout-${projectIndex}`, + `/Users/test/project-timeout-${projectIndex}` + ); + } + const provider = createCountingProvider({ projectReaddirDelayMs: 1100 }); + const scanner = new ProjectScanner(projectsDir, undefined, provider, { + scanBudgetMs: 1000, + }); + + const firstProjects = await scanner.scan(); + const firstRootReads = provider.readdirCounts.get(projectsDir); + const secondProjects = await scanner.scan(); + + expect(firstProjects).toEqual([]); + expect(secondProjects).toEqual([]); + expect(firstRootReads).toBe(1); + expect(provider.readdirCounts.get(projectsDir)).toBe(2); + }); + + it('does not let a timed-out background project scan mutate the subproject registry', async () => { + const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), 'scanner-timeout-registry-')); + tempDirs.push(rootDir); + const projectsDir = path.join(rootDir, 'projects'); + const encodedName = '-Users-test-timeout-registry'; + const firstCwd = '/Users/test/timeout-registry/app'; + const secondCwd = '/Users/test/timeout-registry/docs'; + createSplitProject(projectsDir, encodedName, [firstCwd, secondCwd]); + const provider = createCountingProvider({ statDelayMs: 1200 }); + const scanner = new ProjectScanner(projectsDir, undefined, provider, { + scanBudgetMs: 1050, + scanFileIoConcurrency: 2, + }); + + await expect(scanner.scan()).resolves.toEqual([]); + await delay(1600); + + expect(subprojectRegistry.getCwd(buildCompositeId(encodedName, firstCwd))).toBeNull(); + expect(subprojectRegistry.getCwd(buildCompositeId(encodedName, secondCwd))).toBeNull(); + }); + + it('returns the previous complete scan instead of replacing it with a budget partial', async () => { + const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), 'scanner-budget-cache-')); + tempDirs.push(rootDir); + const projectsDir = path.join(rootDir, 'projects'); + const projectCount = 6; + for (let projectIndex = 0; projectIndex < projectCount; projectIndex += 1) { + createProject( + projectsDir, + `-Users-test-budget-cache-${projectIndex}`, + `/Users/test/budget-cache-${projectIndex}` + ); + } + const provider = createCountingProvider(); + const scanner = new ProjectScanner(projectsDir, undefined, provider, { + scanBudgetMs: 1500, + }); + + const firstProjects = await scanner.scan(); + expect(firstProjects).toHaveLength(projectCount); + + provider.setProjectReaddirDelay(1700); + await delay(2100); + const secondProjects = await scanner.scan(); + + expect(secondProjects).toHaveLength(projectCount); + expect(secondProjects.map((project) => project.id)).toEqual( + firstProjects.map((project) => project.id) + ); + expect(provider.readdirCounts.get(projectsDir)).toBe(2); + }); + + it('restores subproject registry when falling back to a previous complete scan', async () => { + const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), 'scanner-budget-registry-')); + tempDirs.push(rootDir); + const projectsDir = path.join(rootDir, 'projects'); + createSplitProject(projectsDir, '-Users-test-registry-project', [ + '/Users/test/registry-project/app', + '/Users/test/registry-project/docs', + ]); + const provider = createCountingProvider(); + const scanner = new ProjectScanner(projectsDir, undefined, provider, { + scanBudgetMs: 1500, + }); + + const firstProjects = await scanner.scan(); + expect(firstProjects).toHaveLength(2); + expect(firstProjects.every((project) => subprojectRegistry.isComposite(project.id))).toBe(true); + const firstFilters = firstProjects.map((project) => + subprojectRegistry.getSessionFilter(project.id) + ); + expect(firstFilters.every((filter) => filter?.size === 1)).toBe(true); + + provider.setProjectReaddirDelay(1700); + await delay(2100); + const secondProjects = await scanner.scan(); + + expect(secondProjects.map((project) => project.id)).toEqual( + firstProjects.map((project) => project.id) + ); + for (const project of secondProjects) { + expect(subprojectRegistry.getSessionFilter(project.id)?.size).toBe(1); + expect(subprojectRegistry.getCwd(project.id)).toContain('/Users/test/registry-project/'); + } + }); + + it('keeps an in-flight split-project scan complete after cache invalidation', async () => { + const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), 'scanner-split-inflight-')); + tempDirs.push(rootDir); + const projectsDir = path.join(rootDir, 'projects'); + const projectDir = createSplitProject(projectsDir, '-Users-test-split-inflight', [ + '/Users/test/split-inflight/app', + '/Users/test/split-inflight/docs', + ]); + const provider = createCountingProvider(projectDir); + const scanner = new ProjectScanner(projectsDir, undefined, provider); + + const firstScan = scanner.scan(); + await provider.waitForBlockedRead(); + scanner.clearScanCache(); + provider.releaseBlockedRead(); + const projects = await firstScan; + + expect(projects).toHaveLength(2); + expect(projects.every((project) => subprojectRegistry.isComposite(project.id))).toBe(true); + expect( + projects.every((project) => subprojectRegistry.getSessionFilter(project.id)?.size === 1) + ).toBe(true); + }); + it('does not cache an in-flight scan after clearScanCache invalidates it', async () => { const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), 'scanner-dedup-clear-')); tempDirs.push(rootDir);