feat(discovery): bound recent project scans
This commit is contained in:
777genius
parent
ec004e9328
commit
3d1b329221
7 changed files with 1388 additions and 100 deletions
|
|
@ -4,6 +4,7 @@ import path from 'node:path';
|
|||
|
||||
import { resolveProjectFilesystemState } from '@features/recent-projects/main/infrastructure/filesystem/resolveProjectFilesystemState';
|
||||
import { normalizeIdentityPath } from '@features/recent-projects/main/infrastructure/identity/normalizeIdentityPath';
|
||||
import { getAppDataPath } from '@main/utils/pathDecoder';
|
||||
import { isEphemeralProjectPath } from '@shared/utils/ephemeralProjectPath';
|
||||
|
||||
import type { LoggerPort } from '@features/recent-projects/core/application/ports/LoggerPort';
|
||||
|
|
@ -18,12 +19,21 @@ import type { ServiceContext } from '@main/services';
|
|||
const CODEX_SESSION_FILE_PARSE_LIMIT = 500;
|
||||
const CODEX_PROJECT_CANDIDATE_LIMIT = 40;
|
||||
const CODEX_SESSION_FILE_SOURCE_TIMEOUT_MS = 8_000;
|
||||
const CODEX_SESSION_FILE_SOFT_BUDGET_MS = 6_500;
|
||||
const CODEX_SESSION_FILE_MAX_UNCACHED_READS_PER_RUN = 160;
|
||||
const CODEX_SESSION_FILE_READ_BATCH_SIZE = 24;
|
||||
const CODEX_SESSION_FILE_READ_TIMEOUT_MS = 700;
|
||||
const CODEX_SESSION_METADATA_READ_LIMIT_BYTES = 128 * 1024;
|
||||
const CODEX_SESSION_FILE_CACHE_SCHEMA_VERSION = 1;
|
||||
const CODEX_SESSION_FILE_CACHE_RELATIVE_PATH = path.join(
|
||||
'recent-projects',
|
||||
'codex-session-files-index.json'
|
||||
);
|
||||
|
||||
interface CodexSessionFileEntry {
|
||||
filePath: string;
|
||||
mtimeMs: number;
|
||||
size: number;
|
||||
}
|
||||
|
||||
interface CodexSessionEvent {
|
||||
|
|
@ -53,6 +63,50 @@ interface CodexSessionMetadata {
|
|||
branchName?: string;
|
||||
}
|
||||
|
||||
interface CodexSessionFileCacheEntry {
|
||||
filePath: string;
|
||||
mtimeMs: number;
|
||||
size: number;
|
||||
snapshot: CodexSessionProjectSnapshot | null;
|
||||
}
|
||||
|
||||
interface CodexSessionFileCacheFile {
|
||||
schemaVersion: number;
|
||||
entries: Record<string, CodexSessionFileCacheEntry>;
|
||||
}
|
||||
|
||||
interface CodexSessionSnapshotLoadResult {
|
||||
snapshots: CodexSessionProjectSnapshot[];
|
||||
degraded: boolean;
|
||||
stats: {
|
||||
files: number;
|
||||
cached: number;
|
||||
uncachedReads: number;
|
||||
timedOutReads: number;
|
||||
skippedUncached: number;
|
||||
durationMs: number;
|
||||
};
|
||||
}
|
||||
|
||||
function emptyCache(): CodexSessionFileCacheFile {
|
||||
return {
|
||||
schemaVersion: CODEX_SESSION_FILE_CACHE_SCHEMA_VERSION,
|
||||
entries: {},
|
||||
};
|
||||
}
|
||||
|
||||
function isUsableCacheEntry(
|
||||
entry: CodexSessionFileCacheEntry | undefined,
|
||||
file: CodexSessionFileEntry
|
||||
): entry is CodexSessionFileCacheEntry {
|
||||
return (
|
||||
!!entry &&
|
||||
entry.filePath === file.filePath &&
|
||||
entry.mtimeMs === file.mtimeMs &&
|
||||
entry.size === file.size
|
||||
);
|
||||
}
|
||||
|
||||
function isInteractiveSource(source: unknown): boolean {
|
||||
return source === 'vscode' || source === 'cli';
|
||||
}
|
||||
|
|
@ -118,6 +172,45 @@ async function readFirstLine(filePath: string): Promise<string | null> {
|
|||
}
|
||||
}
|
||||
|
||||
async function readFirstLineWithTimeout(
|
||||
filePath: string,
|
||||
timeoutMs: number
|
||||
): Promise<{ firstLine: string | null; timedOut: boolean }> {
|
||||
if (timeoutMs <= 0) {
|
||||
return {
|
||||
firstLine: null,
|
||||
timedOut: true,
|
||||
};
|
||||
}
|
||||
|
||||
let timer: ReturnType<typeof setTimeout> | null = null;
|
||||
const readPromise = readFirstLine(filePath)
|
||||
.then((firstLine) => ({
|
||||
firstLine,
|
||||
timedOut: false,
|
||||
}))
|
||||
.catch(() => ({
|
||||
firstLine: null,
|
||||
timedOut: false,
|
||||
}));
|
||||
const timeoutPromise = new Promise<{ firstLine: null; timedOut: true }>((resolve) => {
|
||||
timer = setTimeout(
|
||||
() =>
|
||||
resolve({
|
||||
firstLine: null,
|
||||
timedOut: true,
|
||||
}),
|
||||
timeoutMs
|
||||
);
|
||||
});
|
||||
|
||||
const result = await Promise.race([readPromise, timeoutPromise]);
|
||||
if (timer) {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
async function listJsonlFiles(root: string, maxDepth: number): Promise<CodexSessionFileEntry[]> {
|
||||
async function walk(directory: string, depth: number): Promise<CodexSessionFileEntry[]> {
|
||||
let entries;
|
||||
|
|
@ -144,6 +237,7 @@ async function listJsonlFiles(root: string, maxDepth: number): Promise<CodexSess
|
|||
{
|
||||
filePath: entryPath,
|
||||
mtimeMs: stats.mtimeMs,
|
||||
size: stats.size,
|
||||
},
|
||||
];
|
||||
} catch {
|
||||
|
|
@ -199,6 +293,7 @@ export class CodexSessionFileRecentProjectsSourceAdapter implements RecentProjec
|
|||
readonly sourceId = 'codex-session-files';
|
||||
readonly timeoutMs = CODEX_SESSION_FILE_SOURCE_TIMEOUT_MS;
|
||||
readonly #codexHome: string;
|
||||
readonly #cachePath: string;
|
||||
|
||||
constructor(
|
||||
private readonly deps: {
|
||||
|
|
@ -207,9 +302,14 @@ export class CodexSessionFileRecentProjectsSourceAdapter implements RecentProjec
|
|||
identityResolver: RecentProjectIdentityResolver;
|
||||
logger: LoggerPort;
|
||||
codexHome?: string;
|
||||
appDataPath?: string;
|
||||
}
|
||||
) {
|
||||
this.#codexHome = getCodexHome(deps.codexHome);
|
||||
this.#cachePath = path.join(
|
||||
deps.appDataPath ?? getAppDataPath(),
|
||||
CODEX_SESSION_FILE_CACHE_RELATIVE_PATH
|
||||
);
|
||||
}
|
||||
|
||||
async list(): Promise<RecentProjectsSourceResult> {
|
||||
|
|
@ -224,9 +324,11 @@ export class CodexSessionFileRecentProjectsSourceAdapter implements RecentProjec
|
|||
}
|
||||
|
||||
try {
|
||||
const snapshots = await this.#listRecentSessionSnapshots();
|
||||
const snapshotResult = await this.#listRecentSessionSnapshots();
|
||||
const candidates = await Promise.all(
|
||||
snapshots.map((snapshot) => this.#toCandidate(snapshot, activeContext.fsProvider))
|
||||
snapshotResult.snapshots.map((snapshot) =>
|
||||
this.#toCandidate(snapshot, activeContext.fsProvider)
|
||||
)
|
||||
);
|
||||
|
||||
const validCandidates = candidates.filter(
|
||||
|
|
@ -236,11 +338,13 @@ export class CodexSessionFileRecentProjectsSourceAdapter implements RecentProjec
|
|||
this.deps.logger.info('codex session-file recent-projects source loaded', {
|
||||
count: validCandidates.length,
|
||||
codexHome: this.#codexHome,
|
||||
degraded: snapshotResult.degraded,
|
||||
...snapshotResult.stats,
|
||||
});
|
||||
|
||||
return {
|
||||
candidates: validCandidates,
|
||||
degraded: false,
|
||||
degraded: snapshotResult.degraded,
|
||||
};
|
||||
} catch (error) {
|
||||
this.deps.logger.warn('codex session-file recent-projects source failed', {
|
||||
|
|
@ -254,15 +358,23 @@ export class CodexSessionFileRecentProjectsSourceAdapter implements RecentProjec
|
|||
}
|
||||
}
|
||||
|
||||
async #listRecentSessionSnapshots(): Promise<CodexSessionProjectSnapshot[]> {
|
||||
async #listRecentSessionSnapshots(): Promise<CodexSessionSnapshotLoadResult> {
|
||||
const startedAt = Date.now();
|
||||
const deadline = startedAt + CODEX_SESSION_FILE_SOFT_BUDGET_MS;
|
||||
const files = [
|
||||
...(await listJsonlFiles(path.join(this.#codexHome, 'sessions'), 4)),
|
||||
...(await listJsonlFiles(path.join(this.#codexHome, 'archived_sessions'), 1)),
|
||||
].sort((left, right) => right.mtimeMs - left.mtimeMs);
|
||||
|
||||
const snapshotsByCwd = new Map<string, CodexSessionProjectSnapshot>();
|
||||
|
||||
const candidateFiles = files.slice(0, CODEX_SESSION_FILE_PARSE_LIMIT);
|
||||
const cache = await this.#readCacheSafe();
|
||||
const nextCacheEntries = new Map<string, CodexSessionFileCacheEntry>();
|
||||
let degraded = false;
|
||||
let cached = 0;
|
||||
let uncachedReads = 0;
|
||||
let timedOutReads = 0;
|
||||
let skippedUncached = 0;
|
||||
|
||||
for (
|
||||
let offset = 0;
|
||||
|
|
@ -270,20 +382,49 @@ export class CodexSessionFileRecentProjectsSourceAdapter implements RecentProjec
|
|||
offset += CODEX_SESSION_FILE_READ_BATCH_SIZE
|
||||
) {
|
||||
const batch = candidateFiles.slice(offset, offset + CODEX_SESSION_FILE_READ_BATCH_SIZE);
|
||||
const firstLines = await Promise.all(
|
||||
batch.map(async (file) => ({
|
||||
file,
|
||||
firstLine: await readFirstLine(file.filePath),
|
||||
}))
|
||||
const metadata = await Promise.all(
|
||||
batch.map(async (file) => {
|
||||
const cachedEntry = cache.entries[file.filePath];
|
||||
if (isUsableCacheEntry(cachedEntry, file)) {
|
||||
cached += 1;
|
||||
nextCacheEntries.set(file.filePath, cachedEntry);
|
||||
return { snapshot: cachedEntry.snapshot, processed: true };
|
||||
}
|
||||
|
||||
if (
|
||||
Date.now() >= deadline ||
|
||||
uncachedReads >= CODEX_SESSION_FILE_MAX_UNCACHED_READS_PER_RUN
|
||||
) {
|
||||
degraded = true;
|
||||
skippedUncached += 1;
|
||||
return { snapshot: null, processed: false };
|
||||
}
|
||||
|
||||
uncachedReads += 1;
|
||||
const readResult = await readFirstLineWithTimeout(
|
||||
file.filePath,
|
||||
Math.min(CODEX_SESSION_FILE_READ_TIMEOUT_MS, deadline - Date.now())
|
||||
);
|
||||
if (readResult.timedOut) {
|
||||
degraded = true;
|
||||
timedOutReads += 1;
|
||||
return { snapshot: null, processed: false };
|
||||
}
|
||||
|
||||
const firstLine = readResult.firstLine;
|
||||
const snapshot = firstLine ? parseSessionSnapshot(firstLine, file.mtimeMs) : null;
|
||||
nextCacheEntries.set(file.filePath, {
|
||||
filePath: file.filePath,
|
||||
mtimeMs: file.mtimeMs,
|
||||
size: file.size,
|
||||
snapshot,
|
||||
});
|
||||
return { snapshot, processed: true };
|
||||
})
|
||||
);
|
||||
|
||||
for (const { file, firstLine } of firstLines) {
|
||||
if (!firstLine) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const snapshot = parseSessionSnapshot(firstLine, file.mtimeMs);
|
||||
if (!snapshot) {
|
||||
for (const { snapshot, processed } of metadata) {
|
||||
if (!processed || !snapshot) {
|
||||
continue;
|
||||
}
|
||||
|
||||
|
|
@ -298,9 +439,82 @@ export class CodexSessionFileRecentProjectsSourceAdapter implements RecentProjec
|
|||
}
|
||||
}
|
||||
|
||||
return Array.from(snapshotsByCwd.values())
|
||||
for (const file of candidateFiles) {
|
||||
const cachedEntry = cache.entries[file.filePath];
|
||||
if (isUsableCacheEntry(cachedEntry, file) && !nextCacheEntries.has(file.filePath)) {
|
||||
nextCacheEntries.set(file.filePath, cachedEntry);
|
||||
}
|
||||
}
|
||||
await this.#writeCacheSafe(nextCacheEntries);
|
||||
|
||||
const snapshots = Array.from(snapshotsByCwd.values())
|
||||
.sort((left, right) => right.lastActivityAt - left.lastActivityAt)
|
||||
.slice(0, CODEX_PROJECT_CANDIDATE_LIMIT);
|
||||
const durationMs = Date.now() - startedAt;
|
||||
if (degraded) {
|
||||
this.deps.logger.warn('codex session-file recent-projects source partial', {
|
||||
files: candidateFiles.length,
|
||||
cached,
|
||||
uncachedReads,
|
||||
timedOutReads,
|
||||
skippedUncached,
|
||||
candidates: snapshots.length,
|
||||
durationMs,
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
snapshots,
|
||||
degraded,
|
||||
stats: {
|
||||
files: candidateFiles.length,
|
||||
cached,
|
||||
uncachedReads,
|
||||
timedOutReads,
|
||||
skippedUncached,
|
||||
durationMs,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async #readCacheSafe(): Promise<CodexSessionFileCacheFile> {
|
||||
try {
|
||||
const raw = await fs.readFile(this.#cachePath, 'utf8');
|
||||
const parsed = JSON.parse(raw) as Partial<CodexSessionFileCacheFile>;
|
||||
if (
|
||||
parsed.schemaVersion !== CODEX_SESSION_FILE_CACHE_SCHEMA_VERSION ||
|
||||
!parsed.entries ||
|
||||
typeof parsed.entries !== 'object' ||
|
||||
Array.isArray(parsed.entries)
|
||||
) {
|
||||
return emptyCache();
|
||||
}
|
||||
return {
|
||||
schemaVersion: CODEX_SESSION_FILE_CACHE_SCHEMA_VERSION,
|
||||
entries: parsed.entries,
|
||||
};
|
||||
} catch {
|
||||
return emptyCache();
|
||||
}
|
||||
}
|
||||
|
||||
async #writeCacheSafe(entries: ReadonlyMap<string, CodexSessionFileCacheEntry>): Promise<void> {
|
||||
let tempPath: string | null = null;
|
||||
try {
|
||||
await fs.mkdir(path.dirname(this.#cachePath), { recursive: true });
|
||||
const cacheFile: CodexSessionFileCacheFile = {
|
||||
schemaVersion: CODEX_SESSION_FILE_CACHE_SCHEMA_VERSION,
|
||||
entries: Object.fromEntries(entries),
|
||||
};
|
||||
tempPath = `${this.#cachePath}.${process.pid}.${Date.now()}.tmp`;
|
||||
await fs.writeFile(tempPath, JSON.stringify(cacheFile), 'utf8');
|
||||
await fs.rename(tempPath, this.#cachePath);
|
||||
} catch {
|
||||
if (tempPath) {
|
||||
await fs.rm(tempPath, { force: true }).catch(() => undefined);
|
||||
}
|
||||
// Cache is an optimization only; never fail recent projects because it is unavailable.
|
||||
}
|
||||
}
|
||||
|
||||
async #toCandidate(
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ import {
|
|||
} from '@main/types';
|
||||
import {
|
||||
analyzeSessionFileMetadata,
|
||||
extractCwd,
|
||||
extractCwdFromKnownJsonlFile,
|
||||
type SessionFileMetadata,
|
||||
} from '@main/utils/jsonl';
|
||||
import {
|
||||
|
|
@ -106,6 +106,17 @@ export interface ProjectScannerOptions {
|
|||
sessionIndexDir?: string;
|
||||
/** Test hook: set to 0 to persist index files without debounce. */
|
||||
sessionIndexPersistDelayMs?: number;
|
||||
/**
|
||||
* Bounds local stat/read work during project discovery.
|
||||
* This keeps startup scans from saturating libuv while other services initialize.
|
||||
*/
|
||||
scanFileIoConcurrency?: number;
|
||||
/**
|
||||
* Upper bound for a full project-directory scan. If exhausted, the previous
|
||||
* complete scan is reused when available; otherwise completed projects are
|
||||
* returned without caching the partial result.
|
||||
*/
|
||||
scanBudgetMs?: number;
|
||||
}
|
||||
|
||||
interface ScanInFlight {
|
||||
|
|
@ -113,6 +124,28 @@ interface ScanInFlight {
|
|||
promise: Promise<Project[]>;
|
||||
}
|
||||
|
||||
interface RepositoryGroupScanInFlight {
|
||||
generation: number;
|
||||
promise: Promise<RepositoryGroup[]>;
|
||||
}
|
||||
|
||||
interface ProjectScanWithTimeoutResult {
|
||||
projects: Project[];
|
||||
timedOut: boolean;
|
||||
}
|
||||
|
||||
interface ScanProjectOptions {
|
||||
shouldCommitSubprojects?: () => boolean;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
class ScanProjectAbortedError extends Error {
|
||||
constructor() {
|
||||
super('Project scan aborted');
|
||||
this.name = 'ScanProjectAbortedError';
|
||||
}
|
||||
}
|
||||
|
||||
function splitPathSegments(value: string): string[] {
|
||||
return value.split(/[/\\]+/).filter(Boolean);
|
||||
}
|
||||
|
|
@ -190,8 +223,11 @@ export class ProjectScanner {
|
|||
// Both getProjects() and getRepositoryGroups() call scan() — the cache deduplicates.
|
||||
private scanCache: { projects: Project[]; timestamp: number } | null = null;
|
||||
private scanInFlight: ScanInFlight | null = null;
|
||||
private repositoryGroupInFlight: RepositoryGroupScanInFlight | null = null;
|
||||
private scanGeneration = 0;
|
||||
private static readonly SCAN_CACHE_TTL_MS = 2000;
|
||||
private static readonly SCAN_BUDGET_MS = 24_000;
|
||||
private static readonly MIN_SCAN_BATCH_BUDGET_MS = 1000;
|
||||
|
||||
/** Cached project list for search — avoids re-scanning disk on every query */
|
||||
private searchProjectCache: { projects: Project[]; timestamp: number } | null = null;
|
||||
|
|
@ -199,6 +235,9 @@ export class ProjectScanner {
|
|||
// Platform-aware batch sizes to avoid UV thread pool saturation on Windows
|
||||
private static readonly LOCAL_SESSION_BATCH = process.platform === 'win32' ? 16 : 64;
|
||||
private static readonly LOCAL_PROJECT_BATCH = process.platform === 'win32' ? 4 : 12;
|
||||
private static readonly LOCAL_SCAN_FILE_IO_CONCURRENCY = process.platform === 'win32' ? 8 : 16;
|
||||
private static readonly EXHAUSTIVE_CWD_SPLIT_FILE_LIMIT = 12;
|
||||
private static readonly CWD_HINT_SAMPLE_FILE_LIMIT = 1;
|
||||
|
||||
// Delegated services
|
||||
private readonly fsProvider: FileSystemProvider;
|
||||
|
|
@ -207,6 +246,10 @@ export class ProjectScanner {
|
|||
private readonly sessionSearcher: SessionSearcher;
|
||||
private readonly projectPathResolver: ProjectPathResolver;
|
||||
private readonly sessionMetadataIndex: SessionMetadataIndex | null;
|
||||
private readonly scanFileIoConcurrency: number;
|
||||
private readonly scanBudgetMs: number;
|
||||
private scanFileIoActive = 0;
|
||||
private readonly scanFileIoQueue: Array<() => void> = [];
|
||||
|
||||
constructor(
|
||||
projectsDir?: string,
|
||||
|
|
@ -230,6 +273,17 @@ export class ProjectScanner {
|
|||
persistDelayMs: options?.sessionIndexPersistDelayMs,
|
||||
})
|
||||
: null;
|
||||
const configuredScanFileIoConcurrency = options?.scanFileIoConcurrency;
|
||||
this.scanFileIoConcurrency =
|
||||
typeof configuredScanFileIoConcurrency === 'number' &&
|
||||
Number.isFinite(configuredScanFileIoConcurrency)
|
||||
? Math.max(1, Math.floor(configuredScanFileIoConcurrency))
|
||||
: ProjectScanner.LOCAL_SCAN_FILE_IO_CONCURRENCY;
|
||||
const configuredScanBudgetMs = options?.scanBudgetMs;
|
||||
this.scanBudgetMs =
|
||||
typeof configuredScanBudgetMs === 'number' && Number.isFinite(configuredScanBudgetMs)
|
||||
? Math.max(1, Math.floor(configuredScanBudgetMs))
|
||||
: ProjectScanner.SCAN_BUDGET_MS;
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
|
|
@ -277,6 +331,7 @@ export class ProjectScanner {
|
|||
private async performScan(generation: number): Promise<Project[]> {
|
||||
const startedAt = Date.now();
|
||||
let stage = 'start';
|
||||
let previousSubprojectRegistry: ReturnType<typeof subprojectRegistry.snapshot> | null = null;
|
||||
const slowWarnAfterMs = 10_000;
|
||||
const slowWarnTimer = setTimeout(() => {
|
||||
logger.warn(
|
||||
|
|
@ -291,6 +346,7 @@ export class ProjectScanner {
|
|||
}
|
||||
|
||||
// Clear the subproject registry on full re-scan
|
||||
previousSubprojectRegistry = subprojectRegistry.snapshot();
|
||||
subprojectRegistry.clear();
|
||||
|
||||
stage = 'readdirProjectsDir';
|
||||
|
|
@ -308,10 +364,9 @@ export class ProjectScanner {
|
|||
|
||||
// Process each project directory (may return multiple projects per dir)
|
||||
stage = 'scanProjects';
|
||||
const projectArrays = await this.collectFulfilledInBatches(
|
||||
const { projectArrays, completedAll, slowest } = await this.scanProjectDirsWithBudget(
|
||||
projectDirs,
|
||||
this.fsProvider.type === 'ssh' ? 8 : ProjectScanner.LOCAL_PROJECT_BATCH,
|
||||
async (dir) => this.scanProjectWithTimeout(dir.name)
|
||||
startedAt
|
||||
);
|
||||
|
||||
// Flatten and sort by most recent
|
||||
|
|
@ -327,14 +382,28 @@ export class ProjectScanner {
|
|||
const ms = Date.now() - startedAt;
|
||||
if (ms >= 5000) {
|
||||
logger.warn(
|
||||
`[scan] completed slow ms=${ms} projectDirs=${projectDirs.length} projects=${validProjects.length}`
|
||||
`[scan] completed slow ms=${ms} projectDirs=${projectDirs.length} projects=${validProjects.length} slowest=${JSON.stringify(
|
||||
slowest.slice(0, 5)
|
||||
)}`
|
||||
);
|
||||
}
|
||||
if (this.scanGeneration === generation) {
|
||||
if (!completedAll && this.scanCache) {
|
||||
if (previousSubprojectRegistry) {
|
||||
subprojectRegistry.restore(previousSubprojectRegistry);
|
||||
}
|
||||
logger.warn(
|
||||
`[scan] returning cached complete result after budget exhaustion projects=${this.scanCache.projects.length}`
|
||||
);
|
||||
return this.scanCache.projects;
|
||||
}
|
||||
if (completedAll && this.scanGeneration === generation) {
|
||||
this.scanCache = { projects: validProjects, timestamp: Date.now() };
|
||||
}
|
||||
return validProjects;
|
||||
} catch (error) {
|
||||
if (previousSubprojectRegistry && this.scanCache) {
|
||||
subprojectRegistry.restore(previousSubprojectRegistry);
|
||||
}
|
||||
logger.error('Error scanning projects directory:', error);
|
||||
return [];
|
||||
} finally {
|
||||
|
|
@ -374,6 +443,31 @@ export class ProjectScanner {
|
|||
* @returns Promise resolving to RepositoryGroups sorted by most recent activity
|
||||
*/
|
||||
async scanWithWorktreeGrouping(): Promise<RepositoryGroup[]> {
|
||||
const generation = this.scanGeneration;
|
||||
const inFlight = this.repositoryGroupInFlight;
|
||||
if (inFlight) {
|
||||
if (inFlight.generation === generation) {
|
||||
return inFlight.promise;
|
||||
}
|
||||
await inFlight.promise.catch(() => undefined);
|
||||
if (this.repositoryGroupInFlight?.promise === inFlight.promise) {
|
||||
this.repositoryGroupInFlight = null;
|
||||
}
|
||||
return this.scanWithWorktreeGrouping();
|
||||
}
|
||||
|
||||
const promise = this.performScanWithWorktreeGrouping();
|
||||
this.repositoryGroupInFlight = { generation, promise };
|
||||
try {
|
||||
return await promise;
|
||||
} finally {
|
||||
if (this.repositoryGroupInFlight?.promise === promise) {
|
||||
this.repositoryGroupInFlight = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async performScanWithWorktreeGrouping(): Promise<RepositoryGroup[]> {
|
||||
try {
|
||||
// 1. Scan all projects using existing logic
|
||||
const projects = await this.scan();
|
||||
|
|
@ -474,34 +568,142 @@ export class ProjectScanner {
|
|||
|
||||
/**
|
||||
* Scans a single project directory with a timeout guard.
|
||||
* Returns empty array if the scan exceeds the timeout.
|
||||
* Timeout is reported separately from an actually empty project directory so
|
||||
* partial scans cannot be mistaken for complete results.
|
||||
*/
|
||||
private async scanProjectWithTimeout(encodedName: string): Promise<Project[]> {
|
||||
private async scanProjectWithTimeout(
|
||||
encodedName: string,
|
||||
timeoutMs = ProjectScanner.SCAN_PROJECT_TIMEOUT_MS,
|
||||
options: { logTimeout?: boolean } = {}
|
||||
): Promise<ProjectScanWithTimeoutResult> {
|
||||
const abortController = new AbortController();
|
||||
let timer: ReturnType<typeof setTimeout> | null = null;
|
||||
const timeout = new Promise<Project[]>((resolve) => {
|
||||
let timedOut = false;
|
||||
const effectiveTimeoutMs = Math.max(1, Math.floor(timeoutMs));
|
||||
const timeout = new Promise<ProjectScanWithTimeoutResult>((resolve) => {
|
||||
timer = setTimeout(() => {
|
||||
logger.warn(
|
||||
`[scanProject] timeout after ${ProjectScanner.SCAN_PROJECT_TIMEOUT_MS}ms project=${encodedName}`
|
||||
);
|
||||
resolve([]);
|
||||
}, ProjectScanner.SCAN_PROJECT_TIMEOUT_MS);
|
||||
timedOut = true;
|
||||
abortController.abort();
|
||||
if (options.logTimeout !== false) {
|
||||
logger.warn(`[scanProject] timeout after ${effectiveTimeoutMs}ms project=${encodedName}`);
|
||||
}
|
||||
resolve({ projects: [], timedOut: true });
|
||||
}, effectiveTimeoutMs);
|
||||
});
|
||||
try {
|
||||
return await Promise.race([this.scanProject(encodedName), timeout]);
|
||||
return await Promise.race([
|
||||
this.scanProject(encodedName, {
|
||||
shouldCommitSubprojects: () => !timedOut,
|
||||
signal: abortController.signal,
|
||||
}).then((projects) => ({ projects, timedOut })),
|
||||
timeout,
|
||||
]);
|
||||
} finally {
|
||||
if (timer) clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
private async scanProjectDirsWithBudget(
|
||||
projectDirs: FsDirent[],
|
||||
scanStartedAt: number
|
||||
): Promise<{
|
||||
projectArrays: Project[][];
|
||||
completedAll: boolean;
|
||||
slowest: Array<{ project: string; ms: number; returned: number }>;
|
||||
}> {
|
||||
const workerCount = Math.max(
|
||||
1,
|
||||
Math.min(
|
||||
this.fsProvider.type === 'ssh' ? 8 : ProjectScanner.LOCAL_PROJECT_BATCH,
|
||||
projectDirs.length
|
||||
)
|
||||
);
|
||||
const projectArrays: Project[][] = [];
|
||||
const slowest: Array<{ project: string; ms: number; returned: number }> = [];
|
||||
let nextIndex = 0;
|
||||
let startedDirs = 0;
|
||||
let completedAll = true;
|
||||
|
||||
const claimNextDir = (): { dir: FsDirent; remainingBudgetMs: number } | null => {
|
||||
if (nextIndex >= projectDirs.length) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const elapsedMs = Date.now() - scanStartedAt;
|
||||
const remainingBudgetMs = this.scanBudgetMs - elapsedMs;
|
||||
if (remainingBudgetMs < ProjectScanner.MIN_SCAN_BATCH_BUDGET_MS) {
|
||||
completedAll = false;
|
||||
return null;
|
||||
}
|
||||
|
||||
const dir = projectDirs[nextIndex];
|
||||
nextIndex += 1;
|
||||
startedDirs += 1;
|
||||
return { dir, remainingBudgetMs };
|
||||
};
|
||||
|
||||
const workers = new Array(workerCount).fill(0).map(async () => {
|
||||
while (true) {
|
||||
const claimed = claimNextDir();
|
||||
if (!claimed) {
|
||||
return;
|
||||
}
|
||||
const { dir, remainingBudgetMs } = claimed;
|
||||
const perProjectTimeoutMs = Math.min(
|
||||
ProjectScanner.SCAN_PROJECT_TIMEOUT_MS,
|
||||
remainingBudgetMs
|
||||
);
|
||||
try {
|
||||
const projectStartedAt = Date.now();
|
||||
const result = await this.scanProjectWithTimeout(dir.name, perProjectTimeoutMs, {
|
||||
logTimeout: false,
|
||||
});
|
||||
const ms = Date.now() - projectStartedAt;
|
||||
if (ms >= 500) {
|
||||
slowest.push({ project: dir.name, ms, returned: result.projects.length });
|
||||
}
|
||||
if (result.timedOut) {
|
||||
completedAll = false;
|
||||
}
|
||||
projectArrays.push(result.projects);
|
||||
} catch {
|
||||
completedAll = false;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
await Promise.all(workers);
|
||||
|
||||
if (startedDirs < projectDirs.length) {
|
||||
completedAll = false;
|
||||
}
|
||||
|
||||
slowest.sort((a, b) => b.ms - a.ms);
|
||||
if (!completedAll) {
|
||||
logger.warn(
|
||||
`[scan] budget exhausted ms=${Date.now() - scanStartedAt} scannedDirs=${startedDirs}/${projectDirs.length} returnedProjectArrays=${projectArrays.length} slowest=${JSON.stringify(
|
||||
slowest.slice(0, 5)
|
||||
)}`
|
||||
);
|
||||
}
|
||||
|
||||
return { projectArrays, completedAll, slowest };
|
||||
}
|
||||
|
||||
/**
|
||||
* Scans a single project directory and returns project metadata.
|
||||
* If sessions have different cwd values, splits into multiple projects.
|
||||
*/
|
||||
private async scanProject(encodedName: string): Promise<Project[]> {
|
||||
private async scanProject(
|
||||
encodedName: string,
|
||||
options: ScanProjectOptions = {}
|
||||
): Promise<Project[]> {
|
||||
try {
|
||||
this.throwIfScanAborted(options.signal);
|
||||
const projectPath = path.join(this.projectsDir, encodedName);
|
||||
const readdirStart = Date.now();
|
||||
const entries = await this.readdirForProjectDiscovery(projectPath);
|
||||
this.throwIfScanAborted(options.signal);
|
||||
const readdirMs = Date.now() - readdirStart;
|
||||
|
||||
// Get session files (.jsonl at root level)
|
||||
|
|
@ -528,46 +730,63 @@ export class ProjectScanner {
|
|||
cwd: string | null;
|
||||
}
|
||||
|
||||
// Reading JSONL heads for cwd across hundreds/thousands of sessions can saturate I/O and
|
||||
// make the renderer appear frozen while waiting for repository groups.
|
||||
// Prefer correctness for small projects; for large ones, skip cwd splitting and fall back
|
||||
// to encoded-path decoding / limited path probing.
|
||||
const MAX_CWD_SPLIT_FILES = 80;
|
||||
// Reading JSONL heads for cwd can dominate startup when cold I/O is already contended.
|
||||
// Keep cwd splitting exact for small project dirs. For larger dirs, read only a newest-session
|
||||
// cwd hint so the project path stays accurate without building a partial subproject split.
|
||||
const shouldSplitByCwd =
|
||||
this.fsProvider.type !== 'ssh' && sessionFiles.length <= MAX_CWD_SPLIT_FILES;
|
||||
this.fsProvider.type !== 'ssh' &&
|
||||
sessionFiles.length <= ProjectScanner.EXHAUSTIVE_CWD_SPLIT_FILE_LIMIT;
|
||||
|
||||
const sessionStatStart = Date.now();
|
||||
const sessionInfos = await this.collectFulfilledInBatches(
|
||||
sessionFiles,
|
||||
this.fsProvider.type === 'ssh' ? 32 : ProjectScanner.LOCAL_SESSION_BATCH,
|
||||
async (file) => {
|
||||
this.throwIfScanAborted(options.signal);
|
||||
const filePath = path.join(projectPath, file.name);
|
||||
const { mtimeMs, birthtimeMs } = await this.resolveFileDetails(file, filePath);
|
||||
let cwd: string | null = null;
|
||||
|
||||
// Over SSH, avoid reading every file body during project discovery.
|
||||
if (shouldSplitByCwd) {
|
||||
try {
|
||||
cwd = await extractCwd(filePath, this.fsProvider);
|
||||
} catch {
|
||||
// Ignore unreadable files
|
||||
}
|
||||
}
|
||||
const { mtimeMs, birthtimeMs } = await this.runScanFileIo(
|
||||
() => this.resolveFileDetails(file, filePath),
|
||||
options.signal
|
||||
);
|
||||
|
||||
return {
|
||||
sessionId: extractSessionId(file.name),
|
||||
filePath,
|
||||
mtimeMs,
|
||||
birthtimeMs,
|
||||
cwd,
|
||||
cwd: null as string | null,
|
||||
} satisfies SessionInfo;
|
||||
}
|
||||
);
|
||||
this.throwIfScanAborted(options.signal);
|
||||
|
||||
if (sessionInfos.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
if (this.fsProvider.type !== 'ssh') {
|
||||
const cwdCandidates = shouldSplitByCwd
|
||||
? sessionInfos
|
||||
: [...sessionInfos]
|
||||
.sort((a, b) => b.mtimeMs - a.mtimeMs)
|
||||
.slice(0, ProjectScanner.CWD_HINT_SAMPLE_FILE_LIMIT);
|
||||
await this.collectFulfilledInBatches(cwdCandidates, 2, async (info) => {
|
||||
try {
|
||||
info.cwd = await this.runScanFileIo(
|
||||
() => extractCwdFromKnownJsonlFile(info.filePath, this.fsProvider),
|
||||
options.signal
|
||||
);
|
||||
} catch (error) {
|
||||
if (error instanceof ScanProjectAbortedError || options.signal?.aborted) {
|
||||
throw error;
|
||||
}
|
||||
// Ignore unreadable files
|
||||
}
|
||||
return null;
|
||||
});
|
||||
this.throwIfScanAborted(options.signal);
|
||||
}
|
||||
|
||||
const sessionStatMs = Date.now() - sessionStatStart;
|
||||
if (sessionFiles.length > 200 || sessionStatMs > 1000) {
|
||||
logger.debug(
|
||||
|
|
@ -607,11 +826,14 @@ export class ProjectScanner {
|
|||
}
|
||||
|
||||
const sessionPaths = sessionInfos.map((s) => s.filePath);
|
||||
this.throwIfScanAborted(options.signal);
|
||||
const actualPath = await this.projectPathResolver.resolveProjectPath(encodedName, {
|
||||
cwdHint: firstCwd ?? undefined,
|
||||
sessionPaths,
|
||||
});
|
||||
this.throwIfScanAborted(options.signal);
|
||||
const filesystemState = await resolveProjectFilesystemState(this.fsProvider, actualPath);
|
||||
this.throwIfScanAborted(options.signal);
|
||||
|
||||
// Derive name from resolved path — more reliable than decodePath for
|
||||
// paths containing dashes (e.g. "test-project" encodes lossily).
|
||||
|
|
@ -632,7 +854,11 @@ export class ProjectScanner {
|
|||
}
|
||||
|
||||
// Multiple unique cwds: split into subprojects
|
||||
const projects: Project[] = [];
|
||||
const pendingProjects: Array<{
|
||||
registryCwd: string;
|
||||
sessionIds: string[];
|
||||
project: Omit<Project, 'id'>;
|
||||
}> = [];
|
||||
|
||||
// Find the "root" cwd (shortest path, or the one matching the decoded name)
|
||||
const cwdKeys = [...cwdGroups.keys()].filter((k) => !k.startsWith('__decoded__'));
|
||||
|
|
@ -644,16 +870,11 @@ export class ProjectScanner {
|
|||
const rootName = path.basename(rootCwd) || baseName;
|
||||
|
||||
for (const [cwdKey, sessions] of cwdGroups) {
|
||||
this.throwIfScanAborted(options.signal);
|
||||
const isDecodedFallback = cwdKey.startsWith('__decoded__');
|
||||
const actualCwd = isDecodedFallback ? null : cwdKey;
|
||||
|
||||
// Register in subproject registry
|
||||
const sessionIds = sessions.map((s) => s.sessionId);
|
||||
const compositeId = subprojectRegistry.register(
|
||||
encodedName,
|
||||
actualCwd ?? decodedFallback,
|
||||
sessionIds
|
||||
);
|
||||
const exportedSessionIds = sessionIds.slice(0, MAX_SESSION_IDS_EXPORTED);
|
||||
|
||||
// Compute timestamps
|
||||
|
|
@ -677,24 +898,41 @@ export class ProjectScanner {
|
|||
const lastSegment = path.basename(actualCwd);
|
||||
displayName = `${rootName} (${lastSegment})`;
|
||||
}
|
||||
|
||||
projects.push({
|
||||
id: compositeId,
|
||||
path: actualCwd ?? decodedFallback,
|
||||
name: displayName,
|
||||
sessions: exportedSessionIds,
|
||||
totalSessions: sessionIds.length,
|
||||
createdAt: Math.floor(createdAt),
|
||||
mostRecentSession: mostRecentSession ? Math.floor(mostRecentSession) : undefined,
|
||||
filesystemState: await resolveProjectFilesystemState(
|
||||
this.fsProvider,
|
||||
actualCwd ?? decodedFallback
|
||||
),
|
||||
const filesystemState = await resolveProjectFilesystemState(
|
||||
this.fsProvider,
|
||||
actualCwd ?? decodedFallback
|
||||
);
|
||||
this.throwIfScanAborted(options.signal);
|
||||
if (options.shouldCommitSubprojects?.() === false) {
|
||||
return [];
|
||||
}
|
||||
pendingProjects.push({
|
||||
registryCwd: actualCwd ?? decodedFallback,
|
||||
sessionIds,
|
||||
project: {
|
||||
path: actualCwd ?? decodedFallback,
|
||||
name: displayName,
|
||||
sessions: exportedSessionIds,
|
||||
totalSessions: sessionIds.length,
|
||||
createdAt: Math.floor(createdAt),
|
||||
mostRecentSession: mostRecentSession ? Math.floor(mostRecentSession) : undefined,
|
||||
filesystemState,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return projects;
|
||||
if (options.shouldCommitSubprojects?.() === false) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return pendingProjects.map(({ registryCwd, sessionIds, project }) => ({
|
||||
id: subprojectRegistry.register(encodedName, registryCwd, sessionIds),
|
||||
...project,
|
||||
}));
|
||||
} catch (error) {
|
||||
if (error instanceof ScanProjectAbortedError || options.signal?.aborted) {
|
||||
return [];
|
||||
}
|
||||
logger.error(`Error scanning project ${encodedName}:`, error);
|
||||
return [];
|
||||
}
|
||||
|
|
@ -1676,6 +1914,79 @@ export class ProjectScanner {
|
|||
return results;
|
||||
}
|
||||
|
||||
private throwIfScanAborted(signal?: AbortSignal): void {
|
||||
if (signal?.aborted) {
|
||||
throw new ScanProjectAbortedError();
|
||||
}
|
||||
}
|
||||
|
||||
private async runScanFileIo<T>(operation: () => Promise<T>, signal?: AbortSignal): Promise<T> {
|
||||
this.throwIfScanAborted(signal);
|
||||
if (this.fsProvider.type !== 'local') {
|
||||
const result = await operation();
|
||||
this.throwIfScanAborted(signal);
|
||||
return result;
|
||||
}
|
||||
|
||||
await this.acquireScanFileIoSlot(signal);
|
||||
try {
|
||||
this.throwIfScanAborted(signal);
|
||||
const result = await operation();
|
||||
this.throwIfScanAborted(signal);
|
||||
return result;
|
||||
} finally {
|
||||
this.releaseScanFileIoSlot();
|
||||
}
|
||||
}
|
||||
|
||||
private async acquireScanFileIoSlot(signal?: AbortSignal): Promise<void> {
|
||||
this.throwIfScanAborted(signal);
|
||||
if (this.scanFileIoActive < this.scanFileIoConcurrency) {
|
||||
this.scanFileIoActive += 1;
|
||||
return;
|
||||
}
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
let resume: (() => void) | null = null;
|
||||
const cleanup = () => {
|
||||
signal?.removeEventListener('abort', onAbort);
|
||||
};
|
||||
const onAbort = () => {
|
||||
const queuedResume = resume;
|
||||
if (!queuedResume) {
|
||||
return;
|
||||
}
|
||||
resume = null;
|
||||
const index = this.scanFileIoQueue.indexOf(queuedResume);
|
||||
if (index >= 0) {
|
||||
this.scanFileIoQueue.splice(index, 1);
|
||||
}
|
||||
cleanup();
|
||||
reject(new ScanProjectAbortedError());
|
||||
};
|
||||
resume = () => {
|
||||
if (!resume) {
|
||||
return;
|
||||
}
|
||||
resume = null;
|
||||
cleanup();
|
||||
resolve();
|
||||
};
|
||||
this.scanFileIoQueue.push(resume);
|
||||
signal?.addEventListener('abort', onAbort, { once: true });
|
||||
});
|
||||
this.throwIfScanAborted(signal);
|
||||
}
|
||||
|
||||
private releaseScanFileIoSlot(): void {
|
||||
const next = this.scanFileIoQueue.shift();
|
||||
if (next) {
|
||||
next();
|
||||
return;
|
||||
}
|
||||
this.scanFileIoActive = Math.max(0, this.scanFileIoActive - 1);
|
||||
}
|
||||
|
||||
private getErrorCode(error: unknown): string {
|
||||
if (typeof error === 'object' && error !== null && 'code' in error) {
|
||||
const code = (error as { code?: unknown }).code;
|
||||
|
|
|
|||
|
|
@ -19,6 +19,13 @@ interface SubprojectEntry {
|
|||
sessionIds: Set<string>;
|
||||
}
|
||||
|
||||
export interface SubprojectRegistrySnapshotEntry {
|
||||
id: string;
|
||||
baseDir: string;
|
||||
cwd: string;
|
||||
sessionIds: string[];
|
||||
}
|
||||
|
||||
class SubprojectRegistryImpl {
|
||||
private readonly entries = new Map<string, SubprojectEntry>();
|
||||
|
||||
|
|
@ -92,6 +99,33 @@ class SubprojectRegistryImpl {
|
|||
clear(): void {
|
||||
this.entries.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* Capture a deep copy of the current registry so an interrupted scan can
|
||||
* restore the previous complete project view.
|
||||
*/
|
||||
snapshot(): SubprojectRegistrySnapshotEntry[] {
|
||||
return [...this.entries.entries()].map(([id, entry]) => ({
|
||||
id,
|
||||
baseDir: entry.baseDir,
|
||||
cwd: entry.cwd,
|
||||
sessionIds: [...entry.sessionIds],
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace the registry contents with a previously captured snapshot.
|
||||
*/
|
||||
restore(snapshot: readonly SubprojectRegistrySnapshotEntry[]): void {
|
||||
this.entries.clear();
|
||||
for (const entry of snapshot) {
|
||||
this.entries.set(entry.id, {
|
||||
baseDir: entry.baseDir,
|
||||
cwd: entry.cwd,
|
||||
sessionIds: new Set(entry.sessionIds),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Module-level singleton */
|
||||
|
|
|
|||
|
|
@ -41,7 +41,11 @@ const logger = createLogger('Util:jsonl');
|
|||
const defaultProvider = new LocalFileSystemProvider();
|
||||
|
||||
// Re-export for backwards compatibility
|
||||
export { extractCwd, extractFirstUserMessagePreview } from './metadataExtraction';
|
||||
export {
|
||||
extractCwd,
|
||||
extractCwdFromKnownJsonlFile,
|
||||
extractFirstUserMessagePreview,
|
||||
} from './metadataExtraction';
|
||||
export { checkMessagesOngoing } from './sessionStateDetection';
|
||||
|
||||
// =============================================================================
|
||||
|
|
|
|||
|
|
@ -89,27 +89,10 @@ async function extractCwdFromLocalFile(filePath: string): Promise<string | null>
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract CWD (current working directory) from the first entry.
|
||||
* Used to get the actual project path from encoded directory names.
|
||||
*/
|
||||
export async function extractCwd(
|
||||
async function extractCwdFromReadableFile(
|
||||
filePath: string,
|
||||
fsProvider: FileSystemProvider = defaultProvider
|
||||
fsProvider: FileSystemProvider
|
||||
): Promise<string | null> {
|
||||
if (!(await fsProvider.exists(filePath))) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const stat = await fsProvider.stat(filePath);
|
||||
if (!stat.isFile()) {
|
||||
return null;
|
||||
}
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (fsProvider.type === 'local') {
|
||||
try {
|
||||
return await extractCwdFromLocalFile(filePath);
|
||||
|
|
@ -173,6 +156,41 @@ export async function extractCwd(
|
|||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract CWD from a JSONL file that the caller already discovered as a file.
|
||||
* This skips exists/stat preflight for hot project-discovery paths.
|
||||
*/
|
||||
export async function extractCwdFromKnownJsonlFile(
|
||||
filePath: string,
|
||||
fsProvider: FileSystemProvider = defaultProvider
|
||||
): Promise<string | null> {
|
||||
return extractCwdFromReadableFile(filePath, fsProvider);
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract CWD (current working directory) from the first entry.
|
||||
* Used to get the actual project path from encoded directory names.
|
||||
*/
|
||||
export async function extractCwd(
|
||||
filePath: string,
|
||||
fsProvider: FileSystemProvider = defaultProvider
|
||||
): Promise<string | null> {
|
||||
if (!(await fsProvider.exists(filePath))) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const stat = await fsProvider.stat(filePath);
|
||||
if (!stat.isFile()) {
|
||||
return null;
|
||||
}
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
return extractCwdFromReadableFile(filePath, fsProvider);
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract a lightweight title preview from the first user message.
|
||||
* For command-style sessions, falls back to a slash-command label.
|
||||
|
|
|
|||
|
|
@ -2,9 +2,8 @@ import fs from 'node:fs/promises';
|
|||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { CodexSessionFileRecentProjectsSourceAdapter } from '@features/recent-projects/main/adapters/output/sources/CodexSessionFileRecentProjectsSourceAdapter';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import type { LoggerPort } from '@features/recent-projects/core/application/ports/LoggerPort';
|
||||
import type { RecentProjectIdentityResolver } from '@features/recent-projects/main/infrastructure/identity/RecentProjectIdentityResolver';
|
||||
|
|
@ -21,6 +20,10 @@ function createLogger(): LoggerPort & {
|
|||
};
|
||||
}
|
||||
|
||||
function getSessionFileCachePath(appDataPath: string): string {
|
||||
return path.join(appDataPath, 'recent-projects', 'codex-session-files-index.json');
|
||||
}
|
||||
|
||||
async function writeRollout(
|
||||
filePath: string,
|
||||
payload: {
|
||||
|
|
@ -91,6 +94,7 @@ describe('CodexSessionFileRecentProjectsSourceAdapter', () => {
|
|||
identityResolver,
|
||||
logger,
|
||||
codexHome,
|
||||
appDataPath: path.join(tempDir, 'app-data'),
|
||||
});
|
||||
|
||||
await expect(adapter.list()).resolves.toEqual({
|
||||
|
|
@ -137,6 +141,7 @@ describe('CodexSessionFileRecentProjectsSourceAdapter', () => {
|
|||
identityResolver,
|
||||
logger,
|
||||
codexHome,
|
||||
appDataPath: path.join(tempDir, 'app-data'),
|
||||
});
|
||||
|
||||
const result = await adapter.list();
|
||||
|
|
@ -172,6 +177,7 @@ describe('CodexSessionFileRecentProjectsSourceAdapter', () => {
|
|||
identityResolver,
|
||||
logger,
|
||||
codexHome,
|
||||
appDataPath: path.join(tempDir, 'app-data'),
|
||||
});
|
||||
|
||||
const result = await adapter.list();
|
||||
|
|
@ -213,6 +219,7 @@ describe('CodexSessionFileRecentProjectsSourceAdapter', () => {
|
|||
identityResolver,
|
||||
logger,
|
||||
codexHome,
|
||||
appDataPath: path.join(tempDir, 'app-data'),
|
||||
});
|
||||
|
||||
const result = await adapter.list();
|
||||
|
|
@ -263,6 +270,7 @@ describe('CodexSessionFileRecentProjectsSourceAdapter', () => {
|
|||
identityResolver,
|
||||
logger,
|
||||
codexHome,
|
||||
appDataPath: path.join(tempDir, 'app-data'),
|
||||
});
|
||||
|
||||
const result = await adapter.list();
|
||||
|
|
@ -273,6 +281,300 @@ describe('CodexSessionFileRecentProjectsSourceAdapter', () => {
|
|||
]);
|
||||
});
|
||||
|
||||
it('reuses cached unchanged session metadata without reopening jsonl files', async () => {
|
||||
const codexHome = path.join(tempDir, '.codex');
|
||||
const appDataPath = path.join(tempDir, 'app-data');
|
||||
const logger = createLogger();
|
||||
const identityResolver = {
|
||||
resolve: vi.fn().mockResolvedValue(null),
|
||||
} as unknown as RecentProjectIdentityResolver;
|
||||
const updatedAt = new Date('2026-04-14T12:00:00.000Z');
|
||||
await writeRollout(
|
||||
path.join(codexHome, 'sessions', '2026', '04', '14', 'rollout-alpha.jsonl'),
|
||||
{
|
||||
cwd: '/Users/test/projects/alpha',
|
||||
branch: 'main',
|
||||
},
|
||||
updatedAt
|
||||
);
|
||||
|
||||
const adapter = new CodexSessionFileRecentProjectsSourceAdapter({
|
||||
getActiveContext: () => ({ type: 'local', id: 'local-1' }) as never,
|
||||
getLocalContext: () => ({ type: 'local', id: 'local-1' }) as never,
|
||||
identityResolver,
|
||||
logger,
|
||||
codexHome,
|
||||
appDataPath,
|
||||
});
|
||||
await expect(adapter.list()).resolves.toEqual({
|
||||
candidates: [
|
||||
expect.objectContaining({
|
||||
primaryPath: '/Users/test/projects/alpha',
|
||||
branchName: 'main',
|
||||
}),
|
||||
],
|
||||
degraded: false,
|
||||
});
|
||||
|
||||
const openSpy = vi.spyOn(fs, 'open');
|
||||
const cachedAdapter = new CodexSessionFileRecentProjectsSourceAdapter({
|
||||
getActiveContext: () => ({ type: 'local', id: 'local-1' }) as never,
|
||||
getLocalContext: () => ({ type: 'local', id: 'local-1' }) as never,
|
||||
identityResolver,
|
||||
logger,
|
||||
codexHome,
|
||||
appDataPath,
|
||||
});
|
||||
|
||||
await expect(cachedAdapter.list()).resolves.toEqual({
|
||||
candidates: [
|
||||
expect.objectContaining({
|
||||
primaryPath: '/Users/test/projects/alpha',
|
||||
branchName: 'main',
|
||||
}),
|
||||
],
|
||||
degraded: false,
|
||||
});
|
||||
expect(openSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('invalidates cached session metadata when the jsonl fingerprint changes', async () => {
|
||||
const codexHome = path.join(tempDir, '.codex');
|
||||
const appDataPath = path.join(tempDir, 'app-data');
|
||||
const logger = createLogger();
|
||||
const identityResolver = {
|
||||
resolve: vi.fn().mockResolvedValue(null),
|
||||
} as unknown as RecentProjectIdentityResolver;
|
||||
const sessionPath = path.join(
|
||||
codexHome,
|
||||
'sessions',
|
||||
'2026',
|
||||
'04',
|
||||
'14',
|
||||
'rollout-active.jsonl'
|
||||
);
|
||||
await writeRollout(
|
||||
sessionPath,
|
||||
{
|
||||
cwd: '/Users/test/projects/alpha',
|
||||
},
|
||||
new Date('2026-04-14T12:00:00.000Z')
|
||||
);
|
||||
|
||||
const adapter = new CodexSessionFileRecentProjectsSourceAdapter({
|
||||
getActiveContext: () => ({ type: 'local', id: 'local-1' }) as never,
|
||||
getLocalContext: () => ({ type: 'local', id: 'local-1' }) as never,
|
||||
identityResolver,
|
||||
logger,
|
||||
codexHome,
|
||||
appDataPath,
|
||||
});
|
||||
await expect(adapter.list()).resolves.toEqual({
|
||||
candidates: [
|
||||
expect.objectContaining({
|
||||
primaryPath: '/Users/test/projects/alpha',
|
||||
}),
|
||||
],
|
||||
degraded: false,
|
||||
});
|
||||
|
||||
await writeRollout(
|
||||
sessionPath,
|
||||
{
|
||||
cwd: '/Users/test/projects/beta',
|
||||
},
|
||||
new Date('2026-04-14T12:01:00.000Z')
|
||||
);
|
||||
|
||||
const refreshedAdapter = new CodexSessionFileRecentProjectsSourceAdapter({
|
||||
getActiveContext: () => ({ type: 'local', id: 'local-1' }) as never,
|
||||
getLocalContext: () => ({ type: 'local', id: 'local-1' }) as never,
|
||||
identityResolver,
|
||||
logger,
|
||||
codexHome,
|
||||
appDataPath,
|
||||
});
|
||||
|
||||
await expect(refreshedAdapter.list()).resolves.toEqual({
|
||||
candidates: [
|
||||
expect.objectContaining({
|
||||
primaryPath: '/Users/test/projects/beta',
|
||||
}),
|
||||
],
|
||||
degraded: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('does not let a slow jsonl read hold the whole source past its timeout budget', async () => {
|
||||
const codexHome = path.join(tempDir, '.codex');
|
||||
const appDataPath = path.join(tempDir, 'app-data');
|
||||
const logger = createLogger();
|
||||
const identityResolver = {
|
||||
resolve: vi.fn().mockResolvedValue(null),
|
||||
} as unknown as RecentProjectIdentityResolver;
|
||||
const baseTime = Date.parse('2026-04-14T12:00:00.000Z');
|
||||
const slowSessionPath = path.join(
|
||||
codexHome,
|
||||
'sessions',
|
||||
'2026',
|
||||
'04',
|
||||
'14',
|
||||
'rollout-slow.jsonl'
|
||||
);
|
||||
await writeRollout(
|
||||
slowSessionPath,
|
||||
{
|
||||
cwd: '/Users/test/projects/slow',
|
||||
},
|
||||
new Date(baseTime)
|
||||
);
|
||||
await writeRollout(
|
||||
path.join(codexHome, 'sessions', '2026', '04', '14', 'rollout-fast.jsonl'),
|
||||
{
|
||||
cwd: '/Users/test/projects/fast',
|
||||
},
|
||||
new Date(baseTime - 1000)
|
||||
);
|
||||
const originalOpen = fs.open.bind(fs);
|
||||
vi.spyOn(fs, 'open').mockImplementation(async (...args) => {
|
||||
if (String(args[0]) === slowSessionPath) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 2500));
|
||||
}
|
||||
return originalOpen(...args);
|
||||
});
|
||||
|
||||
const startedAt = Date.now();
|
||||
const adapter = new CodexSessionFileRecentProjectsSourceAdapter({
|
||||
getActiveContext: () => ({ type: 'local', id: 'local-1' }) as never,
|
||||
getLocalContext: () => ({ type: 'local', id: 'local-1' }) as never,
|
||||
identityResolver,
|
||||
logger,
|
||||
codexHome,
|
||||
appDataPath,
|
||||
});
|
||||
const result = await adapter.list();
|
||||
|
||||
expect(Date.now() - startedAt).toBeLessThan(1600);
|
||||
expect(result.degraded).toBe(true);
|
||||
expect(result.candidates.map((candidate) => candidate.primaryPath)).toEqual([
|
||||
'/Users/test/projects/fast',
|
||||
]);
|
||||
expect(logger.warn).toHaveBeenCalledWith(
|
||||
'codex session-file recent-projects source partial',
|
||||
expect.objectContaining({
|
||||
files: 2,
|
||||
timedOutReads: 1,
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it('ignores a corrupt session-file cache and rebuilds from session files', async () => {
|
||||
const codexHome = path.join(tempDir, '.codex');
|
||||
const appDataPath = path.join(tempDir, 'app-data');
|
||||
const logger = createLogger();
|
||||
const identityResolver = {
|
||||
resolve: vi.fn().mockResolvedValue(null),
|
||||
} as unknown as RecentProjectIdentityResolver;
|
||||
await writeRollout(
|
||||
path.join(codexHome, 'sessions', '2026', '04', '14', 'rollout-alpha.jsonl'),
|
||||
{
|
||||
cwd: '/Users/test/projects/alpha',
|
||||
},
|
||||
new Date('2026-04-14T12:00:00.000Z')
|
||||
);
|
||||
const cachePath = getSessionFileCachePath(appDataPath);
|
||||
await fs.mkdir(path.dirname(cachePath), { recursive: true });
|
||||
await fs.writeFile(cachePath, '{not-json', 'utf8');
|
||||
|
||||
const adapter = new CodexSessionFileRecentProjectsSourceAdapter({
|
||||
getActiveContext: () => ({ type: 'local', id: 'local-1' }) as never,
|
||||
getLocalContext: () => ({ type: 'local', id: 'local-1' }) as never,
|
||||
identityResolver,
|
||||
logger,
|
||||
codexHome,
|
||||
appDataPath,
|
||||
});
|
||||
|
||||
await expect(adapter.list()).resolves.toEqual({
|
||||
candidates: [
|
||||
expect.objectContaining({
|
||||
primaryPath: '/Users/test/projects/alpha',
|
||||
}),
|
||||
],
|
||||
degraded: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('returns a degraded partial result under the uncached read cap and completes on the next cached pass', async () => {
|
||||
const codexHome = path.join(tempDir, '.codex');
|
||||
const appDataPath = path.join(tempDir, 'app-data');
|
||||
const logger = createLogger();
|
||||
const identityResolver = {
|
||||
resolve: vi.fn().mockResolvedValue(null),
|
||||
} as unknown as RecentProjectIdentityResolver;
|
||||
const baseTime = Date.parse('2026-04-14T12:00:00.000Z');
|
||||
|
||||
await Promise.all(
|
||||
Array.from({ length: 170 }).map((_, index) =>
|
||||
writeRollout(
|
||||
path.join(codexHome, 'sessions', '2026', '04', '14', `rollout-alpha-${index}.jsonl`),
|
||||
{
|
||||
cwd: '/Users/test/projects/alpha',
|
||||
branch: 'main',
|
||||
},
|
||||
new Date(baseTime - index * 1000)
|
||||
)
|
||||
)
|
||||
);
|
||||
await writeRollout(
|
||||
path.join(codexHome, 'sessions', '2026', '04', '14', 'rollout-beta.jsonl'),
|
||||
{
|
||||
cwd: '/Users/test/projects/beta',
|
||||
branch: 'main',
|
||||
},
|
||||
new Date(baseTime - 200_000)
|
||||
);
|
||||
|
||||
const firstAdapter = new CodexSessionFileRecentProjectsSourceAdapter({
|
||||
getActiveContext: () => ({ type: 'local', id: 'local-1' }) as never,
|
||||
getLocalContext: () => ({ type: 'local', id: 'local-1' }) as never,
|
||||
identityResolver,
|
||||
logger,
|
||||
codexHome,
|
||||
appDataPath,
|
||||
});
|
||||
const firstResult = await firstAdapter.list();
|
||||
|
||||
expect(firstResult.degraded).toBe(true);
|
||||
expect(firstResult.candidates.map((candidate) => candidate.primaryPath)).toEqual([
|
||||
'/Users/test/projects/alpha',
|
||||
]);
|
||||
expect(logger.warn).toHaveBeenCalledWith(
|
||||
'codex session-file recent-projects source partial',
|
||||
expect.objectContaining({
|
||||
files: 171,
|
||||
uncachedReads: 160,
|
||||
skippedUncached: 11,
|
||||
})
|
||||
);
|
||||
|
||||
const secondAdapter = new CodexSessionFileRecentProjectsSourceAdapter({
|
||||
getActiveContext: () => ({ type: 'local', id: 'local-1' }) as never,
|
||||
getLocalContext: () => ({ type: 'local', id: 'local-1' }) as never,
|
||||
identityResolver,
|
||||
logger,
|
||||
codexHome,
|
||||
appDataPath,
|
||||
});
|
||||
const secondResult = await secondAdapter.list();
|
||||
|
||||
expect(secondResult.degraded).toBe(false);
|
||||
expect(secondResult.candidates.map((candidate) => candidate.primaryPath)).toEqual([
|
||||
'/Users/test/projects/alpha',
|
||||
'/Users/test/projects/beta',
|
||||
]);
|
||||
});
|
||||
|
||||
it('skips non-interactive and ephemeral sessions', async () => {
|
||||
const codexHome = path.join(tempDir, '.codex');
|
||||
const logger = createLogger();
|
||||
|
|
@ -302,6 +604,7 @@ describe('CodexSessionFileRecentProjectsSourceAdapter', () => {
|
|||
identityResolver,
|
||||
logger,
|
||||
codexHome,
|
||||
appDataPath: path.join(tempDir, 'app-data'),
|
||||
});
|
||||
|
||||
await expect(adapter.list()).resolves.toEqual({
|
||||
|
|
@ -322,6 +625,7 @@ describe('CodexSessionFileRecentProjectsSourceAdapter', () => {
|
|||
identityResolver,
|
||||
logger,
|
||||
codexHome: path.join(tempDir, 'missing-codex-home'),
|
||||
appDataPath: path.join(tempDir, 'app-data'),
|
||||
});
|
||||
|
||||
await expect(adapter.list()).resolves.toEqual({
|
||||
|
|
|
|||
|
|
@ -3,10 +3,11 @@ import * as fs from 'node:fs';
|
|||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
|
||||
import { afterEach, describe, expect, it } from 'vitest';
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { ProjectScanner } from '../../../../src/main/services/discovery/ProjectScanner';
|
||||
import { subprojectRegistry } from '../../../../src/main/services/discovery/SubprojectRegistry';
|
||||
import { configManager } from '../../../../src/main/services/infrastructure/ConfigManager';
|
||||
|
||||
import type {
|
||||
FileSystemProvider,
|
||||
|
|
@ -18,6 +19,9 @@ import type {
|
|||
interface CountingProvider extends FileSystemProvider {
|
||||
readonly readdirCounts: Map<string, number>;
|
||||
getMaxConcurrentReaddirs(): number;
|
||||
getMaxConcurrentStats(): number;
|
||||
getStatCount(): number;
|
||||
setProjectReaddirDelay(ms: number): void;
|
||||
releaseBlockedRead(): void;
|
||||
waitForBlockedRead(): Promise<void>;
|
||||
}
|
||||
|
|
@ -32,13 +36,44 @@ function createSessionLine(cwd: string): string {
|
|||
});
|
||||
}
|
||||
|
||||
function createProject(projectsDir: string, encodedName: string, cwd: string): string {
|
||||
function createProject(
|
||||
projectsDir: string,
|
||||
encodedName: string,
|
||||
cwd: string,
|
||||
sessionCount = 1
|
||||
): string {
|
||||
const projectDir = path.join(projectsDir, encodedName);
|
||||
fs.mkdirSync(projectDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(projectDir, 'session-1.jsonl'), `${createSessionLine(cwd)}\n`);
|
||||
for (let index = 1; index <= sessionCount; index += 1) {
|
||||
fs.writeFileSync(
|
||||
path.join(projectDir, `session-${index}.jsonl`),
|
||||
`${createSessionLine(cwd)}\n`
|
||||
);
|
||||
}
|
||||
return projectDir;
|
||||
}
|
||||
|
||||
function createSplitProject(
|
||||
projectsDir: string,
|
||||
encodedName: string,
|
||||
cwds: readonly string[]
|
||||
): string {
|
||||
const projectDir = path.join(projectsDir, encodedName);
|
||||
fs.mkdirSync(projectDir, { recursive: true });
|
||||
cwds.forEach((cwd, index) => {
|
||||
fs.writeFileSync(
|
||||
path.join(projectDir, `split-session-${index + 1}.jsonl`),
|
||||
`${createSessionLine(cwd)}\n`
|
||||
);
|
||||
});
|
||||
return projectDir;
|
||||
}
|
||||
|
||||
function buildCompositeId(baseDir: string, cwd: string): string {
|
||||
const hash = crypto.createHash('sha256').update(cwd).digest('hex').slice(0, 8);
|
||||
return `${baseDir}::${hash}`;
|
||||
}
|
||||
|
||||
function toStatResult(stats: fs.Stats): FsStatResult {
|
||||
return {
|
||||
size: stats.size,
|
||||
|
|
@ -57,12 +92,47 @@ function toDirent(entry: fs.Dirent): FsDirent {
|
|||
};
|
||||
}
|
||||
|
||||
function createCountingProvider(blockFirstReadPath?: string): CountingProvider {
|
||||
interface CountingProviderOptions {
|
||||
blockFirstReadPath?: string;
|
||||
projectReaddirDelayMs?: number;
|
||||
statDelayMs?: number;
|
||||
}
|
||||
|
||||
function delay(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
setTimeout(resolve, ms);
|
||||
});
|
||||
}
|
||||
|
||||
async function waitForCondition(
|
||||
predicate: () => boolean,
|
||||
timeoutMs = 1000,
|
||||
intervalMs = 20
|
||||
): Promise<boolean> {
|
||||
const startedAt = Date.now();
|
||||
while (Date.now() - startedAt < timeoutMs) {
|
||||
if (predicate()) {
|
||||
return true;
|
||||
}
|
||||
await delay(intervalMs);
|
||||
}
|
||||
return predicate();
|
||||
}
|
||||
|
||||
function createCountingProvider(options: string | CountingProviderOptions = {}): CountingProvider {
|
||||
const blockFirstReadPath = typeof options === 'string' ? options : options.blockFirstReadPath;
|
||||
let projectReaddirDelayMs =
|
||||
typeof options === 'string' ? 0 : (options.projectReaddirDelayMs ?? 0);
|
||||
const statDelayMs = typeof options === 'string' ? 0 : (options.statDelayMs ?? 0);
|
||||
const readdirCounts = new Map<string, number>();
|
||||
let rootReaddirPath: string | null = null;
|
||||
let blockedReadStartedResolve: (() => void) | null = null;
|
||||
let releaseBlockedReadResolve: (() => void) | null = null;
|
||||
let activeReaddirs = 0;
|
||||
let maxConcurrentReaddirs = 0;
|
||||
let activeStats = 0;
|
||||
let maxConcurrentStats = 0;
|
||||
let statCount = 0;
|
||||
const blockedReadStarted = blockFirstReadPath
|
||||
? new Promise<void>((resolve) => {
|
||||
blockedReadStartedResolve = resolve;
|
||||
|
|
@ -79,14 +149,28 @@ function createCountingProvider(blockFirstReadPath?: string): CountingProvider {
|
|||
return fs.promises.readFile(filePath, encoding);
|
||||
},
|
||||
async stat(filePath: string): Promise<FsStatResult> {
|
||||
return toStatResult(await fs.promises.stat(filePath));
|
||||
statCount += 1;
|
||||
activeStats += 1;
|
||||
maxConcurrentStats = Math.max(maxConcurrentStats, activeStats);
|
||||
try {
|
||||
if (statDelayMs > 0) {
|
||||
await delay(statDelayMs);
|
||||
}
|
||||
return toStatResult(await fs.promises.stat(filePath));
|
||||
} finally {
|
||||
activeStats -= 1;
|
||||
}
|
||||
},
|
||||
async readdir(dirPath: string): Promise<FsDirent[]> {
|
||||
activeReaddirs += 1;
|
||||
maxConcurrentReaddirs = Math.max(maxConcurrentReaddirs, activeReaddirs);
|
||||
try {
|
||||
rootReaddirPath ??= dirPath;
|
||||
const count = (readdirCounts.get(dirPath) ?? 0) + 1;
|
||||
readdirCounts.set(dirPath, count);
|
||||
if (projectReaddirDelayMs > 0 && dirPath !== rootReaddirPath) {
|
||||
await delay(projectReaddirDelayMs);
|
||||
}
|
||||
if (dirPath === blockFirstReadPath && count === 1) {
|
||||
blockedReadStartedResolve?.();
|
||||
await new Promise<void>((resolve) => {
|
||||
|
|
@ -110,6 +194,15 @@ function createCountingProvider(blockFirstReadPath?: string): CountingProvider {
|
|||
getMaxConcurrentReaddirs(): number {
|
||||
return maxConcurrentReaddirs;
|
||||
},
|
||||
getMaxConcurrentStats(): number {
|
||||
return maxConcurrentStats;
|
||||
},
|
||||
getStatCount(): number {
|
||||
return statCount;
|
||||
},
|
||||
setProjectReaddirDelay(ms: number): void {
|
||||
projectReaddirDelayMs = Math.max(0, ms);
|
||||
},
|
||||
releaseBlockedRead(): void {
|
||||
releaseBlockedReadResolve?.();
|
||||
},
|
||||
|
|
@ -124,6 +217,7 @@ describe('ProjectScanner scan dedup safe e2e', () => {
|
|||
|
||||
afterEach(() => {
|
||||
subprojectRegistry.clear();
|
||||
vi.restoreAllMocks();
|
||||
for (const dir of tempDirs) {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
|
|
@ -150,6 +244,315 @@ describe('ProjectScanner scan dedup safe e2e', () => {
|
|||
expect(provider.readdirCounts.get(projectDir)).toBe(1);
|
||||
});
|
||||
|
||||
it('shares one in-flight repository-group build including custom path enrichment', async () => {
|
||||
const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), 'scanner-groups-dedup-'));
|
||||
tempDirs.push(rootDir);
|
||||
const projectsDir = path.join(rootDir, 'projects');
|
||||
const encodedName = '-Users-test-groups-project';
|
||||
createProject(projectsDir, encodedName, '/Users/test/groups-project');
|
||||
const customPath = path.join(rootDir, 'manual-project');
|
||||
fs.mkdirSync(customPath, { recursive: true });
|
||||
const getCustomProjectPathsSpy = vi
|
||||
.spyOn(configManager, 'getCustomProjectPaths')
|
||||
.mockReturnValue([customPath]);
|
||||
const provider = createCountingProvider();
|
||||
const scanner = new ProjectScanner(projectsDir, undefined, provider);
|
||||
|
||||
const [firstGroups, secondGroups] = await Promise.all([
|
||||
scanner.scanWithWorktreeGrouping(),
|
||||
scanner.scanWithWorktreeGrouping(),
|
||||
]);
|
||||
|
||||
expect(firstGroups).toHaveLength(2);
|
||||
expect(secondGroups).toEqual(firstGroups);
|
||||
expect(getCustomProjectPathsSpy).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('does not reuse stale repository groups when custom project paths change', async () => {
|
||||
const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), 'scanner-groups-custom-fresh-'));
|
||||
tempDirs.push(rootDir);
|
||||
const projectsDir = path.join(rootDir, 'projects');
|
||||
createProject(projectsDir, '-Users-test-custom-fresh-project', '/Users/test/custom-fresh');
|
||||
const firstCustomPath = path.join(rootDir, 'first-manual-project');
|
||||
const secondCustomPath = path.join(rootDir, 'second-manual-project');
|
||||
fs.mkdirSync(firstCustomPath, { recursive: true });
|
||||
fs.mkdirSync(secondCustomPath, { recursive: true });
|
||||
vi.spyOn(configManager, 'getCustomProjectPaths')
|
||||
.mockReturnValueOnce([firstCustomPath])
|
||||
.mockReturnValueOnce([secondCustomPath]);
|
||||
const provider = createCountingProvider();
|
||||
const scanner = new ProjectScanner(projectsDir, undefined, provider);
|
||||
|
||||
const firstGroups = await scanner.scanWithWorktreeGrouping();
|
||||
const secondGroups = await scanner.scanWithWorktreeGrouping();
|
||||
|
||||
expect(firstGroups.some((group) => group.worktrees[0]?.path === firstCustomPath)).toBe(true);
|
||||
expect(secondGroups.some((group) => group.worktrees[0]?.path === secondCustomPath)).toBe(true);
|
||||
expect(secondGroups.some((group) => group.worktrees[0]?.path === firstCustomPath)).toBe(false);
|
||||
});
|
||||
|
||||
it('continues scanning later project directories while one project is slow', async () => {
|
||||
const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), 'scanner-rolling-projects-'));
|
||||
tempDirs.push(rootDir);
|
||||
const projectsDir = path.join(rootDir, 'projects');
|
||||
const projectDirs: string[] = [];
|
||||
for (let projectIndex = 0; projectIndex < 14; projectIndex += 1) {
|
||||
projectDirs.push(
|
||||
createProject(
|
||||
projectsDir,
|
||||
`-Users-test-rolling-${projectIndex}`,
|
||||
`/Users/test/rolling-${projectIndex}`
|
||||
)
|
||||
);
|
||||
}
|
||||
const provider = createCountingProvider(projectDirs[0]);
|
||||
const scanner = new ProjectScanner(projectsDir, undefined, provider);
|
||||
|
||||
const scan = scanner.scan();
|
||||
await provider.waitForBlockedRead();
|
||||
const laterProjectScanned = await waitForCondition(
|
||||
() => (provider.readdirCounts.get(projectDirs[13]) ?? 0) > 0
|
||||
);
|
||||
|
||||
provider.releaseBlockedRead();
|
||||
const projects = await scan;
|
||||
|
||||
expect(laterProjectScanned).toBe(true);
|
||||
expect(projects).toHaveLength(14);
|
||||
});
|
||||
|
||||
it('uses a bounded cwd hint instead of partial splitting for larger project dirs', async () => {
|
||||
const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), 'scanner-bounded-cwd-'));
|
||||
tempDirs.push(rootDir);
|
||||
const projectsDir = path.join(rootDir, 'projects');
|
||||
const encodedName = '-Users-test-bounded-cwd';
|
||||
createSplitProject(
|
||||
projectsDir,
|
||||
encodedName,
|
||||
Array.from({ length: 13 }, (_value, index) => `/Users/test/bounded-cwd/app-${index}`)
|
||||
);
|
||||
const provider = createCountingProvider();
|
||||
const scanner = new ProjectScanner(projectsDir, undefined, provider);
|
||||
|
||||
const projects = await scanner.scan();
|
||||
|
||||
expect(projects).toHaveLength(1);
|
||||
expect(projects[0]).toMatchObject({
|
||||
id: encodedName,
|
||||
totalSessions: 13,
|
||||
});
|
||||
expect(projects[0]?.path).toContain('/Users/test/bounded-cwd/app-');
|
||||
expect(subprojectRegistry.isComposite(projects[0]?.id ?? '')).toBe(false);
|
||||
});
|
||||
|
||||
it('bounds local scan stat concurrency without restatting files during cwd extraction', async () => {
|
||||
const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), 'scanner-io-limit-'));
|
||||
tempDirs.push(rootDir);
|
||||
const projectsDir = path.join(rootDir, 'projects');
|
||||
for (let projectIndex = 0; projectIndex < 6; projectIndex += 1) {
|
||||
createProject(
|
||||
projectsDir,
|
||||
`-Users-test-io-limit-${projectIndex}`,
|
||||
`/Users/test/io-limit-${projectIndex}`,
|
||||
5
|
||||
);
|
||||
}
|
||||
const provider = createCountingProvider({ statDelayMs: 10 });
|
||||
const scanner = new ProjectScanner(projectsDir, undefined, provider, {
|
||||
scanFileIoConcurrency: 2,
|
||||
});
|
||||
|
||||
const projects = await scanner.scan();
|
||||
|
||||
expect(projects).toHaveLength(6);
|
||||
expect(provider.getStatCount()).toBe(30);
|
||||
expect(provider.getMaxConcurrentStats()).toBeLessThanOrEqual(2);
|
||||
});
|
||||
|
||||
it('aborts queued local file I/O after a project scan budget timeout', async () => {
|
||||
const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), 'scanner-io-abort-'));
|
||||
tempDirs.push(rootDir);
|
||||
const projectsDir = path.join(rootDir, 'projects');
|
||||
createProject(projectsDir, '-Users-test-io-abort', '/Users/test/io-abort', 8);
|
||||
const provider = createCountingProvider({ statDelayMs: 250 });
|
||||
const scanner = new ProjectScanner(projectsDir, undefined, provider, {
|
||||
scanBudgetMs: 1300,
|
||||
scanFileIoConcurrency: 1,
|
||||
});
|
||||
|
||||
await expect(scanner.scan()).resolves.toEqual([]);
|
||||
await delay(2400);
|
||||
|
||||
expect(provider.getStatCount()).toBeGreaterThan(0);
|
||||
expect(provider.getStatCount()).toBeLessThan(8);
|
||||
});
|
||||
|
||||
it('returns an uncached partial scan before the scan budget can hit renderer timeout', async () => {
|
||||
const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), 'scanner-budget-'));
|
||||
tempDirs.push(rootDir);
|
||||
const projectsDir = path.join(rootDir, 'projects');
|
||||
const projectCount = 6;
|
||||
for (let projectIndex = 0; projectIndex < projectCount; projectIndex += 1) {
|
||||
createProject(
|
||||
projectsDir,
|
||||
`-Users-test-budget-${projectIndex}`,
|
||||
`/Users/test/budget-${projectIndex}`
|
||||
);
|
||||
}
|
||||
const provider = createCountingProvider({ projectReaddirDelayMs: 250 });
|
||||
const scanner = new ProjectScanner(projectsDir, undefined, provider, {
|
||||
scanBudgetMs: 75,
|
||||
});
|
||||
|
||||
const firstStartedAt = Date.now();
|
||||
const firstProjects = await scanner.scan();
|
||||
const firstMs = Date.now() - firstStartedAt;
|
||||
|
||||
expect(firstMs).toBeLessThan(1000);
|
||||
expect(firstProjects).toEqual([]);
|
||||
expect(provider.readdirCounts.get(projectsDir)).toBe(1);
|
||||
|
||||
await delay(300);
|
||||
await scanner.scan();
|
||||
expect(provider.readdirCounts.get(projectsDir)).toBe(2);
|
||||
});
|
||||
|
||||
it('does not cache a scan when project-level timeout omits directories', async () => {
|
||||
const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), 'scanner-project-timeout-'));
|
||||
tempDirs.push(rootDir);
|
||||
const projectsDir = path.join(rootDir, 'projects');
|
||||
for (let projectIndex = 0; projectIndex < 2; projectIndex += 1) {
|
||||
createProject(
|
||||
projectsDir,
|
||||
`-Users-test-project-timeout-${projectIndex}`,
|
||||
`/Users/test/project-timeout-${projectIndex}`
|
||||
);
|
||||
}
|
||||
const provider = createCountingProvider({ projectReaddirDelayMs: 1100 });
|
||||
const scanner = new ProjectScanner(projectsDir, undefined, provider, {
|
||||
scanBudgetMs: 1000,
|
||||
});
|
||||
|
||||
const firstProjects = await scanner.scan();
|
||||
const firstRootReads = provider.readdirCounts.get(projectsDir);
|
||||
const secondProjects = await scanner.scan();
|
||||
|
||||
expect(firstProjects).toEqual([]);
|
||||
expect(secondProjects).toEqual([]);
|
||||
expect(firstRootReads).toBe(1);
|
||||
expect(provider.readdirCounts.get(projectsDir)).toBe(2);
|
||||
});
|
||||
|
||||
it('does not let a timed-out background project scan mutate the subproject registry', async () => {
|
||||
const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), 'scanner-timeout-registry-'));
|
||||
tempDirs.push(rootDir);
|
||||
const projectsDir = path.join(rootDir, 'projects');
|
||||
const encodedName = '-Users-test-timeout-registry';
|
||||
const firstCwd = '/Users/test/timeout-registry/app';
|
||||
const secondCwd = '/Users/test/timeout-registry/docs';
|
||||
createSplitProject(projectsDir, encodedName, [firstCwd, secondCwd]);
|
||||
const provider = createCountingProvider({ statDelayMs: 1200 });
|
||||
const scanner = new ProjectScanner(projectsDir, undefined, provider, {
|
||||
scanBudgetMs: 1050,
|
||||
scanFileIoConcurrency: 2,
|
||||
});
|
||||
|
||||
await expect(scanner.scan()).resolves.toEqual([]);
|
||||
await delay(1600);
|
||||
|
||||
expect(subprojectRegistry.getCwd(buildCompositeId(encodedName, firstCwd))).toBeNull();
|
||||
expect(subprojectRegistry.getCwd(buildCompositeId(encodedName, secondCwd))).toBeNull();
|
||||
});
|
||||
|
||||
it('returns the previous complete scan instead of replacing it with a budget partial', async () => {
|
||||
const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), 'scanner-budget-cache-'));
|
||||
tempDirs.push(rootDir);
|
||||
const projectsDir = path.join(rootDir, 'projects');
|
||||
const projectCount = 6;
|
||||
for (let projectIndex = 0; projectIndex < projectCount; projectIndex += 1) {
|
||||
createProject(
|
||||
projectsDir,
|
||||
`-Users-test-budget-cache-${projectIndex}`,
|
||||
`/Users/test/budget-cache-${projectIndex}`
|
||||
);
|
||||
}
|
||||
const provider = createCountingProvider();
|
||||
const scanner = new ProjectScanner(projectsDir, undefined, provider, {
|
||||
scanBudgetMs: 1500,
|
||||
});
|
||||
|
||||
const firstProjects = await scanner.scan();
|
||||
expect(firstProjects).toHaveLength(projectCount);
|
||||
|
||||
provider.setProjectReaddirDelay(1700);
|
||||
await delay(2100);
|
||||
const secondProjects = await scanner.scan();
|
||||
|
||||
expect(secondProjects).toHaveLength(projectCount);
|
||||
expect(secondProjects.map((project) => project.id)).toEqual(
|
||||
firstProjects.map((project) => project.id)
|
||||
);
|
||||
expect(provider.readdirCounts.get(projectsDir)).toBe(2);
|
||||
});
|
||||
|
||||
it('restores subproject registry when falling back to a previous complete scan', async () => {
|
||||
const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), 'scanner-budget-registry-'));
|
||||
tempDirs.push(rootDir);
|
||||
const projectsDir = path.join(rootDir, 'projects');
|
||||
createSplitProject(projectsDir, '-Users-test-registry-project', [
|
||||
'/Users/test/registry-project/app',
|
||||
'/Users/test/registry-project/docs',
|
||||
]);
|
||||
const provider = createCountingProvider();
|
||||
const scanner = new ProjectScanner(projectsDir, undefined, provider, {
|
||||
scanBudgetMs: 1500,
|
||||
});
|
||||
|
||||
const firstProjects = await scanner.scan();
|
||||
expect(firstProjects).toHaveLength(2);
|
||||
expect(firstProjects.every((project) => subprojectRegistry.isComposite(project.id))).toBe(true);
|
||||
const firstFilters = firstProjects.map((project) =>
|
||||
subprojectRegistry.getSessionFilter(project.id)
|
||||
);
|
||||
expect(firstFilters.every((filter) => filter?.size === 1)).toBe(true);
|
||||
|
||||
provider.setProjectReaddirDelay(1700);
|
||||
await delay(2100);
|
||||
const secondProjects = await scanner.scan();
|
||||
|
||||
expect(secondProjects.map((project) => project.id)).toEqual(
|
||||
firstProjects.map((project) => project.id)
|
||||
);
|
||||
for (const project of secondProjects) {
|
||||
expect(subprojectRegistry.getSessionFilter(project.id)?.size).toBe(1);
|
||||
expect(subprojectRegistry.getCwd(project.id)).toContain('/Users/test/registry-project/');
|
||||
}
|
||||
});
|
||||
|
||||
it('keeps an in-flight split-project scan complete after cache invalidation', async () => {
|
||||
const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), 'scanner-split-inflight-'));
|
||||
tempDirs.push(rootDir);
|
||||
const projectsDir = path.join(rootDir, 'projects');
|
||||
const projectDir = createSplitProject(projectsDir, '-Users-test-split-inflight', [
|
||||
'/Users/test/split-inflight/app',
|
||||
'/Users/test/split-inflight/docs',
|
||||
]);
|
||||
const provider = createCountingProvider(projectDir);
|
||||
const scanner = new ProjectScanner(projectsDir, undefined, provider);
|
||||
|
||||
const firstScan = scanner.scan();
|
||||
await provider.waitForBlockedRead();
|
||||
scanner.clearScanCache();
|
||||
provider.releaseBlockedRead();
|
||||
const projects = await firstScan;
|
||||
|
||||
expect(projects).toHaveLength(2);
|
||||
expect(projects.every((project) => subprojectRegistry.isComposite(project.id))).toBe(true);
|
||||
expect(
|
||||
projects.every((project) => subprojectRegistry.getSessionFilter(project.id)?.size === 1)
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it('does not cache an in-flight scan after clearScanCache invalidates it', async () => {
|
||||
const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), 'scanner-dedup-clear-'));
|
||||
tempDirs.push(rootDir);
|
||||
|
|
|
|||
Loading…
Reference in a new issue