perf(recent-projects): bound codex session discovery

This commit is contained in:
777genius 2026-05-26 13:24:49 +03:00
parent 4640e1eea4
commit c2bc20bebd
4 changed files with 382 additions and 45 deletions

View file

@ -10,18 +10,48 @@ import type { FastifyInstance } from 'fastify';
const logger = createLogger('Feature:RecentProjects:HTTP'); const logger = createLogger('Feature:RecentProjects:HTTP');
function getPayloadBytes(value: unknown): number {
try {
return Buffer.byteLength(JSON.stringify(value), 'utf8');
} catch {
return -1;
}
}
function getMemoryDiagnostics(): {
rssBytes: number;
heapUsedBytes: number;
heapTotalBytes: number;
} {
const memory = process.memoryUsage();
return {
rssBytes: memory.rss,
heapUsedBytes: memory.heapUsed,
heapTotalBytes: memory.heapTotal,
};
}
export function registerRecentProjectsHttp( export function registerRecentProjectsHttp(
app: FastifyInstance, app: FastifyInstance,
feature: RecentProjectsFeatureFacade feature: RecentProjectsFeatureFacade
): void { ): void {
app.get(DASHBOARD_RECENT_PROJECTS_ROUTE, async (): Promise<DashboardRecentProjectsPayload> => { app.get(DASHBOARD_RECENT_PROJECTS_ROUTE, async (): Promise<DashboardRecentProjectsPayload> => {
const startedAt = Date.now();
try { try {
return ( const payload = normalizeDashboardRecentProjectsPayload(
normalizeDashboardRecentProjectsPayload(await feature.listDashboardRecentProjects()) ?? { await feature.listDashboardRecentProjects()
projects: [], ) ?? {
degraded: true, projects: [],
} degraded: true,
); };
logger.info('dashboard recent-projects HTTP loaded', {
count: payload.projects.length,
degraded: payload.degraded,
durationMs: Date.now() - startedAt,
payloadBytes: getPayloadBytes(payload),
...getMemoryDiagnostics(),
});
return payload;
} catch (error) { } catch (error) {
logger.error('Failed to load dashboard recent projects via HTTP', error); logger.error('Failed to load dashboard recent projects via HTTP', error);
return { projects: [], degraded: true }; return { projects: [], degraded: true };

View file

@ -9,18 +9,48 @@ import type { IpcMain } from 'electron';
const logger = createLogger('Feature:RecentProjects:IPC'); const logger = createLogger('Feature:RecentProjects:IPC');
function getPayloadBytes(value: unknown): number {
try {
return Buffer.byteLength(JSON.stringify(value), 'utf8');
} catch {
return -1;
}
}
function getMemoryDiagnostics(): {
rssBytes: number;
heapUsedBytes: number;
heapTotalBytes: number;
} {
const memory = process.memoryUsage();
return {
rssBytes: memory.rss,
heapUsedBytes: memory.heapUsed,
heapTotalBytes: memory.heapTotal,
};
}
export function registerRecentProjectsIpc( export function registerRecentProjectsIpc(
ipcMain: IpcMain, ipcMain: IpcMain,
feature: RecentProjectsFeatureFacade feature: RecentProjectsFeatureFacade
): void { ): void {
ipcMain.handle(GET_DASHBOARD_RECENT_PROJECTS, async () => { ipcMain.handle(GET_DASHBOARD_RECENT_PROJECTS, async () => {
const startedAt = Date.now();
try { try {
return ( const payload = normalizeDashboardRecentProjectsPayload(
normalizeDashboardRecentProjectsPayload(await feature.listDashboardRecentProjects()) ?? { await feature.listDashboardRecentProjects()
projects: [], ) ?? {
degraded: true, projects: [],
} degraded: true,
); };
logger.info('dashboard recent-projects IPC loaded', {
count: payload.projects.length,
degraded: payload.degraded,
durationMs: Date.now() - startedAt,
payloadBytes: getPayloadBytes(payload),
...getMemoryDiagnostics(),
});
return payload;
} catch (error) { } catch (error) {
logger.error('Failed to load dashboard recent projects via IPC', error); logger.error('Failed to load dashboard recent projects via IPC', error);
return { projects: [], degraded: true }; return { projects: [], degraded: true };

View file

@ -23,6 +23,7 @@ const CODEX_SESSION_FILE_SOFT_BUDGET_MS = 6_500;
const CODEX_SESSION_FILE_MAX_UNCACHED_READS_PER_RUN = 160; const CODEX_SESSION_FILE_MAX_UNCACHED_READS_PER_RUN = 160;
const CODEX_SESSION_FILE_READ_BATCH_SIZE = 24; const CODEX_SESSION_FILE_READ_BATCH_SIZE = 24;
const CODEX_SESSION_FILE_READ_TIMEOUT_MS = 700; const CODEX_SESSION_FILE_READ_TIMEOUT_MS = 700;
const CODEX_SESSION_FILE_DISCOVERY_STAT_BATCH_SIZE = 64;
const CODEX_SESSION_METADATA_READ_LIMIT_BYTES = 128 * 1024; const CODEX_SESSION_METADATA_READ_LIMIT_BYTES = 128 * 1024;
const CODEX_SESSION_FILE_CACHE_SCHEMA_VERSION = 1; const CODEX_SESSION_FILE_CACHE_SCHEMA_VERSION = 1;
const CODEX_SESSION_FILE_CACHE_RELATIVE_PATH = path.join( const CODEX_SESSION_FILE_CACHE_RELATIVE_PATH = path.join(
@ -80,6 +81,11 @@ interface CodexSessionSnapshotLoadResult {
degraded: boolean; degraded: boolean;
stats: { stats: {
files: number; files: number;
visitedFiles: number;
droppedOlderFiles: number;
statFailures: number;
directoriesVisited: number;
discoveryTimedOut: boolean;
cached: number; cached: number;
uncachedReads: number; uncachedReads: number;
timedOutReads: number; timedOutReads: number;
@ -88,6 +94,14 @@ interface CodexSessionSnapshotLoadResult {
}; };
} }
interface CodexSessionFileListingResult {
files: CodexSessionFileEntry[];
visitedFiles: number;
statFailures: number;
directoriesVisited: number;
timedOut: boolean;
}
function emptyCache(): CodexSessionFileCacheFile { function emptyCache(): CodexSessionFileCacheFile {
return { return {
schemaVersion: CODEX_SESSION_FILE_CACHE_SCHEMA_VERSION, schemaVersion: CODEX_SESSION_FILE_CACHE_SCHEMA_VERSION,
@ -95,6 +109,21 @@ function emptyCache(): CodexSessionFileCacheFile {
}; };
} }
function captureMemoryDiagnostics(): {
rssBytes: number;
heapUsedBytes: number;
heapTotalBytes: number;
externalBytes: number;
} {
const memory = process.memoryUsage();
return {
rssBytes: memory.rss,
heapUsedBytes: memory.heapUsed,
heapTotalBytes: memory.heapTotal,
externalBytes: memory.external,
};
}
function isUsableCacheEntry( function isUsableCacheEntry(
entry: CodexSessionFileCacheEntry | undefined, entry: CodexSessionFileCacheEntry | undefined,
file: CodexSessionFileEntry file: CodexSessionFileEntry
@ -211,45 +240,149 @@ async function readFirstLineWithTimeout(
return result; return result;
} }
async function listJsonlFiles(root: string, maxDepth: number): Promise<CodexSessionFileEntry[]> { function insertRecentSessionFile(
async function walk(directory: string, depth: number): Promise<CodexSessionFileEntry[]> { files: CodexSessionFileEntry[],
file: CodexSessionFileEntry,
limit: number
): void {
if (limit <= 0) {
return;
}
if (files.length >= limit && file.mtimeMs <= files[files.length - 1].mtimeMs) {
return;
}
let low = 0;
let high = files.length;
while (low < high) {
const mid = Math.floor((low + high) / 2);
if (file.mtimeMs > files[mid].mtimeMs) {
high = mid;
} else {
low = mid + 1;
}
}
files.splice(low, 0, file);
if (files.length > limit) {
files.pop();
}
}
function selectMostRecentSessionFiles(
files: CodexSessionFileEntry[],
limit: number
): CodexSessionFileEntry[] {
const selected: CodexSessionFileEntry[] = [];
for (const file of files) {
insertRecentSessionFile(selected, file, limit);
}
return selected;
}
async function listRecentJsonlFiles(
root: string,
maxDepth: number,
limit: number,
deadlineMs: number
): Promise<CodexSessionFileListingResult> {
const selectedFiles: CodexSessionFileEntry[] = [];
let visitedFiles = 0;
let statFailures = 0;
let directoriesVisited = 0;
let timedOut = false;
const hasBudget = (): boolean => {
if (Date.now() < deadlineMs) {
return true;
}
timedOut = true;
return false;
};
async function statJsonlFile(filePath: string): Promise<CodexSessionFileEntry | null> {
if (!hasBudget()) {
return null;
}
visitedFiles += 1;
try {
const stats = await fs.stat(filePath);
return {
filePath,
mtimeMs: stats.mtimeMs,
size: stats.size,
};
} catch {
statFailures += 1;
return null;
}
}
async function collectFileStats(filePaths: string[]): Promise<void> {
for (
let offset = 0;
offset < filePaths.length && hasBudget();
offset += CODEX_SESSION_FILE_DISCOVERY_STAT_BATCH_SIZE
) {
const batch = filePaths.slice(offset, offset + CODEX_SESSION_FILE_DISCOVERY_STAT_BATCH_SIZE);
const stats = await Promise.all(batch.map((filePath) => statJsonlFile(filePath)));
for (const file of stats) {
if (file) {
insertRecentSessionFile(selectedFiles, file, limit);
}
}
}
}
async function walk(directory: string, depth: number): Promise<void> {
if (!hasBudget()) {
return;
}
let entries; let entries;
try { try {
entries = await fs.readdir(directory, { withFileTypes: true, encoding: 'utf8' }); entries = await fs.readdir(directory, { withFileTypes: true, encoding: 'utf8' });
} catch { } catch {
return []; return;
} }
const files = await Promise.all( directoriesVisited += 1;
entries.map(async (entry): Promise<CodexSessionFileEntry[]> => { const filePaths: string[] = [];
const entryPath = path.join(directory, entry.name); const childDirectories: string[] = [];
if (entry.isDirectory()) {
return depth < maxDepth ? walk(entryPath, depth + 1) : [];
}
if (!entry.isFile() || !entry.name.endsWith('.jsonl')) { for (const entry of entries) {
return []; const entryPath = path.join(directory, entry.name);
if (entry.isDirectory()) {
if (depth < maxDepth) {
childDirectories.push(entryPath);
} }
continue;
}
try { if (entry.isFile() && entry.name.endsWith('.jsonl')) {
const stats = await fs.stat(entryPath); filePaths.push(entryPath);
return [ }
{ }
filePath: entryPath,
mtimeMs: stats.mtimeMs,
size: stats.size,
},
];
} catch {
return [];
}
})
);
return files.flat(); await collectFileStats(filePaths);
for (const childDirectory of childDirectories) {
if (!hasBudget()) {
return;
}
await walk(childDirectory, depth + 1);
}
} }
return walk(root, 0); await walk(root, 0);
return {
files: selectedFiles,
visitedFiles,
statFailures,
directoriesVisited,
timedOut,
};
} }
function parseSessionSnapshot( function parseSessionSnapshot(
@ -294,6 +427,7 @@ export class CodexSessionFileRecentProjectsSourceAdapter implements RecentProjec
readonly timeoutMs = CODEX_SESSION_FILE_SOURCE_TIMEOUT_MS; readonly timeoutMs = CODEX_SESSION_FILE_SOURCE_TIMEOUT_MS;
readonly #codexHome: string; readonly #codexHome: string;
readonly #cachePath: string; readonly #cachePath: string;
#inFlightList: Promise<RecentProjectsSourceResult> | null = null;
constructor( constructor(
private readonly deps: { private readonly deps: {
@ -313,6 +447,20 @@ export class CodexSessionFileRecentProjectsSourceAdapter implements RecentProjec
} }
async list(): Promise<RecentProjectsSourceResult> { async list(): Promise<RecentProjectsSourceResult> {
if (this.#inFlightList) {
return this.#inFlightList;
}
const request = this.#listUncached().finally(() => {
if (this.#inFlightList === request) {
this.#inFlightList = null;
}
});
this.#inFlightList = request;
return request;
}
async #listUncached(): Promise<RecentProjectsSourceResult> {
const activeContext = this.deps.getActiveContext(); const activeContext = this.deps.getActiveContext();
const localContext = this.deps.getLocalContext(); const localContext = this.deps.getLocalContext();
@ -339,6 +487,7 @@ export class CodexSessionFileRecentProjectsSourceAdapter implements RecentProjec
count: validCandidates.length, count: validCandidates.length,
codexHome: this.#codexHome, codexHome: this.#codexHome,
degraded: snapshotResult.degraded, degraded: snapshotResult.degraded,
...captureMemoryDiagnostics(),
...snapshotResult.stats, ...snapshotResult.stats,
}); });
@ -361,16 +510,34 @@ export class CodexSessionFileRecentProjectsSourceAdapter implements RecentProjec
async #listRecentSessionSnapshots(): Promise<CodexSessionSnapshotLoadResult> { async #listRecentSessionSnapshots(): Promise<CodexSessionSnapshotLoadResult> {
const startedAt = Date.now(); const startedAt = Date.now();
const deadline = startedAt + CODEX_SESSION_FILE_SOFT_BUDGET_MS; const deadline = startedAt + CODEX_SESSION_FILE_SOFT_BUDGET_MS;
const files = [ const sessionFiles = await listRecentJsonlFiles(
...(await listJsonlFiles(path.join(this.#codexHome, 'sessions'), 4)), path.join(this.#codexHome, 'sessions'),
...(await listJsonlFiles(path.join(this.#codexHome, 'archived_sessions'), 1)), 4,
].sort((left, right) => right.mtimeMs - left.mtimeMs); CODEX_SESSION_FILE_PARSE_LIMIT,
deadline
);
const archivedSessionFiles = await listRecentJsonlFiles(
path.join(this.#codexHome, 'archived_sessions'),
1,
CODEX_SESSION_FILE_PARSE_LIMIT,
deadline
);
const files = selectMostRecentSessionFiles(
[...sessionFiles.files, ...archivedSessionFiles.files],
CODEX_SESSION_FILE_PARSE_LIMIT
);
const visitedFiles = sessionFiles.visitedFiles + archivedSessionFiles.visitedFiles;
const statFailures = sessionFiles.statFailures + archivedSessionFiles.statFailures;
const directoriesVisited =
sessionFiles.directoriesVisited + archivedSessionFiles.directoriesVisited;
const droppedOlderFiles = Math.max(0, visitedFiles - statFailures - files.length);
const discoveryTimedOut = sessionFiles.timedOut || archivedSessionFiles.timedOut;
const snapshotsByCwd = new Map<string, CodexSessionProjectSnapshot>(); const snapshotsByCwd = new Map<string, CodexSessionProjectSnapshot>();
const candidateFiles = files.slice(0, CODEX_SESSION_FILE_PARSE_LIMIT); const candidateFiles = files;
const cache = await this.#readCacheSafe(); const cache = await this.#readCacheSafe();
const nextCacheEntries = new Map<string, CodexSessionFileCacheEntry>(); const nextCacheEntries = new Map<string, CodexSessionFileCacheEntry>();
let degraded = false; let degraded = discoveryTimedOut;
let cached = 0; let cached = 0;
let uncachedReads = 0; let uncachedReads = 0;
let timedOutReads = 0; let timedOutReads = 0;
@ -454,6 +621,12 @@ export class CodexSessionFileRecentProjectsSourceAdapter implements RecentProjec
if (degraded) { if (degraded) {
this.deps.logger.warn('codex session-file recent-projects source partial', { this.deps.logger.warn('codex session-file recent-projects source partial', {
files: candidateFiles.length, files: candidateFiles.length,
visitedFiles,
droppedOlderFiles,
statFailures,
directoriesVisited,
discoveryTimedOut,
...captureMemoryDiagnostics(),
cached, cached,
uncachedReads, uncachedReads,
timedOutReads, timedOutReads,
@ -468,6 +641,11 @@ export class CodexSessionFileRecentProjectsSourceAdapter implements RecentProjec
degraded, degraded,
stats: { stats: {
files: candidateFiles.length, files: candidateFiles.length,
visitedFiles,
droppedOlderFiles,
statFailures,
directoriesVisited,
discoveryTimedOut,
cached, cached,
uncachedReads, uncachedReads,
timedOutReads, timedOutReads,

View file

@ -57,6 +57,20 @@ async function writeRollout(
await fs.utimes(filePath, mtime, mtime); await fs.utimes(filePath, mtime, mtime);
} }
function deferred<T>(): {
promise: Promise<T>;
resolve: (value: T) => void;
reject: (error: unknown) => void;
} {
let resolve!: (value: T) => void;
let reject!: (error: unknown) => void;
const promise = new Promise<T>((promiseResolve, promiseReject) => {
resolve = promiseResolve;
reject = promiseReject;
});
return { promise, resolve, reject };
}
describe('CodexSessionFileRecentProjectsSourceAdapter', () => { describe('CodexSessionFileRecentProjectsSourceAdapter', () => {
let tempDir: string; let tempDir: string;
@ -338,6 +352,43 @@ describe('CodexSessionFileRecentProjectsSourceAdapter', () => {
expect(openSpy).not.toHaveBeenCalled(); expect(openSpy).not.toHaveBeenCalled();
}); });
it('coalesces concurrent Codex session-file source reads', async () => {
const codexHome = path.join(tempDir, '.codex');
const logger = createLogger();
const resolveResult = deferred<null>();
const identityResolver = {
resolve: vi.fn().mockReturnValue(resolveResult.promise),
} as unknown as RecentProjectIdentityResolver;
await writeRollout(
path.join(codexHome, 'sessions', '2026', '04', '14', 'rollout-alpha.jsonl'),
{
cwd: '/Users/test/projects/alpha',
branch: 'main',
},
new Date('2026-04-14T12:00:00.000Z')
);
const adapter = new CodexSessionFileRecentProjectsSourceAdapter({
getActiveContext: () => ({ type: 'local', id: 'local-1' }) as never,
getLocalContext: () => ({ type: 'local', id: 'local-1' }) as never,
identityResolver,
logger,
codexHome,
appDataPath: path.join(tempDir, 'app-data'),
});
const first = adapter.list();
await vi.waitFor(() => expect(identityResolver.resolve).toHaveBeenCalledTimes(1));
const second = adapter.list();
resolveResult.resolve(null);
await expect(Promise.all([first, second])).resolves.toEqual([
expect.objectContaining({ degraded: false }),
expect.objectContaining({ degraded: false }),
]);
expect(identityResolver.resolve).toHaveBeenCalledTimes(1);
});
it('invalidates cached session metadata when the jsonl fingerprint changes', async () => { it('invalidates cached session metadata when the jsonl fingerprint changes', async () => {
const codexHome = path.join(tempDir, '.codex'); const codexHome = path.join(tempDir, '.codex');
const appDataPath = path.join(tempDir, 'app-data'); const appDataPath = path.join(tempDir, 'app-data');
@ -575,6 +626,54 @@ describe('CodexSessionFileRecentProjectsSourceAdapter', () => {
]); ]);
}); });
it('bounds discovered Codex session files before reading metadata', async () => {
const codexHome = path.join(tempDir, '.codex');
const appDataPath = path.join(tempDir, 'app-data');
const logger = createLogger();
const identityResolver = {
resolve: vi.fn().mockResolvedValue(null),
} as unknown as RecentProjectIdentityResolver;
const baseTime = Date.parse('2026-04-14T12:00:00.000Z');
await Promise.all(
Array.from({ length: 505 }).map((_, index) =>
writeRollout(
path.join(codexHome, 'sessions', '2026', '04', '14', `rollout-alpha-${index}.jsonl`),
{
cwd: '/Users/test/projects/alpha',
branch: 'main',
},
new Date(baseTime - index * 1000)
)
)
);
const adapter = new CodexSessionFileRecentProjectsSourceAdapter({
getActiveContext: () => ({ type: 'local', id: 'local-1' }) as never,
getLocalContext: () => ({ type: 'local', id: 'local-1' }) as never,
identityResolver,
logger,
codexHome,
appDataPath,
});
const result = await adapter.list();
expect(result.degraded).toBe(true);
expect(result.candidates.map((candidate) => candidate.primaryPath)).toEqual([
'/Users/test/projects/alpha',
]);
expect(logger.warn).toHaveBeenCalledWith(
'codex session-file recent-projects source partial',
expect.objectContaining({
files: 500,
visitedFiles: 505,
droppedOlderFiles: 5,
uncachedReads: 160,
skippedUncached: 340,
})
);
});
it('skips non-interactive and ephemeral sessions', async () => { it('skips non-interactive and ephemeral sessions', async () => {
const codexHome = path.join(tempDir, '.codex'); const codexHome = path.join(tempDir, '.codex');
const logger = createLogger(); const logger = createLogger();