perf(recent-projects): bound codex session discovery
This commit is contained in:
777genius
parent
4640e1eea4
commit
c2bc20bebd
4 changed files with 382 additions and 45 deletions
|
|
@ -10,18 +10,48 @@ import type { FastifyInstance } from 'fastify';
|
|||
|
||||
const logger = createLogger('Feature:RecentProjects:HTTP');
|
||||
|
||||
function getPayloadBytes(value: unknown): number {
|
||||
try {
|
||||
return Buffer.byteLength(JSON.stringify(value), 'utf8');
|
||||
} catch {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
function getMemoryDiagnostics(): {
|
||||
rssBytes: number;
|
||||
heapUsedBytes: number;
|
||||
heapTotalBytes: number;
|
||||
} {
|
||||
const memory = process.memoryUsage();
|
||||
return {
|
||||
rssBytes: memory.rss,
|
||||
heapUsedBytes: memory.heapUsed,
|
||||
heapTotalBytes: memory.heapTotal,
|
||||
};
|
||||
}
|
||||
|
||||
export function registerRecentProjectsHttp(
|
||||
app: FastifyInstance,
|
||||
feature: RecentProjectsFeatureFacade
|
||||
): void {
|
||||
app.get(DASHBOARD_RECENT_PROJECTS_ROUTE, async (): Promise<DashboardRecentProjectsPayload> => {
|
||||
const startedAt = Date.now();
|
||||
try {
|
||||
return (
|
||||
normalizeDashboardRecentProjectsPayload(await feature.listDashboardRecentProjects()) ?? {
|
||||
projects: [],
|
||||
degraded: true,
|
||||
}
|
||||
);
|
||||
const payload = normalizeDashboardRecentProjectsPayload(
|
||||
await feature.listDashboardRecentProjects()
|
||||
) ?? {
|
||||
projects: [],
|
||||
degraded: true,
|
||||
};
|
||||
logger.info('dashboard recent-projects HTTP loaded', {
|
||||
count: payload.projects.length,
|
||||
degraded: payload.degraded,
|
||||
durationMs: Date.now() - startedAt,
|
||||
payloadBytes: getPayloadBytes(payload),
|
||||
...getMemoryDiagnostics(),
|
||||
});
|
||||
return payload;
|
||||
} catch (error) {
|
||||
logger.error('Failed to load dashboard recent projects via HTTP', error);
|
||||
return { projects: [], degraded: true };
|
||||
|
|
|
|||
|
|
@ -9,18 +9,48 @@ import type { IpcMain } from 'electron';
|
|||
|
||||
const logger = createLogger('Feature:RecentProjects:IPC');
|
||||
|
||||
function getPayloadBytes(value: unknown): number {
|
||||
try {
|
||||
return Buffer.byteLength(JSON.stringify(value), 'utf8');
|
||||
} catch {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
function getMemoryDiagnostics(): {
|
||||
rssBytes: number;
|
||||
heapUsedBytes: number;
|
||||
heapTotalBytes: number;
|
||||
} {
|
||||
const memory = process.memoryUsage();
|
||||
return {
|
||||
rssBytes: memory.rss,
|
||||
heapUsedBytes: memory.heapUsed,
|
||||
heapTotalBytes: memory.heapTotal,
|
||||
};
|
||||
}
|
||||
|
||||
export function registerRecentProjectsIpc(
|
||||
ipcMain: IpcMain,
|
||||
feature: RecentProjectsFeatureFacade
|
||||
): void {
|
||||
ipcMain.handle(GET_DASHBOARD_RECENT_PROJECTS, async () => {
|
||||
const startedAt = Date.now();
|
||||
try {
|
||||
return (
|
||||
normalizeDashboardRecentProjectsPayload(await feature.listDashboardRecentProjects()) ?? {
|
||||
projects: [],
|
||||
degraded: true,
|
||||
}
|
||||
);
|
||||
const payload = normalizeDashboardRecentProjectsPayload(
|
||||
await feature.listDashboardRecentProjects()
|
||||
) ?? {
|
||||
projects: [],
|
||||
degraded: true,
|
||||
};
|
||||
logger.info('dashboard recent-projects IPC loaded', {
|
||||
count: payload.projects.length,
|
||||
degraded: payload.degraded,
|
||||
durationMs: Date.now() - startedAt,
|
||||
payloadBytes: getPayloadBytes(payload),
|
||||
...getMemoryDiagnostics(),
|
||||
});
|
||||
return payload;
|
||||
} catch (error) {
|
||||
logger.error('Failed to load dashboard recent projects via IPC', error);
|
||||
return { projects: [], degraded: true };
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ const CODEX_SESSION_FILE_SOFT_BUDGET_MS = 6_500;
|
|||
const CODEX_SESSION_FILE_MAX_UNCACHED_READS_PER_RUN = 160;
|
||||
const CODEX_SESSION_FILE_READ_BATCH_SIZE = 24;
|
||||
const CODEX_SESSION_FILE_READ_TIMEOUT_MS = 700;
|
||||
const CODEX_SESSION_FILE_DISCOVERY_STAT_BATCH_SIZE = 64;
|
||||
const CODEX_SESSION_METADATA_READ_LIMIT_BYTES = 128 * 1024;
|
||||
const CODEX_SESSION_FILE_CACHE_SCHEMA_VERSION = 1;
|
||||
const CODEX_SESSION_FILE_CACHE_RELATIVE_PATH = path.join(
|
||||
|
|
@ -80,6 +81,11 @@ interface CodexSessionSnapshotLoadResult {
|
|||
degraded: boolean;
|
||||
stats: {
|
||||
files: number;
|
||||
visitedFiles: number;
|
||||
droppedOlderFiles: number;
|
||||
statFailures: number;
|
||||
directoriesVisited: number;
|
||||
discoveryTimedOut: boolean;
|
||||
cached: number;
|
||||
uncachedReads: number;
|
||||
timedOutReads: number;
|
||||
|
|
@ -88,6 +94,14 @@ interface CodexSessionSnapshotLoadResult {
|
|||
};
|
||||
}
|
||||
|
||||
interface CodexSessionFileListingResult {
|
||||
files: CodexSessionFileEntry[];
|
||||
visitedFiles: number;
|
||||
statFailures: number;
|
||||
directoriesVisited: number;
|
||||
timedOut: boolean;
|
||||
}
|
||||
|
||||
function emptyCache(): CodexSessionFileCacheFile {
|
||||
return {
|
||||
schemaVersion: CODEX_SESSION_FILE_CACHE_SCHEMA_VERSION,
|
||||
|
|
@ -95,6 +109,21 @@ function emptyCache(): CodexSessionFileCacheFile {
|
|||
};
|
||||
}
|
||||
|
||||
function captureMemoryDiagnostics(): {
|
||||
rssBytes: number;
|
||||
heapUsedBytes: number;
|
||||
heapTotalBytes: number;
|
||||
externalBytes: number;
|
||||
} {
|
||||
const memory = process.memoryUsage();
|
||||
return {
|
||||
rssBytes: memory.rss,
|
||||
heapUsedBytes: memory.heapUsed,
|
||||
heapTotalBytes: memory.heapTotal,
|
||||
externalBytes: memory.external,
|
||||
};
|
||||
}
|
||||
|
||||
function isUsableCacheEntry(
|
||||
entry: CodexSessionFileCacheEntry | undefined,
|
||||
file: CodexSessionFileEntry
|
||||
|
|
@ -211,45 +240,149 @@ async function readFirstLineWithTimeout(
|
|||
return result;
|
||||
}
|
||||
|
||||
async function listJsonlFiles(root: string, maxDepth: number): Promise<CodexSessionFileEntry[]> {
|
||||
async function walk(directory: string, depth: number): Promise<CodexSessionFileEntry[]> {
|
||||
function insertRecentSessionFile(
|
||||
files: CodexSessionFileEntry[],
|
||||
file: CodexSessionFileEntry,
|
||||
limit: number
|
||||
): void {
|
||||
if (limit <= 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (files.length >= limit && file.mtimeMs <= files[files.length - 1].mtimeMs) {
|
||||
return;
|
||||
}
|
||||
|
||||
let low = 0;
|
||||
let high = files.length;
|
||||
while (low < high) {
|
||||
const mid = Math.floor((low + high) / 2);
|
||||
if (file.mtimeMs > files[mid].mtimeMs) {
|
||||
high = mid;
|
||||
} else {
|
||||
low = mid + 1;
|
||||
}
|
||||
}
|
||||
|
||||
files.splice(low, 0, file);
|
||||
if (files.length > limit) {
|
||||
files.pop();
|
||||
}
|
||||
}
|
||||
|
||||
function selectMostRecentSessionFiles(
|
||||
files: CodexSessionFileEntry[],
|
||||
limit: number
|
||||
): CodexSessionFileEntry[] {
|
||||
const selected: CodexSessionFileEntry[] = [];
|
||||
for (const file of files) {
|
||||
insertRecentSessionFile(selected, file, limit);
|
||||
}
|
||||
return selected;
|
||||
}
|
||||
|
||||
async function listRecentJsonlFiles(
|
||||
root: string,
|
||||
maxDepth: number,
|
||||
limit: number,
|
||||
deadlineMs: number
|
||||
): Promise<CodexSessionFileListingResult> {
|
||||
const selectedFiles: CodexSessionFileEntry[] = [];
|
||||
let visitedFiles = 0;
|
||||
let statFailures = 0;
|
||||
let directoriesVisited = 0;
|
||||
let timedOut = false;
|
||||
|
||||
const hasBudget = (): boolean => {
|
||||
if (Date.now() < deadlineMs) {
|
||||
return true;
|
||||
}
|
||||
timedOut = true;
|
||||
return false;
|
||||
};
|
||||
|
||||
async function statJsonlFile(filePath: string): Promise<CodexSessionFileEntry | null> {
|
||||
if (!hasBudget()) {
|
||||
return null;
|
||||
}
|
||||
visitedFiles += 1;
|
||||
try {
|
||||
const stats = await fs.stat(filePath);
|
||||
return {
|
||||
filePath,
|
||||
mtimeMs: stats.mtimeMs,
|
||||
size: stats.size,
|
||||
};
|
||||
} catch {
|
||||
statFailures += 1;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function collectFileStats(filePaths: string[]): Promise<void> {
|
||||
for (
|
||||
let offset = 0;
|
||||
offset < filePaths.length && hasBudget();
|
||||
offset += CODEX_SESSION_FILE_DISCOVERY_STAT_BATCH_SIZE
|
||||
) {
|
||||
const batch = filePaths.slice(offset, offset + CODEX_SESSION_FILE_DISCOVERY_STAT_BATCH_SIZE);
|
||||
const stats = await Promise.all(batch.map((filePath) => statJsonlFile(filePath)));
|
||||
for (const file of stats) {
|
||||
if (file) {
|
||||
insertRecentSessionFile(selectedFiles, file, limit);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function walk(directory: string, depth: number): Promise<void> {
|
||||
if (!hasBudget()) {
|
||||
return;
|
||||
}
|
||||
let entries;
|
||||
try {
|
||||
entries = await fs.readdir(directory, { withFileTypes: true, encoding: 'utf8' });
|
||||
} catch {
|
||||
return [];
|
||||
return;
|
||||
}
|
||||
|
||||
const files = await Promise.all(
|
||||
entries.map(async (entry): Promise<CodexSessionFileEntry[]> => {
|
||||
const entryPath = path.join(directory, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
return depth < maxDepth ? walk(entryPath, depth + 1) : [];
|
||||
}
|
||||
directoriesVisited += 1;
|
||||
const filePaths: string[] = [];
|
||||
const childDirectories: string[] = [];
|
||||
|
||||
if (!entry.isFile() || !entry.name.endsWith('.jsonl')) {
|
||||
return [];
|
||||
for (const entry of entries) {
|
||||
const entryPath = path.join(directory, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
if (depth < maxDepth) {
|
||||
childDirectories.push(entryPath);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const stats = await fs.stat(entryPath);
|
||||
return [
|
||||
{
|
||||
filePath: entryPath,
|
||||
mtimeMs: stats.mtimeMs,
|
||||
size: stats.size,
|
||||
},
|
||||
];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
})
|
||||
);
|
||||
if (entry.isFile() && entry.name.endsWith('.jsonl')) {
|
||||
filePaths.push(entryPath);
|
||||
}
|
||||
}
|
||||
|
||||
return files.flat();
|
||||
await collectFileStats(filePaths);
|
||||
|
||||
for (const childDirectory of childDirectories) {
|
||||
if (!hasBudget()) {
|
||||
return;
|
||||
}
|
||||
await walk(childDirectory, depth + 1);
|
||||
}
|
||||
}
|
||||
|
||||
return walk(root, 0);
|
||||
await walk(root, 0);
|
||||
|
||||
return {
|
||||
files: selectedFiles,
|
||||
visitedFiles,
|
||||
statFailures,
|
||||
directoriesVisited,
|
||||
timedOut,
|
||||
};
|
||||
}
|
||||
|
||||
function parseSessionSnapshot(
|
||||
|
|
@ -294,6 +427,7 @@ export class CodexSessionFileRecentProjectsSourceAdapter implements RecentProjec
|
|||
readonly timeoutMs = CODEX_SESSION_FILE_SOURCE_TIMEOUT_MS;
|
||||
readonly #codexHome: string;
|
||||
readonly #cachePath: string;
|
||||
#inFlightList: Promise<RecentProjectsSourceResult> | null = null;
|
||||
|
||||
constructor(
|
||||
private readonly deps: {
|
||||
|
|
@ -313,6 +447,20 @@ export class CodexSessionFileRecentProjectsSourceAdapter implements RecentProjec
|
|||
}
|
||||
|
||||
async list(): Promise<RecentProjectsSourceResult> {
|
||||
if (this.#inFlightList) {
|
||||
return this.#inFlightList;
|
||||
}
|
||||
|
||||
const request = this.#listUncached().finally(() => {
|
||||
if (this.#inFlightList === request) {
|
||||
this.#inFlightList = null;
|
||||
}
|
||||
});
|
||||
this.#inFlightList = request;
|
||||
return request;
|
||||
}
|
||||
|
||||
async #listUncached(): Promise<RecentProjectsSourceResult> {
|
||||
const activeContext = this.deps.getActiveContext();
|
||||
const localContext = this.deps.getLocalContext();
|
||||
|
||||
|
|
@ -339,6 +487,7 @@ export class CodexSessionFileRecentProjectsSourceAdapter implements RecentProjec
|
|||
count: validCandidates.length,
|
||||
codexHome: this.#codexHome,
|
||||
degraded: snapshotResult.degraded,
|
||||
...captureMemoryDiagnostics(),
|
||||
...snapshotResult.stats,
|
||||
});
|
||||
|
||||
|
|
@ -361,16 +510,34 @@ export class CodexSessionFileRecentProjectsSourceAdapter implements RecentProjec
|
|||
async #listRecentSessionSnapshots(): Promise<CodexSessionSnapshotLoadResult> {
|
||||
const startedAt = Date.now();
|
||||
const deadline = startedAt + CODEX_SESSION_FILE_SOFT_BUDGET_MS;
|
||||
const files = [
|
||||
...(await listJsonlFiles(path.join(this.#codexHome, 'sessions'), 4)),
|
||||
...(await listJsonlFiles(path.join(this.#codexHome, 'archived_sessions'), 1)),
|
||||
].sort((left, right) => right.mtimeMs - left.mtimeMs);
|
||||
const sessionFiles = await listRecentJsonlFiles(
|
||||
path.join(this.#codexHome, 'sessions'),
|
||||
4,
|
||||
CODEX_SESSION_FILE_PARSE_LIMIT,
|
||||
deadline
|
||||
);
|
||||
const archivedSessionFiles = await listRecentJsonlFiles(
|
||||
path.join(this.#codexHome, 'archived_sessions'),
|
||||
1,
|
||||
CODEX_SESSION_FILE_PARSE_LIMIT,
|
||||
deadline
|
||||
);
|
||||
const files = selectMostRecentSessionFiles(
|
||||
[...sessionFiles.files, ...archivedSessionFiles.files],
|
||||
CODEX_SESSION_FILE_PARSE_LIMIT
|
||||
);
|
||||
const visitedFiles = sessionFiles.visitedFiles + archivedSessionFiles.visitedFiles;
|
||||
const statFailures = sessionFiles.statFailures + archivedSessionFiles.statFailures;
|
||||
const directoriesVisited =
|
||||
sessionFiles.directoriesVisited + archivedSessionFiles.directoriesVisited;
|
||||
const droppedOlderFiles = Math.max(0, visitedFiles - statFailures - files.length);
|
||||
const discoveryTimedOut = sessionFiles.timedOut || archivedSessionFiles.timedOut;
|
||||
|
||||
const snapshotsByCwd = new Map<string, CodexSessionProjectSnapshot>();
|
||||
const candidateFiles = files.slice(0, CODEX_SESSION_FILE_PARSE_LIMIT);
|
||||
const candidateFiles = files;
|
||||
const cache = await this.#readCacheSafe();
|
||||
const nextCacheEntries = new Map<string, CodexSessionFileCacheEntry>();
|
||||
let degraded = false;
|
||||
let degraded = discoveryTimedOut;
|
||||
let cached = 0;
|
||||
let uncachedReads = 0;
|
||||
let timedOutReads = 0;
|
||||
|
|
@ -454,6 +621,12 @@ export class CodexSessionFileRecentProjectsSourceAdapter implements RecentProjec
|
|||
if (degraded) {
|
||||
this.deps.logger.warn('codex session-file recent-projects source partial', {
|
||||
files: candidateFiles.length,
|
||||
visitedFiles,
|
||||
droppedOlderFiles,
|
||||
statFailures,
|
||||
directoriesVisited,
|
||||
discoveryTimedOut,
|
||||
...captureMemoryDiagnostics(),
|
||||
cached,
|
||||
uncachedReads,
|
||||
timedOutReads,
|
||||
|
|
@ -468,6 +641,11 @@ export class CodexSessionFileRecentProjectsSourceAdapter implements RecentProjec
|
|||
degraded,
|
||||
stats: {
|
||||
files: candidateFiles.length,
|
||||
visitedFiles,
|
||||
droppedOlderFiles,
|
||||
statFailures,
|
||||
directoriesVisited,
|
||||
discoveryTimedOut,
|
||||
cached,
|
||||
uncachedReads,
|
||||
timedOutReads,
|
||||
|
|
|
|||
|
|
@ -57,6 +57,20 @@ async function writeRollout(
|
|||
await fs.utimes(filePath, mtime, mtime);
|
||||
}
|
||||
|
||||
function deferred<T>(): {
|
||||
promise: Promise<T>;
|
||||
resolve: (value: T) => void;
|
||||
reject: (error: unknown) => void;
|
||||
} {
|
||||
let resolve!: (value: T) => void;
|
||||
let reject!: (error: unknown) => void;
|
||||
const promise = new Promise<T>((promiseResolve, promiseReject) => {
|
||||
resolve = promiseResolve;
|
||||
reject = promiseReject;
|
||||
});
|
||||
return { promise, resolve, reject };
|
||||
}
|
||||
|
||||
describe('CodexSessionFileRecentProjectsSourceAdapter', () => {
|
||||
let tempDir: string;
|
||||
|
||||
|
|
@ -338,6 +352,43 @@ describe('CodexSessionFileRecentProjectsSourceAdapter', () => {
|
|||
expect(openSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('coalesces concurrent Codex session-file source reads', async () => {
|
||||
const codexHome = path.join(tempDir, '.codex');
|
||||
const logger = createLogger();
|
||||
const resolveResult = deferred<null>();
|
||||
const identityResolver = {
|
||||
resolve: vi.fn().mockReturnValue(resolveResult.promise),
|
||||
} as unknown as RecentProjectIdentityResolver;
|
||||
await writeRollout(
|
||||
path.join(codexHome, 'sessions', '2026', '04', '14', 'rollout-alpha.jsonl'),
|
||||
{
|
||||
cwd: '/Users/test/projects/alpha',
|
||||
branch: 'main',
|
||||
},
|
||||
new Date('2026-04-14T12:00:00.000Z')
|
||||
);
|
||||
|
||||
const adapter = new CodexSessionFileRecentProjectsSourceAdapter({
|
||||
getActiveContext: () => ({ type: 'local', id: 'local-1' }) as never,
|
||||
getLocalContext: () => ({ type: 'local', id: 'local-1' }) as never,
|
||||
identityResolver,
|
||||
logger,
|
||||
codexHome,
|
||||
appDataPath: path.join(tempDir, 'app-data'),
|
||||
});
|
||||
|
||||
const first = adapter.list();
|
||||
await vi.waitFor(() => expect(identityResolver.resolve).toHaveBeenCalledTimes(1));
|
||||
const second = adapter.list();
|
||||
|
||||
resolveResult.resolve(null);
|
||||
await expect(Promise.all([first, second])).resolves.toEqual([
|
||||
expect.objectContaining({ degraded: false }),
|
||||
expect.objectContaining({ degraded: false }),
|
||||
]);
|
||||
expect(identityResolver.resolve).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('invalidates cached session metadata when the jsonl fingerprint changes', async () => {
|
||||
const codexHome = path.join(tempDir, '.codex');
|
||||
const appDataPath = path.join(tempDir, 'app-data');
|
||||
|
|
@ -575,6 +626,54 @@ describe('CodexSessionFileRecentProjectsSourceAdapter', () => {
|
|||
]);
|
||||
});
|
||||
|
||||
it('bounds discovered Codex session files before reading metadata', async () => {
|
||||
const codexHome = path.join(tempDir, '.codex');
|
||||
const appDataPath = path.join(tempDir, 'app-data');
|
||||
const logger = createLogger();
|
||||
const identityResolver = {
|
||||
resolve: vi.fn().mockResolvedValue(null),
|
||||
} as unknown as RecentProjectIdentityResolver;
|
||||
const baseTime = Date.parse('2026-04-14T12:00:00.000Z');
|
||||
|
||||
await Promise.all(
|
||||
Array.from({ length: 505 }).map((_, index) =>
|
||||
writeRollout(
|
||||
path.join(codexHome, 'sessions', '2026', '04', '14', `rollout-alpha-${index}.jsonl`),
|
||||
{
|
||||
cwd: '/Users/test/projects/alpha',
|
||||
branch: 'main',
|
||||
},
|
||||
new Date(baseTime - index * 1000)
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
const adapter = new CodexSessionFileRecentProjectsSourceAdapter({
|
||||
getActiveContext: () => ({ type: 'local', id: 'local-1' }) as never,
|
||||
getLocalContext: () => ({ type: 'local', id: 'local-1' }) as never,
|
||||
identityResolver,
|
||||
logger,
|
||||
codexHome,
|
||||
appDataPath,
|
||||
});
|
||||
const result = await adapter.list();
|
||||
|
||||
expect(result.degraded).toBe(true);
|
||||
expect(result.candidates.map((candidate) => candidate.primaryPath)).toEqual([
|
||||
'/Users/test/projects/alpha',
|
||||
]);
|
||||
expect(logger.warn).toHaveBeenCalledWith(
|
||||
'codex session-file recent-projects source partial',
|
||||
expect.objectContaining({
|
||||
files: 500,
|
||||
visitedFiles: 505,
|
||||
droppedOlderFiles: 5,
|
||||
uncachedReads: 160,
|
||||
skippedUncached: 340,
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it('skips non-interactive and ephemeral sessions', async () => {
|
||||
const codexHome = path.join(tempDir, '.codex');
|
||||
const logger = createLogger();
|
||||
|
|
|
|||
Loading…
Reference in a new issue