perf(team): defer task log stream work until opened
This commit is contained in:
parent
ba06fba5a5
commit
11d5851953
6 changed files with 131 additions and 12 deletions
|
|
@ -1,10 +1,16 @@
|
||||||
import * as fs from 'fs/promises';
|
import * as fs from 'fs/promises';
|
||||||
import * as path from 'path';
|
import * as path from 'path';
|
||||||
|
|
||||||
|
import { createLogger } from '@shared/utils/logger';
|
||||||
|
|
||||||
import { TeamTranscriptProjectResolver } from '../../TeamTranscriptProjectResolver';
|
import { TeamTranscriptProjectResolver } from '../../TeamTranscriptProjectResolver';
|
||||||
|
|
||||||
import type { TeamConfig } from '@shared/types';
|
import type { TeamConfig } from '@shared/types';
|
||||||
|
|
||||||
|
const logger = createLogger('Service:TeamTranscriptSourceLocator');
|
||||||
|
const TRANSCRIPT_DISCOVERY_WARN_MS = 3_000;
|
||||||
|
const TRANSCRIPT_DISCOVERY_FILE_COUNT_WARN = 500;
|
||||||
|
|
||||||
export interface TeamTranscriptSourceContext {
|
export interface TeamTranscriptSourceContext {
|
||||||
projectDir: string;
|
projectDir: string;
|
||||||
projectId: string;
|
projectId: string;
|
||||||
|
|
@ -25,7 +31,17 @@ export class TeamTranscriptSourceLocator {
|
||||||
}
|
}
|
||||||
|
|
||||||
const { projectDir, projectId, config, sessionIds } = context;
|
const { projectDir, projectId, config, sessionIds } = context;
|
||||||
|
const startedAt = Date.now();
|
||||||
const transcriptFiles = await this.listTranscriptFilesForSessions(projectDir, sessionIds);
|
const transcriptFiles = await this.listTranscriptFilesForSessions(projectDir, sessionIds);
|
||||||
|
const elapsedMs = Date.now() - startedAt;
|
||||||
|
if (
|
||||||
|
elapsedMs >= TRANSCRIPT_DISCOVERY_WARN_MS ||
|
||||||
|
transcriptFiles.length >= TRANSCRIPT_DISCOVERY_FILE_COUNT_WARN
|
||||||
|
) {
|
||||||
|
logger.warn(
|
||||||
|
`Large task-log transcript discovery: team=${teamName} sessions=${sessionIds.length} files=${transcriptFiles.length} elapsedMs=${elapsedMs}`
|
||||||
|
);
|
||||||
|
}
|
||||||
return { projectDir, projectId, config, sessionIds, transcriptFiles };
|
return { projectDir, projectId, config, sessionIds, transcriptFiles };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -2,12 +2,38 @@ import * as fs from 'fs/promises';
|
||||||
|
|
||||||
import type { BoardTaskExactLogFileVersion } from './BoardTaskExactLogTypes';
|
import type { BoardTaskExactLogFileVersion } from './BoardTaskExactLogTypes';
|
||||||
|
|
||||||
|
const FILE_VERSION_STAT_CONCURRENCY = process.platform === 'win32' ? 8 : 16;
|
||||||
|
|
||||||
|
async function mapLimit<T, R>(
|
||||||
|
items: readonly T[],
|
||||||
|
limit: number,
|
||||||
|
fn: (item: T) => Promise<R>
|
||||||
|
): Promise<R[]> {
|
||||||
|
const results = new Array<R>(items.length);
|
||||||
|
let index = 0;
|
||||||
|
const workerCount = Math.max(1, Math.min(limit, items.length));
|
||||||
|
const workers = new Array(workerCount).fill(0).map(async () => {
|
||||||
|
while (true) {
|
||||||
|
const currentIndex = index;
|
||||||
|
index += 1;
|
||||||
|
if (currentIndex >= items.length) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
results[currentIndex] = await fn(items[currentIndex]!);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
await Promise.all(workers);
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
||||||
export async function getBoardTaskExactLogFileVersions(
|
export async function getBoardTaskExactLogFileVersions(
|
||||||
filePaths: Iterable<string>
|
filePaths: Iterable<string>
|
||||||
): Promise<Map<string, BoardTaskExactLogFileVersion>> {
|
): Promise<Map<string, BoardTaskExactLogFileVersion>> {
|
||||||
const uniqueFilePaths = [...new Set(filePaths)];
|
const uniqueFilePaths = [...new Set(filePaths)];
|
||||||
const results = await Promise.all(
|
const results = await mapLimit(
|
||||||
uniqueFilePaths.map(async (filePath) => {
|
uniqueFilePaths,
|
||||||
|
FILE_VERSION_STAT_CONCURRENCY,
|
||||||
|
async (filePath) => {
|
||||||
try {
|
try {
|
||||||
const stat = await fs.stat(filePath);
|
const stat = await fs.stat(filePath);
|
||||||
if (!stat.isFile()) {
|
if (!stat.isFile()) {
|
||||||
|
|
@ -21,7 +47,7 @@ export async function getBoardTaskExactLogFileVersions(
|
||||||
} catch {
|
} catch {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
})
|
}
|
||||||
);
|
);
|
||||||
|
|
||||||
const byPath = new Map<string, BoardTaskExactLogFileVersion>();
|
const byPath = new Map<string, BoardTaskExactLogFileVersion>();
|
||||||
|
|
|
||||||
|
|
@ -66,6 +66,7 @@ const INFERRED_RECORD_RANGE_BEFORE_MS = 5 * 60_000;
|
||||||
const INFERRED_RECORD_RANGE_AFTER_MS = 60_000;
|
const INFERRED_RECORD_RANGE_AFTER_MS = 60_000;
|
||||||
const STREAM_LAYOUT_CACHE_TTL_MS = 1_000;
|
const STREAM_LAYOUT_CACHE_TTL_MS = 1_000;
|
||||||
const STREAM_LAYOUT_BUILD_WARN_MS = 3_000;
|
const STREAM_LAYOUT_BUILD_WARN_MS = 3_000;
|
||||||
|
const RUNTIME_FALLBACK_WARN_MS = 3_000;
|
||||||
const HISTORICAL_BOARD_LIFECYCLE_TOOL_NAMES = new Set([
|
const HISTORICAL_BOARD_LIFECYCLE_TOOL_NAMES = new Set([
|
||||||
'task_complete',
|
'task_complete',
|
||||||
'task_set_status',
|
'task_set_status',
|
||||||
|
|
@ -1925,10 +1926,18 @@ export class BoardTaskLogStreamService {
|
||||||
|
|
||||||
const layout = await this.getStreamLayout(teamName, taskId);
|
const layout = await this.getStreamLayout(teamName, taskId);
|
||||||
if (layout.visibleSlices.length === 0) {
|
if (layout.visibleSlices.length === 0) {
|
||||||
return (
|
const startedAt = Date.now();
|
||||||
(await this.runtimeFallbackSource.getTaskLogStream(teamName, taskId)) ?? emptyResponse()
|
const fallback = await this.runtimeFallbackSource.getTaskLogStream(teamName, taskId);
|
||||||
|
const elapsedMs = Date.now() - startedAt;
|
||||||
|
if (elapsedMs >= RUNTIME_FALLBACK_WARN_MS) {
|
||||||
|
logger.warn(
|
||||||
|
`Slow OpenCode task-log runtime fallback: team=${teamName} task=${taskId} hit=${Boolean(
|
||||||
|
fallback
|
||||||
|
)} elapsedMs=${elapsedMs}`
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
return fallback ?? emptyResponse();
|
||||||
|
}
|
||||||
|
|
||||||
const segments: BoardTaskLogSegment[] = [];
|
const segments: BoardTaskLogSegment[] = [];
|
||||||
let currentSegmentSlices: StreamSlice[] = [];
|
let currentSegmentSlices: StreamSlice[] = [];
|
||||||
|
|
|
||||||
|
|
@ -64,8 +64,9 @@ export const TaskLogsPanel = ({
|
||||||
const pulseTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
const pulseTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||||
const countReloadTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
const countReloadTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||||
const countRequestSeqRef = useRef(0);
|
const countRequestSeqRef = useRef(0);
|
||||||
const taskLogTrackingEnabled = task.status === 'in_progress' && availableTabs.includes('stream');
|
const taskLogTrackingEnabled =
|
||||||
const taskLogSummaryEnabled = availableTabs.includes('stream');
|
hasOpenedContent && task.status === 'in_progress' && availableTabs.includes('stream');
|
||||||
|
const taskLogSummaryEnabled = hasOpenedContent && availableTabs.includes('stream');
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
setActiveTab(defaultTab);
|
setActiveTab(defaultTab);
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,47 @@
|
||||||
|
import { describe, expect, it, vi } from 'vitest';
|
||||||
|
|
||||||
|
const statMock = vi.hoisted(() => vi.fn());
|
||||||
|
|
||||||
|
vi.mock('fs/promises', () => ({
|
||||||
|
stat: statMock,
|
||||||
|
}));
|
||||||
|
|
||||||
|
import { getBoardTaskExactLogFileVersions } from '../../../../src/main/services/team/taskLogs/exact/fileVersions';
|
||||||
|
|
||||||
|
describe('getBoardTaskExactLogFileVersions', () => {
|
||||||
|
it('deduplicates paths and bounds concurrent stat calls', async () => {
|
||||||
|
let activeStats = 0;
|
||||||
|
let maxActiveStats = 0;
|
||||||
|
const uniquePaths = Array.from({ length: 40 }, (_value, index) => `/tmp/task-${index}.jsonl`);
|
||||||
|
|
||||||
|
statMock.mockImplementation(async (filePath: string) => {
|
||||||
|
activeStats += 1;
|
||||||
|
maxActiveStats = Math.max(maxActiveStats, activeStats);
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 1));
|
||||||
|
activeStats -= 1;
|
||||||
|
|
||||||
|
return {
|
||||||
|
isFile: () => !filePath.endsWith('17.jsonl'),
|
||||||
|
mtimeMs: 1000 + uniquePaths.indexOf(filePath),
|
||||||
|
size: 2000 + uniquePaths.indexOf(filePath),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await getBoardTaskExactLogFileVersions([
|
||||||
|
...uniquePaths,
|
||||||
|
uniquePaths[0]!,
|
||||||
|
uniquePaths[1]!,
|
||||||
|
]);
|
||||||
|
|
||||||
|
expect(statMock).toHaveBeenCalledTimes(uniquePaths.length);
|
||||||
|
expect(maxActiveStats).toBeGreaterThan(1);
|
||||||
|
expect(maxActiveStats).toBeLessThanOrEqual(16);
|
||||||
|
expect(result.size).toBe(uniquePaths.length - 1);
|
||||||
|
expect(result.has('/tmp/task-17.jsonl')).toBe(false);
|
||||||
|
expect(result.get('/tmp/task-0.jsonl')).toEqual({
|
||||||
|
filePath: '/tmp/task-0.jsonl',
|
||||||
|
mtimeMs: 1000,
|
||||||
|
size: 2000,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
@ -370,7 +370,7 @@ describe('TaskLogsPanel', () => {
|
||||||
expect(apiState.setTaskLogStreamTracking).toHaveBeenLastCalledWith('demo', false);
|
expect(apiState.setTaskLogStreamTracking).toHaveBeenLastCalledWith('demo', false);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('does not mount Task Log Stream content while the section is collapsed but still pulses on matching updates', async () => {
|
it('defers Task Log Stream work while collapsed, then starts tracking after first open', async () => {
|
||||||
vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true);
|
vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true);
|
||||||
vi.useFakeTimers();
|
vi.useFakeTimers();
|
||||||
|
|
||||||
|
|
@ -401,23 +401,43 @@ describe('TaskLogsPanel', () => {
|
||||||
|
|
||||||
expect(host.querySelector('[data-testid="task-log-stream"]')).toBeNull();
|
expect(host.querySelector('[data-testid="task-log-stream"]')).toBeNull();
|
||||||
expect(taskLogStreamProps.calls).toHaveLength(0);
|
expect(taskLogStreamProps.calls).toHaveLength(0);
|
||||||
|
expect(apiState.getTaskLogStreamSummary).not.toHaveBeenCalled();
|
||||||
|
expect(apiState.setTaskLogStreamTracking).not.toHaveBeenCalled();
|
||||||
|
expect(apiState.onTeamChange).not.toHaveBeenCalled();
|
||||||
|
expect(handler).toBeNull();
|
||||||
|
expect(activityStates).toEqual([false]);
|
||||||
|
|
||||||
|
await act(async () => {
|
||||||
|
root.render(
|
||||||
|
React.createElement(TaskLogsPanel, {
|
||||||
|
teamName: 'demo',
|
||||||
|
task: makeTask(),
|
||||||
|
isOpen: true,
|
||||||
|
onTaskLogActivityChange: (isActive: boolean) => activityStates.push(isActive),
|
||||||
|
})
|
||||||
|
);
|
||||||
|
await flushMicrotasks();
|
||||||
|
await flushMicrotasks();
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(host.querySelector('[data-testid="task-log-stream"]')).not.toBeNull();
|
||||||
|
expect(apiState.getTaskLogStreamSummary).toHaveBeenCalledWith('demo', 'task-1');
|
||||||
expect(apiState.setTaskLogStreamTracking).toHaveBeenCalledWith('demo', true);
|
expect(apiState.setTaskLogStreamTracking).toHaveBeenCalledWith('demo', true);
|
||||||
expect(handler).toBeTypeOf('function');
|
expect(handler).toBeTypeOf('function');
|
||||||
expect(activityStates).toEqual([false]);
|
|
||||||
|
|
||||||
await act(async () => {
|
await act(async () => {
|
||||||
handler?.(null, { teamName: 'demo', type: 'task-log-change', taskId: 'task-1' });
|
handler?.(null, { teamName: 'demo', type: 'task-log-change', taskId: 'task-1' });
|
||||||
await flushMicrotasks();
|
await flushMicrotasks();
|
||||||
});
|
});
|
||||||
|
|
||||||
expect(activityStates).toEqual([false, true]);
|
expect(activityStates).toEqual([false, false, true]);
|
||||||
|
|
||||||
await act(async () => {
|
await act(async () => {
|
||||||
vi.advanceTimersByTime(1800);
|
vi.advanceTimersByTime(1800);
|
||||||
await flushMicrotasks();
|
await flushMicrotasks();
|
||||||
});
|
});
|
||||||
|
|
||||||
expect(activityStates).toEqual([false, true, false]);
|
expect(activityStates).toEqual([false, false, true, false]);
|
||||||
|
|
||||||
await act(async () => {
|
await act(async () => {
|
||||||
root.unmount();
|
root.unmount();
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue