import type { ChildProcess } from 'child_process'; import { EventEmitter } from 'events'; import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; const hoisted = vi.hoisted(() => ({ paths: { claudeRoot: '', teamsBase: '', tasksBase: '', projectsBase: '', }, })); let tempClaudeRoot = ''; let tempTeamsBase = ''; let tempTasksBase = ''; let tempProjectsBase = ''; vi.mock('@main/services/team/ClaudeBinaryResolver', () => ({ ClaudeBinaryResolver: { resolve: vi.fn() }, })); vi.mock('@features/tmux-installer/main', () => ({ killTmuxPaneForCurrentPlatformSync: vi.fn(), listTmuxPanePidsForCurrentPlatform: vi.fn(async () => new Map()), isTmuxRuntimeReadyForCurrentPlatform: vi.fn(async () => true), })); vi.mock('pidusage', () => { const pidusageMock = vi.fn(); return { default: pidusageMock, }; }); vi.mock('@main/services/team/TeamTaskReader', () => ({ TeamTaskReader: class { async getTasks() { return []; } }, })); vi.mock('@main/utils/childProcess', () => ({ execCli: vi.fn(async (_binaryPath: string | null, args: string[]) => { if (args[0] === 'model') { return { stdout: JSON.stringify({ schemaVersion: 1, providers: { anthropic: { defaultModel: 'opus[1m]', models: [ { id: 'opus', label: 'Opus 4.7', description: 'Anthropic default family alias' }, { id: 'opus[1m]', label: 'Opus 4.7 (1M)', description: 'Anthropic long-context default', }, ], }, codex: { defaultModel: 'gpt-5.4', models: [{ id: 'gpt-5.4', label: 'GPT-5.4', description: 'Codex default' }], }, gemini: { defaultModel: 'gemini-2.5-pro', models: [{ id: 'gemini-2.5-pro', label: 'Gemini 2.5 Pro', description: 'Default' }], }, }, }), stderr: '', }; } if (args[0] === 'runtime') { return { stdout: JSON.stringify({ providers: { codex: { runtimeCapabilities: { modelCatalog: { dynamic: false, source: 'runtime' }, reasoningEffort: { supported: true, values: ['low', 'medium', 'high'], configPassthrough: false, }, }, }, }, }), stderr: '', }; } return { stdout: '', stderr: '' }; }), spawnCli: vi.fn(), killProcessTree: vi.fn(), })); vi.mock('@main/utils/processKill', () => ({ killProcessByPid: vi.fn(), })); vi.mock('@main/utils/pathDecoder', async (importOriginal) => { const actual = await importOriginal(); return { ...actual, getAutoDetectedClaudeBasePath: () => hoisted.paths.claudeRoot, getClaudeBasePath: () => hoisted.paths.claudeRoot, getHomeDir: () => hoisted.paths.claudeRoot, getProjectsBasePath: () => hoisted.paths.projectsBase, getTasksBasePath: () => hoisted.paths.tasksBase, getTeamsBasePath: () => hoisted.paths.teamsBase, }; }); import { TeamProvisioningService } from '@main/services/team/TeamProvisioningService'; import { clearAutoResumeService, getAutoResumeService, initializeAutoResumeService, } from '@main/services/team/AutoResumeService'; import { getTeamBootstrapStatePath } from '@main/services/team/TeamBootstrapStateReader'; import { createPersistedLaunchSnapshot } from '@main/services/team/TeamLaunchStateEvaluator'; import { getTeamLaunchStatePath } from '@main/services/team/TeamLaunchStateStore'; import { ClaudeBinaryResolver } from '@main/services/team/ClaudeBinaryResolver'; import { spawnCli } from '@main/utils/childProcess'; import { killProcessByPid } from '@main/utils/processKill'; import { encodePath } from '@main/utils/pathDecoder'; import { AGENT_TEAMS_NAMESPACED_LEAD_BOOTSTRAP_TOOL_NAMES, AGENT_TEAMS_NAMESPACED_TEAMMATE_OPERATIONAL_TOOL_NAMES, } from 'agent-teams-controller'; import { killTmuxPaneForCurrentPlatformSync, listTmuxPanePidsForCurrentPlatform, } from '@features/tmux-installer/main'; import pidusage from 'pidusage'; function allowConsoleLogs() { vi.spyOn(console, 'error').mockImplementation(() => {}); vi.spyOn(console, 'warn').mockImplementation(() => {}); } function createFakeChild(exitCode: number): ChildProcess { const child = Object.assign(new EventEmitter(), { stdout: null, stderr: null, stdin: null, }) as unknown as ChildProcess; setImmediate(() => child.emit('close', exitCode)); return child; } function createRunningChild() { return Object.assign(new EventEmitter(), { pid: 12345, stdin: { writable: true, write: vi.fn(() => true), end: vi.fn(), }, stdout: new EventEmitter(), stderr: new EventEmitter(), kill: vi.fn(), }); } function createPidusageStat(pid: number, memory: number) { return { cpu: 0, memory, ppid: 1, pid, ctime: 0, elapsed: 0, timestamp: Date.now(), }; } function writeLaunchConfig( teamName: string, projectPath: string, leadSessionId: string, members: string[] ): void { const teamDir = path.join(tempTeamsBase, teamName); fs.mkdirSync(teamDir, { recursive: true }); fs.writeFileSync( path.join(teamDir, 'config.json'), JSON.stringify({ name: teamName, projectPath, leadSessionId, members: [ { name: 'team-lead', agentType: 'team-lead' }, ...members.map((name) => ({ name })), ], }), 'utf8' ); } function writeLaunchState( teamName: string, leadSessionId: string, members: Record> ): void { const snapshot = createPersistedLaunchSnapshot({ teamName, leadSessionId, launchPhase: 'finished', expectedMembers: Object.keys(members), members: Object.fromEntries( Object.entries(members).map(([name, member]) => [ name, { name, launchState: 'failed_to_start', agentToolAccepted: false, runtimeAlive: false, bootstrapConfirmed: false, hardFailure: true, hardFailureReason: 'Teammate was never spawned during launch.', lastEvaluatedAt: new Date().toISOString(), ...member, }, ]) ) as any, }); fs.writeFileSync( getTeamLaunchStatePath(teamName), `${JSON.stringify(snapshot, null, 2)}\n`, 'utf8' ); } function writeBootstrapState( teamName: string, members: { name: string; status: string; lastAttemptAt?: number; lastObservedAt?: number }[], updatedAt = new Date().toISOString() ): void { fs.writeFileSync( getTeamBootstrapStatePath(teamName), `${JSON.stringify( { version: 1, teamName, updatedAt, phase: 'completed', members, }, null, 2 )}\n`, 'utf8' ); } function createMemberSpawnStatusEntry( overrides: Record = {} ): Record { return { status: 'waiting', launchState: 'runtime_pending_bootstrap', error: undefined, updatedAt: new Date().toISOString(), runtimeAlive: false, livenessSource: undefined, bootstrapConfirmed: false, hardFailure: false, agentToolAccepted: true, firstSpawnAcceptedAt: new Date().toISOString(), lastHeartbeatAt: undefined, ...overrides, }; } function createMemberSpawnRun(params?: { runId?: string; teamName?: string; startedAt?: string; expectedMembers?: string[]; memberSpawnStatuses?: Map>; memberSpawnLeadInboxCursorByMember?: Map; }) { const teamName = params?.teamName ?? 'member-spawn-team'; const expectedMembers = params?.expectedMembers ?? ['alice']; const memberSpawnStatuses = params?.memberSpawnStatuses ?? new Map([ [ expectedMembers[0]!, createMemberSpawnStatusEntry({ firstSpawnAcceptedAt: new Date(Date.now() - 5_000).toISOString(), }), ], ]); return { runId: params?.runId ?? 'run-member-spawn-1', teamName, startedAt: params?.startedAt ?? new Date(Date.now() - 60_000).toISOString(), request: { members: [], }, expectedMembers, memberSpawnStatuses, memberSpawnToolUseIds: new Map(), pendingMemberRestarts: new Map(), memberSpawnLeadInboxCursorByMember: params?.memberSpawnLeadInboxCursorByMember ?? new Map(), provisioningOutputParts: [], activeToolCalls: new Map(), isLaunch: false, provisioningComplete: false, } as any; } function createClaudeLogsRun(overrides: Record = {}) { return { runId: 'run-logs-1', teamName: 'logs-team', startedAt: '2026-04-19T10:00:00.000Z', isLaunch: false, provisioningComplete: true, processKilled: false, cancelRequested: false, timeoutHandle: null, fsMonitorHandle: null, stallCheckHandle: null, silentUserDmForwardClearHandle: null, child: null, leadActivityState: 'idle', activeToolCalls: new Map(), pendingDirectCrossTeamSendRefresh: false, memberSpawnStatuses: new Map(), activeCrossTeamReplyHints: [], pendingInboxRelayCandidates: [], pendingApprovals: new Map(), mcpConfigPath: null, bootstrapSpecPath: null, bootstrapUserPromptPath: null, claudeLogLines: ['[stdout]', 'first line', '[stderr]', 'boom'], claudeLogsUpdatedAt: '2026-04-19T10:00:01.000Z', progress: { updatedAt: '2026-04-19T10:00:01.000Z', state: 'ready', }, ...overrides, } as any; } describe('TeamProvisioningService', () => { beforeEach(() => { vi.clearAllMocks(); tempClaudeRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'claude-team-provisioning-')); tempTeamsBase = path.join(tempClaudeRoot, 'teams'); tempTasksBase = path.join(tempClaudeRoot, 'tasks'); tempProjectsBase = path.join(tempClaudeRoot, 'projects'); hoisted.paths.claudeRoot = tempClaudeRoot; hoisted.paths.teamsBase = tempTeamsBase; hoisted.paths.tasksBase = tempTasksBase; hoisted.paths.projectsBase = tempProjectsBase; fs.mkdirSync(tempTeamsBase, { recursive: true }); fs.mkdirSync(tempTasksBase, { recursive: true }); fs.mkdirSync(tempProjectsBase, { recursive: true }); }); afterEach(() => { clearAutoResumeService(); vi.useRealTimers(); try { fs.rmSync(tempClaudeRoot, { recursive: true, force: true }); } catch { // ignore temp cleanup failures } hoisted.paths.claudeRoot = ''; hoisted.paths.teamsBase = ''; hoisted.paths.tasksBase = ''; hoisted.paths.projectsBase = ''; }); describe('warmup', () => { it('does not throw when spawnCli rejects', async () => { allowConsoleLogs(); vi.mocked(ClaudeBinaryResolver.resolve).mockResolvedValue('C:\\path\\claude'); let callCount = 0; vi.mocked(spawnCli).mockImplementation(() => { callCount++; if (callCount === 1) { throw new Error('spawn EINVAL'); } return createFakeChild(0); }); const svc = new TeamProvisioningService(); await expect(svc.warmup()).resolves.not.toThrow(); expect(spawnCli).toHaveBeenCalled(); }); }); describe('getClaudeLogs', () => { it('retains the last logs after cleanupRun removes the live run', async () => { const svc = new TeamProvisioningService(); const run = createClaudeLogsRun(); (svc as any).runs.set(run.runId, run); (svc as any).aliveRunByTeam.set(run.teamName, run.runId); await expect(svc.getClaudeLogs(run.teamName)).resolves.toEqual({ lines: ['boom', '[stderr]', 'first line', '[stdout]'], total: 4, hasMore: false, updatedAt: '2026-04-19T10:00:01.000Z', }); (svc as any).cleanupRun(run); await expect(svc.getClaudeLogs(run.teamName)).resolves.toEqual({ lines: ['boom', '[stderr]', 'first line', '[stdout]'], total: 4, hasMore: false, updatedAt: '2026-04-19T10:00:01.000Z', }); }); it('falls back to the persisted lead transcript when no live run exists', async () => { const svc = new TeamProvisioningService(); const teamName = 'offline-logs-team'; const projectPath = '/tmp/offline-logs-project'; const leadSessionId = 'lead-session-1'; const projectDir = path.join(tempProjectsBase, encodePath(projectPath)); writeLaunchConfig(teamName, projectPath, leadSessionId, []); fs.mkdirSync(projectDir, { recursive: true }); fs.writeFileSync( path.join(projectDir, `${leadSessionId}.jsonl`), [ '{"type":"user","message":{"role":"user","content":"first"}}', '{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"second"}]}}', '{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"third"}]}}', ].join('\n') + '\n', 'utf8' ); await expect(svc.getClaudeLogs(teamName)).resolves.toEqual({ lines: [ '{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"third"}]}}', '{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"second"}]}}', '{"type":"user","message":{"role":"user","content":"first"}}', ], total: 3, hasMore: false, updatedAt: expect.any(String), }); }); it('clears retained logs when a new run starts for the same team', async () => { const svc = new TeamProvisioningService(); (svc as any).retainedClaudeLogsByTeam.set('logs-team', { lines: ['[stdout]', 'stale line'], updatedAt: '2026-04-19T10:00:01.000Z', }); (svc as any).resetTeamScopedTransientStateForNewRun('logs-team'); await expect(svc.getClaudeLogs('logs-team')).resolves.toEqual({ lines: [], total: 0, hasMore: false, }); }); }); describe('getTeamAgentRuntimeSnapshot', () => { it('uses batched pidusage rss values for lead and teammates', async () => { const svc = new TeamProvisioningService(); (svc as any).configReader = { getConfig: vi.fn(async () => ({ members: [ { name: 'team-lead', agentType: 'team-lead' }, { name: 'alice', model: 'gpt-5.4-mini' }, ], })), }; (svc as any).readPersistedRuntimeMembers = vi.fn(() => [ { name: 'alice', agentId: 'alice@runtime-team', tmuxPaneId: '%1', backendType: 'tmux', }, ]); (svc as any).aliveRunByTeam.set('runtime-team', 'run-1'); (svc as any).runs.set('run-1', { runId: 'run-1', child: { pid: 111 }, request: { model: 'gpt-5.4' }, processKilled: false, cancelRequested: false, spawnContext: null, }); vi.mocked(listTmuxPanePidsForCurrentPlatform).mockResolvedValueOnce(new Map([['%1', 222]])); vi.mocked(pidusage).mockResolvedValueOnce({ '111': createPidusageStat(111, 123_000_000), '222': createPidusageStat(222, 456_000_000), } as any); const snapshot = await svc.getTeamAgentRuntimeSnapshot('runtime-team'); expect(pidusage).toHaveBeenCalledWith([111, 222], { maxage: 0 }); expect(snapshot.members['team-lead']).toMatchObject({ pid: 111, rssBytes: 123_000_000, runtimeModel: 'gpt-5.4', }); expect(snapshot.members.alice).toMatchObject({ pid: 222, rssBytes: 456_000_000, runtimeModel: 'gpt-5.4-mini', }); }); it('exposes providerBackendId from the live run request when available', async () => { const svc = new TeamProvisioningService(); (svc as any).configReader = { getConfig: vi.fn(async () => ({ members: [{ name: 'team-lead', agentType: 'team-lead' }], })), }; (svc as any).teamMetaStore = { getMeta: vi.fn(async () => ({ providerBackendId: 'adapter' })), }; (svc as any).aliveRunByTeam.set('runtime-team', 'run-1'); (svc as any).runs.set('run-1', { runId: 'run-1', child: { pid: 111 }, request: { model: 'gpt-5.4', providerBackendId: 'codex-native' }, processKilled: false, cancelRequested: false, spawnContext: null, }); vi.mocked(pidusage).mockResolvedValueOnce({ '111': createPidusageStat(111, 123_000_000), } as any); const snapshot = await svc.getTeamAgentRuntimeSnapshot('runtime-team'); expect(snapshot.providerBackendId).toBe('codex-native'); }); it('falls back to persisted team meta backend when no live run exists', async () => { const svc = new TeamProvisioningService(); (svc as any).configReader = { getConfig: vi.fn(async () => ({ members: [{ name: 'team-lead', agentType: 'team-lead' }], })), }; (svc as any).teamMetaStore = { getMeta: vi.fn(async () => ({ providerBackendId: 'codex-native' })), }; const snapshot = await svc.getTeamAgentRuntimeSnapshot('runtime-team'); expect(snapshot.providerBackendId).toBe('codex-native'); }); it('falls back to per-pid pidusage reads when batched sampling fails', async () => { const svc = new TeamProvisioningService(); (svc as any).configReader = { getConfig: vi.fn(async () => ({ members: [ { name: 'team-lead', agentType: 'team-lead' }, { name: 'alice', model: 'gpt-5.4-mini' }, ], })), }; (svc as any).readPersistedRuntimeMembers = vi.fn(() => [ { name: 'alice', agentId: 'alice@runtime-team', tmuxPaneId: '%1', backendType: 'tmux', }, ]); (svc as any).aliveRunByTeam.set('runtime-team', 'run-1'); (svc as any).runs.set('run-1', { runId: 'run-1', child: { pid: 111 }, request: { model: 'gpt-5.4' }, processKilled: false, cancelRequested: false, spawnContext: null, }); vi.mocked(listTmuxPanePidsForCurrentPlatform).mockResolvedValueOnce(new Map([['%1', 222]])); vi.mocked(pidusage) .mockRejectedValueOnce(new Error('ps: process exited')) .mockResolvedValueOnce(createPidusageStat(111, 123_000_000) as any) .mockResolvedValueOnce(createPidusageStat(222, 456_000_000) as any); const snapshot = await svc.getTeamAgentRuntimeSnapshot('runtime-team'); expect(pidusage).toHaveBeenNthCalledWith(1, [111, 222], { maxage: 0 }); expect(pidusage).toHaveBeenNthCalledWith(2, 111, { maxage: 0 }); expect(pidusage).toHaveBeenNthCalledWith(3, 222, { maxage: 0 }); expect(snapshot.members['team-lead']?.rssBytes).toBe(123_000_000); expect(snapshot.members.alice?.rssBytes).toBe(456_000_000); }); it('falls back to direct agent process lookup when tmux pane pid lookup is unavailable', async () => { const svc = new TeamProvisioningService(); (svc as any).configReader = { getConfig: vi.fn(async () => ({ members: [ { name: 'team-lead', agentType: 'team-lead' }, { name: 'alice', model: 'gpt-5.2' }, ], })), }; (svc as any).readPersistedRuntimeMembers = vi.fn(() => [ { name: 'alice', agentId: 'alice@nice-team', tmuxPaneId: '%0', backendType: 'tmux', }, ]); (svc as any).aliveRunByTeam.set('nice-team', 'run-1'); (svc as any).runs.set('run-1', { runId: 'run-1', child: { pid: 111 }, request: { model: 'gpt-5.4' }, processKilled: false, cancelRequested: false, spawnContext: null, }); (svc as any).readUnixProcessTableRows = vi.fn(() => [ { pid: 333, command: '/Users/belief/.bun/bin/bun cli.js --agent-id alice@nice-team --agent-name alice --team-name nice-team --model gpt-5.2', }, ]); vi.mocked(listTmuxPanePidsForCurrentPlatform).mockResolvedValueOnce(new Map()); vi.mocked(pidusage).mockResolvedValueOnce({ '111': createPidusageStat(111, 123_000_000), '333': createPidusageStat(333, 456_000_000), } as any); const snapshot = await svc.getTeamAgentRuntimeSnapshot('nice-team'); expect(snapshot.members['team-lead']).toMatchObject({ pid: 111, rssBytes: 123_000_000, }); expect(snapshot.members.alice).toMatchObject({ pid: 333, rssBytes: 456_000_000, runtimeModel: 'gpt-5.2', }); }); it('prefers the newest matching agent pid when multiple processes match the same teammate', async () => { const svc = new TeamProvisioningService(); (svc as any).configReader = { getConfig: vi.fn(async () => ({ members: [ { name: 'team-lead', agentType: 'team-lead' }, { name: 'alice', model: 'gpt-5.2' }, ], })), }; (svc as any).readPersistedRuntimeMembers = vi.fn(() => [ { name: 'alice', agentId: 'alice@nice-team', tmuxPaneId: '%0', backendType: 'tmux', }, ]); (svc as any).aliveRunByTeam.set('nice-team', 'run-1'); (svc as any).runs.set('run-1', { runId: 'run-1', child: { pid: 111 }, request: { model: 'gpt-5.4' }, processKilled: false, cancelRequested: false, spawnContext: null, }); (svc as any).readUnixProcessTableRows = vi.fn(() => [ { pid: 222, command: '/Users/belief/.bun/bin/bun cli.js --agent-id alice@nice-team --agent-name alice --team-name nice-team --model gpt-5.2', }, { pid: 333, command: '/Users/belief/.bun/bin/bun cli.js --team-name nice-team --agent-id alice@nice-team --agent-name alice --model gpt-5.2', }, ]); vi.mocked(listTmuxPanePidsForCurrentPlatform).mockResolvedValueOnce(new Map()); vi.mocked(pidusage).mockResolvedValueOnce({ '111': createPidusageStat(111, 123_000_000), '333': createPidusageStat(333, 456_000_000), } as any); const snapshot = await svc.getTeamAgentRuntimeSnapshot('nice-team'); expect(snapshot.members.alice).toMatchObject({ pid: 333, rssBytes: 456_000_000, }); }); }); describe('restartMember', () => { it('uses members meta runtime settings when config members are stale or absent', async () => { const svc = new TeamProvisioningService(); const run = createMemberSpawnRun({ teamName: 'edited-team', expectedMembers: ['alice'], memberSpawnStatuses: new Map([ [ 'alice', createMemberSpawnStatusEntry({ status: 'online', launchState: 'confirmed_alive', agentToolAccepted: true, runtimeAlive: true, bootstrapConfirmed: true, livenessSource: 'heartbeat', firstSpawnAcceptedAt: new Date().toISOString(), lastHeartbeatAt: new Date().toISOString(), }), ], ]), }); run.child = { pid: 111 }; run.processKilled = false; run.cancelRequested = false; const sendMessageToRun = vi.fn(async () => {}); (svc as any).sendMessageToRun = sendMessageToRun; (svc as any).configReader = { getConfig: vi.fn(async () => ({ name: 'Edited Team', members: [{ name: 'team-lead', agentType: 'team-lead' }], })), }; (svc as any).membersMetaStore = { getMembers: vi.fn(async () => [ { name: 'alice', role: 'Reviewer', workflow: 'Use checklist', providerId: 'codex', model: 'gpt-5.4-mini', effort: 'high', agentType: 'general-purpose', }, ]), }; (svc as any).readPersistedRuntimeMembers = vi.fn(() => []); (svc as any).getLiveTeamAgentRuntimeMetadata = vi.fn(async () => new Map()); (svc as any).aliveRunByTeam.set('edited-team', run.runId); (svc as any).runs.set(run.runId, run); await svc.restartMember('edited-team', 'alice'); expect(sendMessageToRun).toHaveBeenCalledTimes(1); const restartCall = sendMessageToRun.mock.calls[0] as unknown as | [unknown, string] | undefined; const restartMessage = restartCall?.[1] ?? ''; expect(restartMessage).toContain('provider="codex"'); expect(restartMessage).toContain('model="gpt-5.4-mini"'); expect(restartMessage).toContain('effort="high"'); expect(restartMessage).toContain('with role "Reviewer"'); expect(restartMessage).toContain('Their workflow: Use checklist'); }); it('re-reads teammate runtime settings immediately before respawn so stale edit snapshots are not reused', async () => { const svc = new TeamProvisioningService(); const run = createMemberSpawnRun({ teamName: 'edited-team', expectedMembers: ['alice'], memberSpawnStatuses: new Map([ [ 'alice', createMemberSpawnStatusEntry({ status: 'online', launchState: 'confirmed_alive', agentToolAccepted: true, runtimeAlive: true, bootstrapConfirmed: true, livenessSource: 'heartbeat', firstSpawnAcceptedAt: new Date().toISOString(), lastHeartbeatAt: new Date().toISOString(), }), ], ]), }); run.child = { pid: 111 }; run.processKilled = false; run.cancelRequested = false; const sendMessageToRun = vi.fn(async () => {}); const getConfig = vi .fn() .mockResolvedValue({ name: 'Edited Team', members: [{ name: 'team-lead', agentType: 'team-lead' }], }); const getMembers = vi .fn() .mockResolvedValueOnce([ { name: 'alice', role: 'Reviewer', workflow: 'Use checklist', providerId: 'codex', model: 'gpt-5.4-mini', effort: 'high', agentType: 'general-purpose', }, ]) .mockResolvedValueOnce([ { name: 'alice', role: 'Approver', workflow: 'Use the updated checklist', providerId: 'codex', model: 'gpt-5.4', effort: 'medium', agentType: 'general-purpose', }, ]); (svc as any).sendMessageToRun = sendMessageToRun; (svc as any).configReader = { getConfig }; (svc as any).membersMetaStore = { getMembers }; (svc as any).readPersistedRuntimeMembers = vi.fn(() => []); (svc as any).getLiveTeamAgentRuntimeMetadata = vi.fn(async () => new Map()); (svc as any).aliveRunByTeam.set('edited-team', run.runId); (svc as any).runs.set(run.runId, run); await svc.restartMember('edited-team', 'alice'); expect(getMembers).toHaveBeenCalledTimes(2); expect(sendMessageToRun).toHaveBeenCalledTimes(1); const restartCall = sendMessageToRun.mock.calls[0] as unknown as | [unknown, string] | undefined; const restartMessage = restartCall?.[1] ?? ''; expect(restartMessage).toContain('provider="codex"'); expect(restartMessage).toContain('model="gpt-5.4"'); expect(restartMessage).toContain('effort="medium"'); expect(restartMessage).toContain('with role "Approver"'); expect(restartMessage).toContain('Their workflow: Use the updated checklist'); }); it('aborts restart if the teammate is removed before respawn is requested', async () => { const svc = new TeamProvisioningService(); const run = createMemberSpawnRun({ teamName: 'edited-team', expectedMembers: ['alice'], memberSpawnStatuses: new Map([ [ 'alice', createMemberSpawnStatusEntry({ status: 'online', launchState: 'confirmed_alive', agentToolAccepted: true, runtimeAlive: true, bootstrapConfirmed: true, livenessSource: 'heartbeat', firstSpawnAcceptedAt: new Date().toISOString(), lastHeartbeatAt: new Date().toISOString(), }), ], ]), }); run.child = { pid: 111 }; run.processKilled = false; run.cancelRequested = false; const sendMessageToRun = vi.fn(async () => {}); const getConfig = vi .fn() .mockResolvedValue({ name: 'Edited Team', members: [{ name: 'team-lead', agentType: 'team-lead' }], }); const getMembers = vi .fn() .mockResolvedValueOnce([ { name: 'alice', role: 'Reviewer', providerId: 'codex', model: 'gpt-5.4-mini', effort: 'high', agentType: 'general-purpose', }, ]) .mockResolvedValueOnce([ { name: 'alice', role: 'Reviewer', providerId: 'codex', model: 'gpt-5.4-mini', effort: 'high', agentType: 'general-purpose', removedAt: new Date().toISOString(), }, ]); (svc as any).sendMessageToRun = sendMessageToRun; (svc as any).configReader = { getConfig }; (svc as any).membersMetaStore = { getMembers }; (svc as any).readPersistedRuntimeMembers = vi.fn(() => []); (svc as any).getLiveTeamAgentRuntimeMetadata = vi.fn(async () => new Map()); (svc as any).aliveRunByTeam.set('edited-team', run.runId); (svc as any).runs.set(run.runId, run); await expect(svc.restartMember('edited-team', 'alice')).rejects.toThrow( 'Member "alice" was removed while restart was in progress' ); expect(sendMessageToRun).not.toHaveBeenCalled(); expect(run.pendingMemberRestarts.has('alice')).toBe(false); expect(run.memberSpawnStatuses.get('alice')).toMatchObject({ status: 'offline', launchState: 'starting', runtimeAlive: false, }); }); it('aborts restart if team config disappears before respawn is requested', async () => { const svc = new TeamProvisioningService(); const run = createMemberSpawnRun({ teamName: 'edited-team', expectedMembers: ['alice'], memberSpawnStatuses: new Map([ [ 'alice', createMemberSpawnStatusEntry({ status: 'online', launchState: 'confirmed_alive', agentToolAccepted: true, runtimeAlive: true, bootstrapConfirmed: true, livenessSource: 'heartbeat', firstSpawnAcceptedAt: new Date().toISOString(), lastHeartbeatAt: new Date().toISOString(), }), ], ]), }); run.child = { pid: 111 }; run.processKilled = false; run.cancelRequested = false; const sendMessageToRun = vi.fn(async () => {}); const getConfig = vi .fn() .mockResolvedValueOnce({ name: 'Edited Team', members: [{ name: 'team-lead', agentType: 'team-lead' }], }) .mockResolvedValueOnce(null); const getMembers = vi.fn(async () => [ { name: 'alice', role: 'Reviewer', providerId: 'codex', model: 'gpt-5.4-mini', effort: 'high', agentType: 'general-purpose', }, ]); (svc as any).sendMessageToRun = sendMessageToRun; (svc as any).configReader = { getConfig }; (svc as any).membersMetaStore = { getMembers }; (svc as any).readPersistedRuntimeMembers = vi.fn(() => []); (svc as any).getLiveTeamAgentRuntimeMetadata = vi.fn(async () => new Map()); (svc as any).aliveRunByTeam.set('edited-team', run.runId); (svc as any).runs.set(run.runId, run); await expect(svc.restartMember('edited-team', 'alice')).rejects.toThrow( 'Team "edited-team" configuration disappeared while restart was in progress' ); expect(sendMessageToRun).not.toHaveBeenCalled(); expect(run.pendingMemberRestarts.has('alice')).toBe(false); expect(run.memberSpawnStatuses.get('alice')).toMatchObject({ status: 'offline', launchState: 'starting', runtimeAlive: false, }); }); it('treats duplicate_skipped already_running as a failed codex restart because the old runtime is still active', async () => { const svc = new TeamProvisioningService(); const run = createMemberSpawnRun({ teamName: 'codex-team', expectedMembers: ['bob'], memberSpawnStatuses: new Map(), }); run.child = { pid: 111 }; run.processKilled = false; run.cancelRequested = false; const sendMessageToRun = vi.fn(async () => {}); (svc as any).sendMessageToRun = sendMessageToRun; (svc as any).configReader = { getConfig: vi.fn(async () => ({ name: 'Codex Team', members: [{ name: 'team-lead', agentType: 'team-lead' }], })), }; (svc as any).membersMetaStore = { getMembers: vi.fn(async () => [ { name: 'bob', role: 'Developer', providerId: 'codex', model: 'gpt-5.2', effort: 'medium', agentType: 'general-purpose', }, ]), }; (svc as any).readPersistedRuntimeMembers = vi.fn(() => []); (svc as any).getLiveTeamAgentRuntimeMetadata = vi.fn(async () => new Map()); (svc as any).aliveRunByTeam.set('codex-team', run.runId); (svc as any).runs.set(run.runId, run); await svc.restartMember('codex-team', 'bob'); expect(run.memberSpawnStatuses.get('bob')).toMatchObject({ status: 'spawning', launchState: 'starting', }); expect(sendMessageToRun).toHaveBeenCalledWith( run, expect.stringContaining('provider="codex", model="gpt-5.2", effort="medium"') ); run.activeToolCalls.set('tool-agent-1', { memberName: 'bob', toolUseId: 'tool-agent-1', toolName: 'Agent', preview: 'Spawn teammate bob', startedAt: new Date().toISOString(), state: 'running', source: 'runtime', }); run.memberSpawnToolUseIds.set('tool-agent-1', 'bob'); (svc as any).finishRuntimeToolActivity( run, 'tool-agent-1', [ { type: 'text', text: 'status: duplicate_skipped\nreason: already_running\nname: bob\nteam_name: codex-team', }, ], false ); expect(run.memberSpawnStatuses.get('bob')).toMatchObject({ status: 'error', launchState: 'failed_to_start', runtimeAlive: false, hardFailure: true, hardFailureReason: 'Restart for teammate "bob" was skipped because the previous runtime still appears to be active. The requested settings may not have been applied.', }); expect(run.pendingMemberRestarts.has('bob')).toBe(false); }); it('keeps a codex teammate restart pending instead of failed when lead reports duplicate_skipped bootstrap_pending', async () => { const svc = new TeamProvisioningService(); const run = createMemberSpawnRun({ teamName: 'codex-team', expectedMembers: ['bob'], memberSpawnStatuses: new Map(), }); run.child = { pid: 111 }; run.processKilled = false; run.cancelRequested = false; (svc as any).sendMessageToRun = vi.fn(async () => {}); (svc as any).configReader = { getConfig: vi.fn(async () => ({ name: 'Codex Team', members: [{ name: 'team-lead', agentType: 'team-lead' }], })), }; (svc as any).membersMetaStore = { getMembers: vi.fn(async () => [ { name: 'bob', role: 'Developer', providerId: 'codex', model: 'gpt-5.2', effort: 'medium', agentType: 'general-purpose', }, ]), }; (svc as any).readPersistedRuntimeMembers = vi.fn(() => []); (svc as any).getLiveTeamAgentRuntimeMetadata = vi.fn(async () => new Map()); (svc as any).aliveRunByTeam.set('codex-team', run.runId); (svc as any).runs.set(run.runId, run); await svc.restartMember('codex-team', 'bob'); run.activeToolCalls.set('tool-agent-1', { memberName: 'bob', toolUseId: 'tool-agent-1', toolName: 'Agent', preview: 'Spawn teammate bob', startedAt: new Date().toISOString(), state: 'running', source: 'runtime', }); run.memberSpawnToolUseIds.set('tool-agent-1', 'bob'); (svc as any).finishRuntimeToolActivity( run, 'tool-agent-1', [ { type: 'text', text: 'status: duplicate_skipped\nreason: bootstrap_pending\nname: bob\nteam_name: codex-team', }, ], false ); expect(run.memberSpawnStatuses.get('bob')).toMatchObject({ status: 'waiting', launchState: 'runtime_pending_bootstrap', runtimeAlive: false, agentToolAccepted: true, hardFailure: false, hardFailureReason: undefined, }); expect(run.pendingMemberRestarts.has('bob')).toBe(true); }); it('fails a codex teammate restart immediately when Agent returns duplicate_skipped without a reason', async () => { allowConsoleLogs(); const svc = new TeamProvisioningService(); const run = createMemberSpawnRun({ teamName: 'codex-team', expectedMembers: ['jack'], memberSpawnStatuses: new Map([ [ 'jack', createMemberSpawnStatusEntry({ launchState: 'failed_to_start', hardFailure: true, hardFailureReason: 'Teammate was never spawned during launch.', error: 'Teammate was never spawned during launch.', }), ], ]), }); run.child = { pid: 111 }; run.processKilled = false; run.cancelRequested = false; (svc as any).sendMessageToRun = vi.fn(async () => {}); (svc as any).configReader = { getConfig: vi.fn(async () => ({ name: 'Codex Team', members: [{ name: 'team-lead', agentType: 'team-lead' }], })), }; (svc as any).membersMetaStore = { getMembers: vi.fn(async () => [ { name: 'jack', role: 'Developer', providerId: 'codex', model: 'gpt-5.4', effort: 'medium', agentType: 'general-purpose', }, ]), }; (svc as any).readPersistedRuntimeMembers = vi.fn(() => []); (svc as any).getLiveTeamAgentRuntimeMetadata = vi.fn(async () => new Map()); (svc as any).aliveRunByTeam.set('codex-team', run.runId); (svc as any).runs.set(run.runId, run); await svc.restartMember('codex-team', 'jack'); run.activeToolCalls.set('tool-agent-1', { memberName: 'jack', toolUseId: 'tool-agent-1', toolName: 'Agent', preview: 'Spawn teammate jack', startedAt: new Date().toISOString(), state: 'running', source: 'runtime', }); run.memberSpawnToolUseIds.set('tool-agent-1', 'jack'); (svc as any).finishRuntimeToolActivity( run, 'tool-agent-1', [ { type: 'text', text: 'status: duplicate_skipped\nname: jack\nteam_name: codex-team', }, ], false ); expect(run.pendingMemberRestarts.has('jack')).toBe(false); expect(run.memberSpawnStatuses.get('jack')).toMatchObject({ status: 'error', launchState: 'failed_to_start', runtimeAlive: false, hardFailure: true, hardFailureReason: 'Restart for teammate "jack" could not be confirmed and may not have applied. Agent returned duplicate_skipped without a reason.', }); }); it('waits for a killed tmux pane to disappear before sending a restart request', async () => { vi.useFakeTimers(); const svc = new TeamProvisioningService(); const run = createMemberSpawnRun({ teamName: 'tmux-team', expectedMembers: ['forge'], memberSpawnStatuses: new Map(), }); run.child = { pid: 111 }; run.processKilled = false; run.cancelRequested = false; const sendMessageToRun = vi.fn(async () => {}); (svc as any).sendMessageToRun = sendMessageToRun; (svc as any).configReader = { getConfig: vi.fn(async () => ({ name: 'Tmux Team', members: [{ name: 'team-lead', agentType: 'team-lead' }], })), }; (svc as any).membersMetaStore = { getMembers: vi.fn(async () => [ { name: 'forge', role: 'Developer', providerId: 'codex', model: 'gpt-5.4', effort: 'medium', agentType: 'general-purpose', }, ]), }; (svc as any).readPersistedRuntimeMembers = vi.fn(() => [ { name: 'forge', agentId: 'forge@tmux-team', backendType: 'tmux', tmuxPaneId: '%2', }, ]); (svc as any).getLiveTeamAgentRuntimeMetadata = vi.fn(async () => new Map()); (svc as any).aliveRunByTeam.set('tmux-team', run.runId); (svc as any).runs.set(run.runId, run); vi.mocked(listTmuxPanePidsForCurrentPlatform) .mockResolvedValueOnce(new Map([['%2', 999]])) .mockResolvedValueOnce(new Map()); const restartPromise = svc.restartMember('tmux-team', 'forge'); await Promise.resolve(); expect(sendMessageToRun).not.toHaveBeenCalled(); await vi.advanceTimersByTimeAsync(100); await restartPromise; expect(sendMessageToRun).toHaveBeenCalledTimes(1); }); it('fails early when the previous tmux pane does not exit before restart', async () => { vi.useFakeTimers(); const svc = new TeamProvisioningService(); const run = createMemberSpawnRun({ teamName: 'tmux-team', expectedMembers: ['forge'], memberSpawnStatuses: new Map(), }); run.child = { pid: 111 }; run.processKilled = false; run.cancelRequested = false; const sendMessageToRun = vi.fn(async () => {}); (svc as any).sendMessageToRun = sendMessageToRun; (svc as any).configReader = { getConfig: vi.fn(async () => ({ name: 'Tmux Team', members: [{ name: 'team-lead', agentType: 'team-lead' }], })), }; (svc as any).membersMetaStore = { getMembers: vi.fn(async () => [ { name: 'forge', role: 'Developer', providerId: 'codex', model: 'gpt-5.4', effort: 'medium', agentType: 'general-purpose', }, ]), }; (svc as any).readPersistedRuntimeMembers = vi.fn(() => [ { name: 'forge', agentId: 'forge@tmux-team', backendType: 'tmux', tmuxPaneId: '%2', }, ]); (svc as any).getLiveTeamAgentRuntimeMetadata = vi.fn(async () => new Map()); (svc as any).aliveRunByTeam.set('tmux-team', run.runId); (svc as any).runs.set(run.runId, run); vi.mocked(listTmuxPanePidsForCurrentPlatform).mockImplementation(async () => new Map([['%2', 999]]) ); const restartPromise = expect(svc.restartMember('tmux-team', 'forge')).rejects.toThrow( 'Restart for teammate "forge" is still waiting for the previous tmux pane to exit (%2).' ); await vi.advanceTimersByTimeAsync(1_500); await restartPromise; expect(sendMessageToRun).not.toHaveBeenCalled(); }); it('still verifies tmux pane exit when pane kill throws, and blocks restart if the pane remains alive', async () => { vi.useFakeTimers(); const svc = new TeamProvisioningService(); const run = createMemberSpawnRun({ teamName: 'tmux-team', expectedMembers: ['forge'], memberSpawnStatuses: new Map(), }); run.child = { pid: 111 }; run.processKilled = false; run.cancelRequested = false; const sendMessageToRun = vi.fn(async () => {}); (svc as any).sendMessageToRun = sendMessageToRun; (svc as any).configReader = { getConfig: vi.fn(async () => ({ name: 'Tmux Team', members: [{ name: 'team-lead', agentType: 'team-lead' }], })), }; (svc as any).membersMetaStore = { getMembers: vi.fn(async () => [ { name: 'forge', role: 'Developer', providerId: 'codex', model: 'gpt-5.4', effort: 'medium', agentType: 'general-purpose', }, ]), }; (svc as any).readPersistedRuntimeMembers = vi.fn(() => [ { name: 'forge', agentId: 'forge@tmux-team', backendType: 'tmux', tmuxPaneId: '%2', }, ]); (svc as any).getLiveTeamAgentRuntimeMetadata = vi.fn(async () => new Map()); (svc as any).aliveRunByTeam.set('tmux-team', run.runId); (svc as any).runs.set(run.runId, run); vi.mocked(killTmuxPaneForCurrentPlatformSync).mockImplementation(() => { throw new Error('pane kill failed'); }); vi.mocked(listTmuxPanePidsForCurrentPlatform).mockImplementation(async () => new Map([['%2', 999]]) ); const restartPromise = expect(svc.restartMember('tmux-team', 'forge')).rejects.toThrow( 'Restart for teammate "forge" is still waiting for the previous tmux pane to exit (%2).' ); await vi.advanceTimersByTimeAsync(1_500); await restartPromise; expect(sendMessageToRun).not.toHaveBeenCalled(); }); it('does not treat tmux pane lookup failures as a successful restart precondition', async () => { vi.useFakeTimers(); const svc = new TeamProvisioningService(); const run = createMemberSpawnRun({ teamName: 'tmux-team', expectedMembers: ['forge'], memberSpawnStatuses: new Map(), }); run.child = { pid: 111 }; run.processKilled = false; run.cancelRequested = false; const sendMessageToRun = vi.fn(async () => {}); (svc as any).sendMessageToRun = sendMessageToRun; (svc as any).configReader = { getConfig: vi.fn(async () => ({ name: 'Tmux Team', members: [{ name: 'team-lead', agentType: 'team-lead' }], })), }; (svc as any).membersMetaStore = { getMembers: vi.fn(async () => [ { name: 'forge', role: 'Developer', providerId: 'codex', model: 'gpt-5.4', effort: 'medium', agentType: 'general-purpose', }, ]), }; (svc as any).readPersistedRuntimeMembers = vi.fn(() => [ { name: 'forge', agentId: 'forge@tmux-team', backendType: 'tmux', tmuxPaneId: '%2', }, ]); (svc as any).getLiveTeamAgentRuntimeMetadata = vi.fn(async () => new Map()); (svc as any).aliveRunByTeam.set('tmux-team', run.runId); (svc as any).runs.set(run.runId, run); vi.mocked(listTmuxPanePidsForCurrentPlatform).mockRejectedValue( new Error('tmux list-panes failed') ); const restartPromise = expect(svc.restartMember('tmux-team', 'forge')).rejects.toThrow( 'Restart for teammate "forge" could not verify that the previous tmux pane exited: tmux list-panes failed' ); await vi.advanceTimersByTimeAsync(1_500); await restartPromise; expect(sendMessageToRun).not.toHaveBeenCalled(); }); it('treats a dead tmux server as successful pane exit verification after kill', async () => { vi.useFakeTimers(); const svc = new TeamProvisioningService(); const run = createMemberSpawnRun({ teamName: 'tmux-team', expectedMembers: ['forge'], memberSpawnStatuses: new Map(), }); run.child = { pid: 111 }; run.processKilled = false; run.cancelRequested = false; const sendMessageToRun = vi.fn(async () => {}); (svc as any).sendMessageToRun = sendMessageToRun; (svc as any).configReader = { getConfig: vi.fn(async () => ({ name: 'Tmux Team', members: [{ name: 'team-lead', agentType: 'team-lead' }], })), }; (svc as any).membersMetaStore = { getMembers: vi.fn(async () => [ { name: 'forge', role: 'Developer', providerId: 'codex', model: 'gpt-5.4', effort: 'medium', agentType: 'general-purpose', }, ]), }; (svc as any).readPersistedRuntimeMembers = vi.fn(() => [ { name: 'forge', agentId: 'forge@tmux-team', backendType: 'tmux', tmuxPaneId: '%2', }, ]); (svc as any).getLiveTeamAgentRuntimeMetadata = vi.fn(async () => new Map()); (svc as any).aliveRunByTeam.set('tmux-team', run.runId); (svc as any).runs.set(run.runId, run); vi.mocked(listTmuxPanePidsForCurrentPlatform).mockRejectedValue( new Error('no server running on /private/tmp/tmux-501/default') ); await svc.restartMember('tmux-team', 'forge'); expect(sendMessageToRun).toHaveBeenCalledTimes(1); }); it('fails early when the previous process backend runtime does not exit before restart', async () => { vi.useFakeTimers(); const svc = new TeamProvisioningService(); const run = createMemberSpawnRun({ teamName: 'process-team', expectedMembers: ['forge'], memberSpawnStatuses: new Map(), }); run.child = { pid: 111 }; run.processKilled = false; run.cancelRequested = false; const sendMessageToRun = vi.fn(async () => {}); (svc as any).sendMessageToRun = sendMessageToRun; (svc as any).configReader = { getConfig: vi.fn(async () => ({ name: 'Process Team', members: [{ name: 'team-lead', agentType: 'team-lead' }], })), }; (svc as any).membersMetaStore = { getMembers: vi.fn(async () => [ { name: 'forge', role: 'Developer', providerId: 'codex', model: 'gpt-5.4', effort: 'medium', agentType: 'general-purpose', }, ]), }; (svc as any).readPersistedRuntimeMembers = vi.fn(() => []); (svc as any).getLiveTeamAgentRuntimeMetadata = vi.fn( async () => new Map([ [ 'forge', { alive: true, backendType: 'process', pid: process.pid, agentId: 'forge@process-team', }, ], ]) ); (svc as any).aliveRunByTeam.set('process-team', run.runId); (svc as any).runs.set(run.runId, run); const restartPromise = expect(svc.restartMember('process-team', 'forge')).rejects.toThrow( `Restart for teammate "forge" is still waiting for the previous process to exit (${process.pid}).` ); await vi.advanceTimersByTimeAsync(1_500); await restartPromise; expect(vi.mocked(killProcessByPid)).toHaveBeenCalledWith(process.pid); expect(sendMessageToRun).not.toHaveBeenCalled(); }); it('bypasses stale live runtime metadata cache before restarting a process backend teammate', async () => { vi.useFakeTimers(); const svc = new TeamProvisioningService(); const run = createMemberSpawnRun({ teamName: 'process-team', expectedMembers: ['forge'], memberSpawnStatuses: new Map(), }); run.child = { pid: 111 }; run.processKilled = false; run.cancelRequested = false; const sendMessageToRun = vi.fn(async () => {}); (svc as any).sendMessageToRun = sendMessageToRun; (svc as any).configReader = { getConfig: vi.fn(async () => ({ name: 'Process Team', members: [{ name: 'team-lead', agentType: 'team-lead' }], })), }; (svc as any).membersMetaStore = { getMembers: vi.fn(async () => [ { name: 'forge', role: 'Developer', providerId: 'codex', model: 'gpt-5.4', effort: 'medium', agentType: 'general-purpose', }, ]), }; (svc as any).readPersistedRuntimeMembers = vi.fn(() => [ { name: 'forge', agentId: 'forge@process-team', backendType: 'process', }, ]); (svc as any).findLiveProcessPidByAgentId = vi.fn(() => new Map([['forge@process-team', process.pid]]) ); (svc as any).liveTeamAgentRuntimeMetadataCache.set('process-team', { expiresAtMs: Date.now() + 60_000, metadata: new Map([ [ 'forge', { alive: false, backendType: 'process', agentId: 'forge@process-team', }, ], ]), }); (svc as any).aliveRunByTeam.set('process-team', run.runId); (svc as any).runs.set(run.runId, run); const restartPromise = expect(svc.restartMember('process-team', 'forge')).rejects.toThrow( `Restart for teammate "forge" is still waiting for the previous process to exit (${process.pid}).` ); await vi.advanceTimersByTimeAsync(1_500); await restartPromise; expect(vi.mocked(killProcessByPid)).toHaveBeenCalledWith(process.pid); expect(sendMessageToRun).not.toHaveBeenCalled(); }); it('uses members.meta agentId to detect a live process backend teammate when config runtime identity is stale', async () => { vi.useFakeTimers(); const svc = new TeamProvisioningService(); const run = createMemberSpawnRun({ teamName: 'process-team', expectedMembers: ['forge'], memberSpawnStatuses: new Map(), }); run.child = { pid: 111 }; run.processKilled = false; run.cancelRequested = false; const sendMessageToRun = vi.fn(async () => {}); (svc as any).sendMessageToRun = sendMessageToRun; (svc as any).configReader = { getConfig: vi.fn(async () => ({ name: 'Process Team', members: [{ name: 'team-lead', agentType: 'team-lead' }], })), }; (svc as any).membersMetaStore = { getMembers: vi.fn(async () => [ { name: 'forge', role: 'Developer', providerId: 'codex', model: 'gpt-5.4', effort: 'medium', agentType: 'general-purpose', agentId: 'forge@process-team', }, ]), }; (svc as any).readPersistedRuntimeMembers = vi.fn(() => []); (svc as any).findLiveProcessPidByAgentId = vi.fn(() => new Map([['forge@process-team', process.pid]]) ); (svc as any).aliveRunByTeam.set('process-team', run.runId); (svc as any).runs.set(run.runId, run); const restartPromise = expect(svc.restartMember('process-team', 'forge')).rejects.toThrow( `Restart for teammate "forge" is still waiting for the previous process to exit (${process.pid}).` ); await vi.advanceTimersByTimeAsync(1_500); await restartPromise; expect(vi.mocked(killProcessByPid)).toHaveBeenCalledWith(process.pid); expect(sendMessageToRun).not.toHaveBeenCalled(); }); it('rejects a second restart request while the first restart is still in flight', async () => { const svc = new TeamProvisioningService(); const run = createMemberSpawnRun({ teamName: 'codex-team', expectedMembers: ['bob'], memberSpawnStatuses: new Map(), }); run.child = { pid: 111 }; run.processKilled = false; run.cancelRequested = false; run.pendingMemberRestarts.set('bob', { requestedAt: new Date().toISOString(), desired: { name: 'bob', providerId: 'codex', model: 'gpt-5.2', effort: 'medium', }, }); (svc as any).configReader = { getConfig: vi.fn(async () => ({ name: 'Codex Team', members: [{ name: 'team-lead', agentType: 'team-lead' }], })), }; (svc as any).membersMetaStore = { getMembers: vi.fn(async () => [ { name: 'bob', role: 'Developer', providerId: 'codex', model: 'gpt-5.2', effort: 'medium', agentType: 'general-purpose', }, ]), }; (svc as any).aliveRunByTeam.set('codex-team', run.runId); (svc as any).runs.set(run.runId, run); await expect(svc.restartMember('codex-team', 'bob')).rejects.toThrow( 'Restart for teammate "bob" is already in progress' ); }); it('clears stale member spawn tool tracking before starting a manual restart', async () => { const svc = new TeamProvisioningService(); const run = createMemberSpawnRun({ teamName: 'codex-team', expectedMembers: ['bob'], memberSpawnStatuses: new Map([ [ 'bob', createMemberSpawnStatusEntry({ status: 'waiting', launchState: 'runtime_pending_bootstrap', agentToolAccepted: true, firstSpawnAcceptedAt: new Date().toISOString(), }), ], ]), }); run.child = { pid: 111 }; run.processKilled = false; run.cancelRequested = false; run.activeToolCalls.set('tool-agent-old', { memberName: 'bob', toolUseId: 'tool-agent-old', toolName: 'Agent', preview: 'Spawn teammate bob', startedAt: new Date().toISOString(), state: 'running', source: 'runtime', }); run.memberSpawnToolUseIds.set('tool-agent-old', 'bob'); const sendMessageToRun = vi.fn(async () => {}); (svc as any).sendMessageToRun = sendMessageToRun; (svc as any).configReader = { getConfig: vi.fn(async () => ({ name: 'Codex Team', members: [{ name: 'team-lead', agentType: 'team-lead' }], })), }; (svc as any).membersMetaStore = { getMembers: vi.fn(async () => [ { name: 'bob', role: 'Developer', providerId: 'codex', model: 'gpt-5.2', effort: 'medium', agentType: 'general-purpose', }, ]), }; (svc as any).readPersistedRuntimeMembers = vi.fn(() => []); (svc as any).getLiveTeamAgentRuntimeMetadata = vi.fn(async () => new Map()); (svc as any).aliveRunByTeam.set('codex-team', run.runId); (svc as any).runs.set(run.runId, run); await svc.restartMember('codex-team', 'bob'); expect(run.activeToolCalls.has('tool-agent-old')).toBe(false); expect(run.memberSpawnToolUseIds.has('tool-agent-old')).toBe(false); expect(sendMessageToRun).toHaveBeenCalledTimes(1); (svc as any).finishRuntimeToolActivity( run, 'tool-agent-old', [{ type: 'text', text: 'late stale result' }], true ); expect(run.memberSpawnStatuses.get('bob')).toMatchObject({ status: 'spawning', launchState: 'starting', }); expect(run.pendingMemberRestarts.has('bob')).toBe(true); }); it('marks a pending restart as failed when the teammate never rejoins within the restart grace window', async () => { const svc = new TeamProvisioningService(); const run = createMemberSpawnRun({ teamName: 'codex-team', expectedMembers: ['bob'], memberSpawnStatuses: new Map([ [ 'bob', createMemberSpawnStatusEntry({ status: 'waiting', launchState: 'runtime_pending_bootstrap', agentToolAccepted: true, firstSpawnAcceptedAt: new Date(Date.now() - 120_000).toISOString(), }), ], ]), }); run.pendingMemberRestarts.set('bob', { requestedAt: new Date(Date.now() - 120_000).toISOString(), desired: { name: 'bob', providerId: 'codex', model: 'gpt-5.2', effort: 'medium', }, }); (svc as any).refreshMemberSpawnStatusesFromLeadInbox = vi.fn(async () => {}); (svc as any).maybeAuditMemberSpawnStatuses = vi.fn(async () => {}); await (svc as any).reevaluateMemberLaunchStatus(run, 'bob'); expect(run.memberSpawnStatuses.get('bob')).toMatchObject({ status: 'error', launchState: 'failed_to_start', error: 'Teammate "bob" did not rejoin within the restart grace window.', hardFailureReason: 'Teammate "bob" did not rejoin within the restart grace window.', }); expect(run.pendingMemberRestarts.has('bob')).toBe(false); }); }); it('removes generated MCP config when createTeam spawn fails synchronously', async () => { allowConsoleLogs(); vi.mocked(ClaudeBinaryResolver.resolve).mockResolvedValue('/mock/claude'); vi.mocked(spawnCli).mockImplementation(() => { throw new Error('spawn EINVAL'); }); const mcpConfigBuilder = { writeConfigFile: vi.fn(async () => '/mock/mcp-config-create.json'), removeConfigFile: vi.fn(async () => {}), }; const membersMetaStore = { writeMembers: vi.fn(async () => {}), }; const teamMetaStore = { writeMeta: vi.fn(async () => {}), deleteMeta: vi.fn(async () => {}), }; const svc = new TeamProvisioningService( undefined, undefined, membersMetaStore as any, undefined, mcpConfigBuilder as any, teamMetaStore as any ); (svc as any).buildProvisioningEnv = vi.fn(async () => ({ env: { ANTHROPIC_API_KEY: 'test' }, authSource: 'anthropic_api_key', })); (svc as any).validateAgentTeamsMcpRuntime = vi.fn(async () => {}); (svc as any).pathExists = vi.fn(async () => false); await expect( svc.createTeam( { teamName: 'cleanup-team', cwd: tempClaudeRoot, members: [{ name: 'alice' }], }, () => {} ) ).rejects.toThrow('spawn EINVAL'); expect(mcpConfigBuilder.writeConfigFile).toHaveBeenCalledWith(tempClaudeRoot); expect(mcpConfigBuilder.removeConfigFile).toHaveBeenCalledWith('/mock/mcp-config-create.json'); expect(teamMetaStore.deleteMeta).toHaveBeenCalledWith('cleanup-team'); }); it('removes generated MCP config when launchTeam spawn fails synchronously', async () => { allowConsoleLogs(); const teamName = 'launch-cleanup-team'; const teamDir = path.join(tempTeamsBase, teamName); fs.mkdirSync(teamDir, { recursive: true }); fs.writeFileSync( path.join(teamDir, 'config.json'), JSON.stringify({ name: teamName, projectPath: tempClaudeRoot, members: [{ name: 'team-lead', agentType: 'team-lead' }, { name: 'alice' }], }), 'utf8' ); vi.mocked(ClaudeBinaryResolver.resolve).mockResolvedValue('/mock/claude'); vi.mocked(spawnCli).mockImplementation(() => { throw new Error('launch spawn EINVAL'); }); const mcpConfigBuilder = { writeConfigFile: vi.fn(async () => '/mock/mcp-config-launch.json'), removeConfigFile: vi.fn(async () => {}), }; const restorePrelaunchConfig = vi.fn(async () => {}); const svc = new TeamProvisioningService( undefined, undefined, undefined, undefined, mcpConfigBuilder as any ); (svc as any).buildProvisioningEnv = vi.fn(async () => ({ env: { ANTHROPIC_API_KEY: 'test' }, authSource: 'anthropic_api_key', })); (svc as any).resolveLaunchExpectedMembers = vi.fn(async () => ({ members: [{ name: 'alice' }], source: 'members-meta', warning: undefined, })); (svc as any).normalizeTeamConfigForLaunch = vi.fn(async () => {}); (svc as any).assertConfigLeadOnlyForLaunch = vi.fn(async () => {}); (svc as any).updateConfigProjectPath = vi.fn(async () => {}); (svc as any).restorePrelaunchConfig = restorePrelaunchConfig; (svc as any).validateAgentTeamsMcpRuntime = vi.fn(async () => {}); (svc as any).pathExists = vi.fn(async () => false); await expect(svc.launchTeam({ teamName, cwd: tempClaudeRoot }, () => {})).rejects.toThrow( 'launch spawn EINVAL' ); expect(mcpConfigBuilder.writeConfigFile).toHaveBeenCalledWith(tempClaudeRoot); expect(mcpConfigBuilder.removeConfigFile).toHaveBeenCalledWith('/mock/mcp-config-launch.json'); expect(restorePrelaunchConfig).toHaveBeenCalledWith(teamName); }); it('regenerates a missing --mcp-config before auth-failure respawn', async () => { vi.useFakeTimers(); allowConsoleLogs(); vi.mocked(ClaudeBinaryResolver.resolve).mockResolvedValue('/mock/claude'); const firstChild = createRunningChild(); const secondChild = createRunningChild(); vi.mocked(spawnCli) .mockImplementationOnce(() => firstChild as any) .mockImplementationOnce(() => secondChild as any); const mcpConfigBuilder = { writeConfigFile: vi .fn() .mockResolvedValueOnce('/missing/original-mcp-config.json') .mockResolvedValueOnce('/regenerated/mcp-config.json'), removeConfigFile: vi.fn(async () => {}), }; const membersMetaStore = { writeMembers: vi.fn(async () => {}), }; const teamMetaStore = { writeMeta: vi.fn(async () => {}), deleteMeta: vi.fn(async () => {}), }; const svc = new TeamProvisioningService( undefined, undefined, membersMetaStore as any, undefined, mcpConfigBuilder as any, teamMetaStore as any ); (svc as any).buildProvisioningEnv = vi.fn(async () => ({ env: { ANTHROPIC_API_KEY: 'test' }, authSource: 'anthropic_api_key', })); (svc as any).validateAgentTeamsMcpRuntime = vi.fn(async () => {}); (svc as any).pathExists = vi.fn(async () => false); (svc as any).startFilesystemMonitor = vi.fn(); (svc as any).stopFilesystemMonitor = vi.fn(); (svc as any).startStallWatchdog = vi.fn(); (svc as any).stopStallWatchdog = vi.fn(); (svc as any).attachStdoutHandler = vi.fn(); (svc as any).attachStderrHandler = vi.fn(); const { runId } = await svc.createTeam( { teamName: 'retry-team', cwd: tempClaudeRoot, members: [{ name: 'alice' }], }, () => {} ); const run = (svc as any).runs.get(runId); expect(run).toBeTruthy(); const mcpFlagIdx = run.spawnContext.args.indexOf('--mcp-config'); expect(mcpFlagIdx).toBeGreaterThanOrEqual(0); run.spawnContext.args[mcpFlagIdx + 1] = path.join(tempClaudeRoot, 'deleted-mcp-config.json'); run.mcpConfigPath = run.spawnContext.args[mcpFlagIdx + 1]; run.authRetryInProgress = true; const respawnPromise = (svc as any).respawnAfterAuthFailure(run); await vi.advanceTimersByTimeAsync(2000); await respawnPromise; expect(mcpConfigBuilder.writeConfigFile).toHaveBeenNthCalledWith(2, tempClaudeRoot); expect(run.spawnContext.args[mcpFlagIdx + 1]).toBe('/regenerated/mcp-config.json'); expect(run.mcpConfigPath).toBe('/regenerated/mcp-config.json'); expect(vi.mocked(spawnCli)).toHaveBeenNthCalledWith( 2, '/mock/claude', run.spawnContext.args, expect.objectContaining({ cwd: tempClaudeRoot, stdio: ['pipe', 'pipe', 'pipe'], }) ); expect(run.child).toBe(secondChild); if (run.timeoutHandle) { clearTimeout(run.timeoutHandle); run.timeoutHandle = null; } }); it('pre-seeds lead bootstrap MCP permissions before createTeam spawn', async () => { allowConsoleLogs(); vi.mocked(ClaudeBinaryResolver.resolve).mockResolvedValue('/mock/claude'); vi.mocked(spawnCli).mockImplementation(() => { throw new Error('spawn EINVAL'); }); const mcpConfigBuilder = { writeConfigFile: vi.fn(async () => '/mock/mcp-config-create.json'), removeConfigFile: vi.fn(async () => {}), }; const membersMetaStore = { writeMembers: vi.fn(async () => {}), }; const teamMetaStore = { writeMeta: vi.fn(async () => {}), deleteMeta: vi.fn(async () => {}), }; const svc = new TeamProvisioningService( undefined, undefined, membersMetaStore as any, undefined, mcpConfigBuilder as any, teamMetaStore as any ); (svc as any).buildProvisioningEnv = vi.fn(async () => ({ env: { ANTHROPIC_API_KEY: 'test' }, authSource: 'anthropic_api_key', })); (svc as any).validateAgentTeamsMcpRuntime = vi.fn(async () => {}); (svc as any).pathExists = vi.fn(async () => false); await expect( svc.createTeam( { teamName: 'seeded-team', cwd: tempClaudeRoot, members: [{ name: 'alice' }], skipPermissions: false, }, () => {} ) ).rejects.toThrow('spawn EINVAL'); const settingsPath = path.join(tempClaudeRoot, '.claude', 'settings.local.json'); const settings = JSON.parse(fs.readFileSync(settingsPath, 'utf8')) as { permissions?: { allow?: string[] }; }; expect(settings.permissions?.allow).toEqual( expect.arrayContaining([...AGENT_TEAMS_NAMESPACED_LEAD_BOOTSTRAP_TOOL_NAMES]) ); expect(settings.permissions?.allow).toContain('mcp__agent-teams__lead_briefing'); expect(settings.permissions?.allow).not.toContain('mcp__agent-teams__team_stop'); expect(settings.permissions?.allow).not.toContain('mcp__agent-teams__kanban_clear'); }); it('expands teammate permission suggestions to the operational tool set only', async () => { allowConsoleLogs(); const svc = new TeamProvisioningService({ getConfig: vi.fn(async () => ({ projectPath: tempClaudeRoot, members: [{ cwd: tempClaudeRoot }], })), } as any); await (svc as any).respondToTeammatePermission( { teamName: 'ops-team' }, 'alice', 'req-1', true, undefined, [ { type: 'addRules', behavior: 'allow', destination: 'localSettings', rules: [{ toolName: 'mcp__agent-teams__task_get' }], }, ] ); const settingsPath = path.join(tempClaudeRoot, '.claude', 'settings.local.json'); const settings = JSON.parse(fs.readFileSync(settingsPath, 'utf8')) as { permissions?: { allow?: string[] }; }; expect(settings.permissions?.allow).toEqual( expect.arrayContaining([...AGENT_TEAMS_NAMESPACED_TEAMMATE_OPERATIONAL_TOOL_NAMES]) ); expect(settings.permissions?.allow).not.toContain('mcp__agent-teams__team_stop'); expect(settings.permissions?.allow).not.toContain('mcp__agent-teams__kanban_clear'); }); it('does not broaden admin/runtime teammate permission suggestions', async () => { allowConsoleLogs(); const svc = new TeamProvisioningService({ getConfig: vi.fn(async () => ({ projectPath: tempClaudeRoot, members: [{ cwd: tempClaudeRoot }], })), } as any); await (svc as any).respondToTeammatePermission( { teamName: 'ops-team' }, 'alice', 'req-2', true, undefined, [ { type: 'addRules', behavior: 'allow', destination: 'localSettings', rules: [{ toolName: 'mcp__agent-teams__team_stop' }], }, ] ); const settingsPath = path.join(tempClaudeRoot, '.claude', 'settings.local.json'); const settings = JSON.parse(fs.readFileSync(settingsPath, 'utf8')) as { permissions?: { allow?: string[] }; }; expect(settings.permissions?.allow).toEqual(['mcp__agent-teams__team_stop']); }); it('uses a non-alarming cloud delay message before 2 minutes of silence', () => { const svc = new TeamProvisioningService(); expect((svc as any).buildStallProgressMessage(90, '1m 30s')).toBe( 'Waiting on Cloud response for 1m 30s — logs can be delayed, this is still OK' ); expect( (svc as any).buildStallWarningText(90, { request: { model: 'sonnet' }, }) ).toContain('Logs can sometimes show up after 1-1.5 minutes, and that is still okay.'); }); it('marks a cloud wait as unusual after 2 minutes of silence', () => { const svc = new TeamProvisioningService(); expect((svc as any).buildStallProgressMessage(120, '2m')).toBe( 'Still waiting on Cloud response for 2m — this is unusual' ); expect( (svc as any).buildStallWarningText(120, { request: { model: 'sonnet' }, }) ).toContain('but no logs for 2m is already unusual.'); }); it('formats AskUserQuestion approvals with readable question text', () => { const svc = new TeamProvisioningService(); expect( (svc as any).formatToolApprovalBody('AskUserQuestion', { questions: [ { question: 'Я испытываю технические трудности с отправкой сообщений с помощью инструмента `SendMessage`.', }, ], }) ).toBe( 'Question: Я испытываю технические трудности с отправкой сообщений с помощью инструмента `SendMessage`.' ); }); it('formats AskUserQuestion approvals with a compact multi-question summary', () => { const svc = new TeamProvisioningService(); expect( (svc as any).formatToolApprovalBody('AskUserQuestion', { questions: [ { question: ' First question with extra spacing. ' }, { question: 'Second question.' }, ], }) ).toBe('Questions (2): First question with extra spacing.'); }); it('skips --resume when the persisted launch state shows no teammate ever spawned', async () => { allowConsoleLogs(); const teamName = 'resume-skip-team'; const leadSessionId = 'lead-session-skip'; writeLaunchConfig(teamName, tempClaudeRoot, leadSessionId, ['alice', 'bob']); writeLaunchState(teamName, leadSessionId, { alice: { launchState: 'failed_to_start', }, bob: { launchState: 'starting', hardFailure: false, }, }); vi.mocked(ClaudeBinaryResolver.resolve).mockResolvedValue('/mock/claude'); vi.mocked(spawnCli).mockImplementation(() => { throw new Error('launch spawn EINVAL'); }); const svc = new TeamProvisioningService(undefined, undefined, undefined, undefined, { writeConfigFile: vi.fn(async () => '/mock/mcp-config-launch.json'), removeConfigFile: vi.fn(async () => {}), } as any); (svc as any).buildProvisioningEnv = vi.fn(async () => ({ env: { ANTHROPIC_API_KEY: 'test' }, authSource: 'anthropic_api_key', })); (svc as any).resolveLaunchExpectedMembers = vi.fn(async () => ({ members: [{ name: 'alice' }, { name: 'bob' }], source: 'members-meta', warning: undefined, })); (svc as any).normalizeTeamConfigForLaunch = vi.fn(async () => {}); (svc as any).assertConfigLeadOnlyForLaunch = vi.fn(async () => {}); (svc as any).updateConfigProjectPath = vi.fn(async () => {}); (svc as any).restorePrelaunchConfig = vi.fn(async () => {}); (svc as any).validateAgentTeamsMcpRuntime = vi.fn(async () => {}); (svc as any).pathExists = vi.fn(async (targetPath: string) => targetPath.endsWith(`${leadSessionId}.jsonl`) ); await expect(svc.launchTeam({ teamName, cwd: tempClaudeRoot }, () => {})).rejects.toThrow( 'launch spawn EINVAL' ); const launchArgs = vi.mocked(spawnCli).mock.calls[0]?.[1] as string[]; expect(launchArgs).toBeTruthy(); expect(launchArgs).not.toContain('--resume'); expect(launchArgs).not.toContain(leadSessionId); }); it('keeps --resume when a teammate had an accepted spawn before failing bootstrap', async () => { allowConsoleLogs(); const teamName = 'resume-keep-team'; const leadSessionId = 'lead-session-keep'; const acceptedAt = '2026-04-14T12:00:00.000Z'; writeLaunchConfig(teamName, tempClaudeRoot, leadSessionId, ['alice']); writeLaunchState(teamName, leadSessionId, { alice: { launchState: 'failed_to_start', agentToolAccepted: true, firstSpawnAcceptedAt: acceptedAt, hardFailureReason: 'Teammate did not join within the launch grace window.', }, }); vi.mocked(ClaudeBinaryResolver.resolve).mockResolvedValue('/mock/claude'); vi.mocked(spawnCli).mockImplementation(() => { throw new Error('launch spawn EINVAL'); }); const svc = new TeamProvisioningService(undefined, undefined, undefined, undefined, { writeConfigFile: vi.fn(async () => '/mock/mcp-config-launch.json'), removeConfigFile: vi.fn(async () => {}), } as any); (svc as any).buildProvisioningEnv = vi.fn(async () => ({ env: { ANTHROPIC_API_KEY: 'test' }, authSource: 'anthropic_api_key', })); (svc as any).resolveLaunchExpectedMembers = vi.fn(async () => ({ members: [{ name: 'alice' }], source: 'members-meta', warning: undefined, })); (svc as any).normalizeTeamConfigForLaunch = vi.fn(async () => {}); (svc as any).assertConfigLeadOnlyForLaunch = vi.fn(async () => {}); (svc as any).updateConfigProjectPath = vi.fn(async () => {}); (svc as any).restorePrelaunchConfig = vi.fn(async () => {}); (svc as any).validateAgentTeamsMcpRuntime = vi.fn(async () => {}); (svc as any).pathExists = vi.fn(async (targetPath: string) => targetPath.endsWith(`${leadSessionId}.jsonl`) ); await expect(svc.launchTeam({ teamName, cwd: tempClaudeRoot }, () => {})).rejects.toThrow( 'launch spawn EINVAL' ); const launchArgs = vi.mocked(spawnCli).mock.calls[0]?.[1] as string[]; expect(launchArgs).toContain('--resume'); expect(launchArgs).toContain(leadSessionId); }); it('keeps --resume when a persisted legacy Codex backend normalizes to codex-native', async () => { allowConsoleLogs(); const teamName = 'resume-backend-change-team'; const leadSessionId = 'lead-session-backend-change'; writeLaunchConfig(teamName, tempClaudeRoot, leadSessionId, ['alice']); vi.mocked(ClaudeBinaryResolver.resolve).mockResolvedValue('/mock/claude'); vi.mocked(spawnCli).mockImplementation(() => { throw new Error('launch spawn EINVAL'); }); const svc = new TeamProvisioningService(undefined, undefined, undefined, undefined, { writeConfigFile: vi.fn(async () => '/mock/mcp-config-launch.json'), removeConfigFile: vi.fn(async () => {}), } as any); (svc as any).buildProvisioningEnv = vi.fn(async () => ({ env: { CODEX_API_KEY: 'test' }, authSource: 'codex_runtime', })); (svc as any).resolveLaunchExpectedMembers = vi.fn(async () => ({ members: [{ name: 'alice' }], source: 'members-meta', warning: undefined, })); (svc as any).normalizeTeamConfigForLaunch = vi.fn(async () => {}); (svc as any).assertConfigLeadOnlyForLaunch = vi.fn(async () => {}); (svc as any).updateConfigProjectPath = vi.fn(async () => {}); (svc as any).restorePrelaunchConfig = vi.fn(async () => {}); (svc as any).validateAgentTeamsMcpRuntime = vi.fn(async () => {}); (svc as any).pathExists = vi.fn(async (targetPath: string) => targetPath.endsWith(`${leadSessionId}.jsonl`) ); (svc as any).teamMetaStore = { getMeta: vi.fn(async () => ({ providerBackendId: 'adapter' })), writeMeta: vi.fn(async () => {}), deleteMeta: vi.fn(async () => {}), }; await expect( svc.launchTeam( { teamName, cwd: tempClaudeRoot, providerId: 'codex', providerBackendId: 'codex-native', model: 'gpt-5.4', }, () => {} ) ).rejects.toThrow('launch spawn EINVAL'); const launchArgs = vi.mocked(spawnCli).mock.calls.at(-1)?.[1] as string[]; expect(launchArgs).toBeTruthy(); expect(launchArgs).toContain('--resume'); expect(launchArgs).toContain(leadSessionId); }); it('seeds the current lead session id immediately when launch resumes an existing session', async () => { allowConsoleLogs(); const teamName = 'resume-seed-session-team'; const leadSessionId = 'lead-session-seeded'; writeLaunchConfig(teamName, tempClaudeRoot, leadSessionId, ['alice']); vi.mocked(ClaudeBinaryResolver.resolve).mockResolvedValue('/mock/claude'); const child = createRunningChild(); vi.mocked(spawnCli).mockReturnValue(child as any); const svc = new TeamProvisioningService(undefined, undefined, undefined, undefined, { writeConfigFile: vi.fn(async () => '/mock/mcp-config-launch.json'), removeConfigFile: vi.fn(async () => {}), } as any); (svc as any).buildProvisioningEnv = vi.fn(async () => ({ env: { ANTHROPIC_API_KEY: 'test' }, authSource: 'anthropic_api_key', })); (svc as any).resolveLaunchExpectedMembers = vi.fn(async () => ({ members: [{ name: 'alice' }], source: 'members-meta', warning: undefined, })); (svc as any).normalizeTeamConfigForLaunch = vi.fn(async () => {}); (svc as any).assertConfigLeadOnlyForLaunch = vi.fn(async () => {}); (svc as any).updateConfigProjectPath = vi.fn(async () => {}); (svc as any).restorePrelaunchConfig = vi.fn(async () => {}); (svc as any).validateAgentTeamsMcpRuntime = vi.fn(async () => {}); (svc as any).persistLaunchStateSnapshot = vi.fn(async () => {}); (svc as any).startFilesystemMonitor = vi.fn(); (svc as any).pathExists = vi.fn(async (targetPath: string) => targetPath.endsWith(`${leadSessionId}.jsonl`) ); const { runId } = await svc.launchTeam({ teamName, cwd: tempClaudeRoot }, () => {}); expect(svc.getCurrentLeadSessionId(teamName)).toBe(leadSessionId); await svc.cancelProvisioning(runId); }); it('clears stale team-scoped transient state before starting a new launch run', async () => { allowConsoleLogs(); vi.useFakeTimers(); const teamName = 'launch-clears-stale-runtime-state'; const leadSessionId = 'lead-session-stale-state'; writeLaunchConfig(teamName, tempClaudeRoot, leadSessionId, ['alice']); vi.mocked(ClaudeBinaryResolver.resolve).mockResolvedValue('/mock/claude'); vi.mocked(spawnCli).mockImplementation(() => { throw new Error('launch spawn EINVAL'); }); const svc = new TeamProvisioningService(undefined, undefined, undefined, undefined, { writeConfigFile: vi.fn(async () => '/mock/mcp-config-launch.json'), removeConfigFile: vi.fn(async () => {}), } as any); (svc as any).buildProvisioningEnv = vi.fn(async () => ({ env: { ANTHROPIC_API_KEY: 'test' }, authSource: 'anthropic_api_key', })); (svc as any).resolveLaunchExpectedMembers = vi.fn(async () => ({ members: [{ name: 'alice' }], source: 'members-meta', warning: undefined, })); (svc as any).normalizeTeamConfigForLaunch = vi.fn(async () => {}); (svc as any).assertConfigLeadOnlyForLaunch = vi.fn(async () => {}); (svc as any).updateConfigProjectPath = vi.fn(async () => {}); (svc as any).restorePrelaunchConfig = vi.fn(async () => {}); (svc as any).validateAgentTeamsMcpRuntime = vi.fn(async () => {}); (svc as any).pathExists = vi.fn(async (targetPath: string) => targetPath.endsWith(`${leadSessionId}.jsonl`) ); const autoResumeProvisioning = { getCurrentRunId: vi.fn(() => 'run-1' as string | null), isTeamAlive: vi.fn(() => true), sendMessageToTeam: vi.fn(async () => undefined), }; initializeAutoResumeService(autoResumeProvisioning); const configManagerModule = await import('@main/services/infrastructure/ConfigManager'); const configManager = configManagerModule.ConfigManager.getInstance(); const actualConfig = configManager.getConfig(); const getConfigSpy = vi.spyOn(configManager, 'getConfig').mockImplementation( () => ({ ...actualConfig, notifications: { ...actualConfig.notifications, autoResumeOnRateLimit: true, }, }) as never ); try { getAutoResumeService().handleRateLimitMessage( teamName, "You've hit your limit. Resets in 5 minutes.", new Date('2026-04-17T12:00:00.000Z') ); (svc as any).relayedLeadInboxMessageIds.set(teamName, new Set(['stale-msg'])); (svc as any).liveLeadProcessMessages.set(teamName, [ { from: 'team-lead', text: 'Old transient message', timestamp: '2026-04-17T12:00:00.000Z', read: true, source: 'lead_process', messageId: 'lead-turn-old-run-1', }, ]); (svc as any).pendingTimeouts.set( `same-team-deferred:${teamName}`, setTimeout(() => undefined, 60_000) ); await expect(svc.launchTeam({ teamName, cwd: tempClaudeRoot }, () => {})).rejects.toThrow( 'launch spawn EINVAL' ); expect((svc as any).relayedLeadInboxMessageIds.has(teamName)).toBe(false); expect((svc as any).liveLeadProcessMessages.has(teamName)).toBe(false); expect((svc as any).pendingTimeouts.has(`same-team-deferred:${teamName}`)).toBe(false); await vi.advanceTimersByTimeAsync(5 * 60 * 1000 + 30 * 1000 + 100); expect(autoResumeProvisioning.sendMessageToTeam).not.toHaveBeenCalled(); } finally { getConfigSpy.mockRestore(); } }); it('marks persisted bootstrap as failed when member transcript shows an unsupported model error', async () => { allowConsoleLogs(); const teamName = 'zz-unit-bootstrap-unsupported-model'; const leadSessionId = 'lead-session'; const memberSessionId = 'jack-session'; const projectPath = '/Users/test/proj'; const projectId = '-Users-test-proj'; const acceptedAt = new Date(Date.now() - 5_000).toISOString(); const errorAt = new Date(Date.now() - 4_000).toISOString(); writeLaunchConfig(teamName, projectPath, leadSessionId, ['jack']); writeLaunchState(teamName, leadSessionId, { jack: { launchState: 'runtime_pending_bootstrap', agentToolAccepted: true, runtimeAlive: false, bootstrapConfirmed: false, hardFailure: false, hardFailureReason: undefined, firstSpawnAcceptedAt: acceptedAt, }, }); const projectRoot = path.join(tempProjectsBase, projectId); fs.mkdirSync(projectRoot, { recursive: true }); fs.writeFileSync( path.join(projectRoot, `${leadSessionId}.jsonl`), `${JSON.stringify({ timestamp: new Date(Date.now() - 10_000).toISOString(), teamName, type: 'user', message: { role: 'user', content: 'Lead bootstrap context' }, })}\n`, 'utf8' ); fs.writeFileSync( path.join(projectRoot, `${memberSessionId}.jsonl`), [ JSON.stringify({ timestamp: acceptedAt, teamName, agentName: 'jack', type: 'user', message: { role: 'user', content: `You are bootstrapping into team "${teamName}" as member "jack".`, }, }), JSON.stringify({ timestamp: errorAt, teamName, agentName: 'jack', type: 'assistant', isApiErrorMessage: true, message: { role: 'assistant', content: [ { type: 'text', text: `API Error: 400 {"type":"error","error":{"type":"api_error","message":"Codex API error (400): {\\"detail\\":\\"The 'gpt-5.2-codex' model is not supported when using Codex with a ChatGPT account.\\"}"}}`, }, ], }, }), ].join('\n') + '\n', 'utf8' ); const svc = new TeamProvisioningService(); const result = await svc.getMemberSpawnStatuses(teamName); expect(result.statuses.jack?.status).toBe('error'); expect(result.statuses.jack?.launchState).toBe('failed_to_start'); expect(result.statuses.jack?.error).toContain('gpt-5.2-codex'); expect(result.statuses.jack?.hardFailureReason).toContain('not supported'); expect(result.teamLaunchState).toBe('partial_failure'); }); it('marks an online teammate bootstrap as failed when transcript shows model unavailability', async () => { allowConsoleLogs(); const teamName = 'zz-live-bootstrap-model-unavailable'; const leadSessionId = 'lead-session'; const memberSessionId = 'jack-session'; const projectPath = '/Users/test/proj'; const projectId = '-Users-test-proj'; const acceptedAt = new Date(Date.now() - 5_000).toISOString(); const errorAt = new Date(Date.now() - 4_000).toISOString(); writeLaunchConfig(teamName, projectPath, leadSessionId, ['jack']); const projectRoot = path.join(tempProjectsBase, projectId); fs.mkdirSync(projectRoot, { recursive: true }); fs.writeFileSync( path.join(projectRoot, `${memberSessionId}.jsonl`), [ JSON.stringify({ timestamp: acceptedAt, teamName, agentName: 'jack', type: 'user', message: { role: 'user', content: `You are bootstrapping into team "${teamName}" as member "jack".`, }, }), JSON.stringify({ timestamp: errorAt, teamName, agentName: 'jack', type: 'assistant', isApiErrorMessage: true, message: { role: 'assistant', content: [ { type: 'text', text: 'API Error: 400 {"detail":"The requested model is not available for your account."}', }, ], }, }), ].join('\n') + '\n', 'utf8' ); const svc = new TeamProvisioningService(); const run = { runId: 'run-live-1', teamName, startedAt: new Date(Date.now() - 60_000).toISOString(), request: { members: [], }, expectedMembers: ['jack'], memberSpawnStatuses: new Map([ [ 'jack', { status: 'waiting', launchState: 'runtime_pending_bootstrap', error: undefined, updatedAt: acceptedAt, runtimeAlive: true, livenessSource: 'process', bootstrapConfirmed: false, hardFailure: false, agentToolAccepted: true, firstSpawnAcceptedAt: acceptedAt, lastHeartbeatAt: undefined, }, ], ]), provisioningOutputParts: [], activeToolCalls: new Map(), isLaunch: false, } as any; (svc as any).runs.set(run.runId, run); (svc as any).provisioningRunByTeam.set(teamName, run.runId); await (svc as any).reconcileBootstrapTranscriptFailures(run); expect(run.memberSpawnStatuses.get('jack')).toMatchObject({ status: 'error', launchState: 'failed_to_start', hardFailure: true, }); expect(run.memberSpawnStatuses.get('jack')?.error).toContain( 'requested model is not available' ); expect(run.provisioningOutputParts.join('\n')).toContain('requested model is not available'); }); it('marks a persisted online teammate bootstrap as failed when transcript shows model unavailability', async () => { allowConsoleLogs(); const teamName = 'zz-persisted-live-bootstrap-model-unavailable'; const leadSessionId = 'lead-session'; const memberSessionId = 'jack-session'; const projectPath = '/Users/test/proj'; const projectId = '-Users-test-proj'; const acceptedAt = new Date(Date.now() - 5_000).toISOString(); const errorAt = new Date(Date.now() - 4_000).toISOString(); writeLaunchConfig(teamName, projectPath, leadSessionId, ['jack']); writeLaunchState(teamName, leadSessionId, { jack: { launchState: 'runtime_pending_bootstrap', agentToolAccepted: true, runtimeAlive: false, bootstrapConfirmed: false, hardFailure: false, hardFailureReason: undefined, firstSpawnAcceptedAt: acceptedAt, }, }); const projectRoot = path.join(tempProjectsBase, projectId); fs.mkdirSync(projectRoot, { recursive: true }); fs.writeFileSync( path.join(projectRoot, `${memberSessionId}.jsonl`), [ JSON.stringify({ timestamp: acceptedAt, teamName, agentName: 'jack', type: 'user', message: { role: 'user', content: `You are bootstrapping into team "${teamName}" as member "jack".`, }, }), JSON.stringify({ timestamp: errorAt, teamName, agentName: 'jack', type: 'assistant', isApiErrorMessage: true, message: { role: 'assistant', content: [ { type: 'text', text: 'API Error: 400 {"detail":"The requested model is not available for your account."}', }, ], }, }), ].join('\n') + '\n', 'utf8' ); const svc = new TeamProvisioningService(); (svc as any).getLiveTeamAgentNames = vi.fn(() => new Set(['jack'])); const result = await svc.getMemberSpawnStatuses(teamName); expect(result.statuses.jack).toMatchObject({ status: 'error', launchState: 'failed_to_start', runtimeAlive: true, }); expect(result.statuses.jack?.error).toContain('requested model is not available'); expect(result.statuses.jack?.hardFailureReason).toContain('requested model is not available'); expect(result.teamLaunchState).toBe('partial_failure'); }); it('does not reprocess already-seen teammate lead inbox messages', async () => { const svc = new TeamProvisioningService(); const run = createMemberSpawnRun({ startedAt: '2026-04-16T09:00:00.000Z', memberSpawnLeadInboxCursorByMember: new Map([ [ 'alice', { timestamp: '2026-04-16T10:00:00.000Z', messageId: 'msg-2', }, ], ]), }); vi.spyOn((svc as any).inboxReader, 'getMessagesFor').mockResolvedValue([ { from: 'alice', text: 'heartbeat', timestamp: '2026-04-16T10:00:00.000Z', messageId: 'msg-1', read: false, }, { from: 'alice', text: 'heartbeat', timestamp: '2026-04-16T10:00:00.000Z', messageId: 'msg-2', read: false, }, ]); const applySignalSpy = vi.spyOn(svc as any, 'applyLeadInboxSpawnSignal'); await (svc as any).refreshMemberSpawnStatusesFromLeadInbox(run); expect(applySignalSpy).not.toHaveBeenCalled(); }); it('processes an unseen teammate heartbeat on the first refresh', async () => { const svc = new TeamProvisioningService(); const run = createMemberSpawnRun({ startedAt: '2026-04-16T09:00:00.000Z', }); vi.spyOn((svc as any).inboxReader, 'getMessagesFor').mockResolvedValue([ { from: 'alice', text: '{"type":"heartbeat","timestamp":"2026-04-16T10:00:00.000Z"}', timestamp: '2026-04-16T10:00:00.000Z', messageId: 'msg-1', read: false, }, ]); await (svc as any).refreshMemberSpawnStatusesFromLeadInbox(run); expect(run.memberSpawnStatuses.get('alice')).toMatchObject({ status: 'online', launchState: 'confirmed_alive', bootstrapConfirmed: true, hardFailure: false, lastHeartbeatAt: '2026-04-16T10:00:00.000Z', }); expect(run.memberSpawnLeadInboxCursorByMember.get('alice')).toEqual({ timestamp: '2026-04-16T10:00:00.000Z', messageId: 'msg-1', }); }); it('ignores teammate lead inbox signals that predate the current run', async () => { const svc = new TeamProvisioningService(); const run = createMemberSpawnRun({ startedAt: '2026-04-16T10:00:00.000Z', }); vi.spyOn((svc as any).inboxReader, 'getMessagesFor').mockResolvedValue([ { from: 'alice', text: '{"type":"heartbeat","timestamp":"2026-04-16T09:59:59.000Z"}', timestamp: '2026-04-16T09:59:59.000Z', messageId: 'msg-early', read: false, }, ]); const applySignalSpy = vi.spyOn(svc as any, 'applyLeadInboxSpawnSignal'); await (svc as any).refreshMemberSpawnStatusesFromLeadInbox(run); expect(applySignalSpy).not.toHaveBeenCalled(); expect(run.memberSpawnLeadInboxCursorByMember.size).toBe(0); expect(run.memberSpawnStatuses.get('alice')).toMatchObject({ status: 'waiting', launchState: 'runtime_pending_bootstrap', bootstrapConfirmed: false, }); }); it('ignores an unseen older lead inbox signal without replaying older state', async () => { const latestHeartbeatAt = '2026-04-16T10:05:00.000Z'; const existingEntry = createMemberSpawnStatusEntry({ status: 'online', launchState: 'confirmed_alive', runtimeAlive: true, livenessSource: 'heartbeat', bootstrapConfirmed: true, lastHeartbeatAt: latestHeartbeatAt, }); const run = createMemberSpawnRun({ startedAt: '2026-04-16T09:00:00.000Z', memberSpawnStatuses: new Map([['alice', existingEntry]]), memberSpawnLeadInboxCursorByMember: new Map([ [ 'alice', { timestamp: latestHeartbeatAt, messageId: 'msg-3', }, ], ]), }); const svc = new TeamProvisioningService(); vi.spyOn((svc as any).inboxReader, 'getMessagesFor').mockResolvedValue([ { from: 'alice', text: 'Bootstrap failed: unsupported model', timestamp: '2026-04-16T10:04:00.000Z', messageId: 'msg-2b', read: false, }, { from: 'alice', text: 'heartbeat', timestamp: latestHeartbeatAt, messageId: 'msg-3', read: false, }, ]); const applySignalSpy = vi.spyOn(svc as any, 'applyLeadInboxSpawnSignal'); await (svc as any).refreshMemberSpawnStatusesFromLeadInbox(run); expect(applySignalSpy).not.toHaveBeenCalled(); expect(run.memberSpawnStatuses.get('alice')).toBe(existingEntry); expect(run.memberSpawnLeadInboxCursorByMember.get('alice')).toEqual({ timestamp: latestHeartbeatAt, messageId: 'msg-3', }); }); it('applies an unseen newer failure signal and transitions the member to failed_to_start', async () => { const latestHeartbeatAt = '2026-04-16T10:00:00.000Z'; const run = createMemberSpawnRun({ startedAt: '2026-04-16T09:00:00.000Z', memberSpawnStatuses: new Map([ [ 'alice', createMemberSpawnStatusEntry({ status: 'online', launchState: 'confirmed_alive', runtimeAlive: true, livenessSource: 'heartbeat', bootstrapConfirmed: true, lastHeartbeatAt: latestHeartbeatAt, }), ], ]), memberSpawnLeadInboxCursorByMember: new Map([ [ 'alice', { timestamp: latestHeartbeatAt, messageId: 'msg-1', }, ], ]), }); const svc = new TeamProvisioningService(); vi.spyOn((svc as any).inboxReader, 'getMessagesFor').mockResolvedValue([ { from: 'alice', text: 'Bootstrap failed: unsupported model', timestamp: '2026-04-16T10:01:00.000Z', messageId: 'msg-2', read: false, }, ]); await (svc as any).refreshMemberSpawnStatusesFromLeadInbox(run); expect(run.memberSpawnStatuses.get('alice')).toMatchObject({ status: 'error', launchState: 'failed_to_start', hardFailure: true, hardFailureReason: 'Bootstrap failed: unsupported model', }); expect(run.memberSpawnLeadInboxCursorByMember.get('alice')).toEqual({ timestamp: '2026-04-16T10:01:00.000Z', messageId: 'msg-2', }); }); it('applies an unseen same-timestamp signal with a greater messageId and advances the cursor', async () => { const run = createMemberSpawnRun({ startedAt: '2026-04-16T09:00:00.000Z', memberSpawnLeadInboxCursorByMember: new Map([ [ 'alice', { timestamp: '2026-04-16T10:00:00.000Z', messageId: 'msg-2', }, ], ]), }); const svc = new TeamProvisioningService(); vi.spyOn((svc as any).inboxReader, 'getMessagesFor').mockResolvedValue([ { from: 'alice', text: 'heartbeat', timestamp: '2026-04-16T10:00:00.000Z', messageId: 'msg-2', read: false, }, { from: 'alice', text: 'heartbeat', timestamp: '2026-04-16T10:00:00.000Z', messageId: 'msg-3', read: false, }, ]); const applySignalSpy = vi.spyOn(svc as any, 'applyLeadInboxSpawnSignal'); await (svc as any).refreshMemberSpawnStatusesFromLeadInbox(run); expect(applySignalSpy).toHaveBeenCalledTimes(1); expect(applySignalSpy).toHaveBeenCalledWith( run, 'alice', expect.objectContaining({ messageId: 'msg-3' }) ); expect(run.memberSpawnLeadInboxCursorByMember.get('alice')).toEqual({ timestamp: '2026-04-16T10:00:00.000Z', messageId: 'msg-3', }); }); it('does not bump lastHeartbeatAt for an equal heartbeat timestamp', () => { const existingEntry = createMemberSpawnStatusEntry({ status: 'online', launchState: 'confirmed_alive', runtimeAlive: true, livenessSource: 'heartbeat', bootstrapConfirmed: true, lastHeartbeatAt: '2026-04-16T10:00:00.000Z', }); const run = createMemberSpawnRun({ memberSpawnStatuses: new Map([['alice', existingEntry]]), }); const svc = new TeamProvisioningService(); (svc as any).setMemberSpawnStatus( run, 'alice', 'online', undefined, 'heartbeat', '2026-04-16T10:00:00.000Z' ); expect(run.memberSpawnStatuses.get('alice')).toBe(existingEntry); }); it('does not bump lastHeartbeatAt for an older heartbeat timestamp', () => { const existingEntry = createMemberSpawnStatusEntry({ status: 'online', launchState: 'confirmed_alive', runtimeAlive: true, livenessSource: 'heartbeat', bootstrapConfirmed: true, lastHeartbeatAt: '2026-04-16T10:00:00.000Z', }); const run = createMemberSpawnRun({ memberSpawnStatuses: new Map([['alice', existingEntry]]), }); const svc = new TeamProvisioningService(); (svc as any).setMemberSpawnStatus( run, 'alice', 'online', undefined, 'heartbeat', '2026-04-16T09:59:59.000Z' ); expect(run.memberSpawnStatuses.get('alice')).toBe(existingEntry); }); it('treats duplicate_skipped already_running as process-confirmed online', () => { const run = createMemberSpawnRun(); run.activeToolCalls.set('tool-agent-1', { memberName: 'alice', toolUseId: 'tool-agent-1', toolName: 'Agent', preview: 'Spawn teammate alice', startedAt: new Date().toISOString(), state: 'running', source: 'runtime', }); run.memberSpawnToolUseIds.set('tool-agent-1', 'alice'); const svc = new TeamProvisioningService(); (svc as any).finishRuntimeToolActivity( run, 'tool-agent-1', [ { type: 'text', text: 'status: duplicate_skipped\nreason: already_running\nname: alice\nteam_name: nice-team', }, ], false ); expect(run.memberSpawnStatuses.get('alice')).toMatchObject({ status: 'online', launchState: 'runtime_pending_bootstrap', runtimeAlive: true, livenessSource: 'process', hardFailure: false, }); }); it('clears a pending restart when the teammate is confirmed online via process liveness', () => { const run = createMemberSpawnRun({ memberSpawnStatuses: new Map([ [ 'alice', createMemberSpawnStatusEntry({ status: 'waiting', launchState: 'runtime_pending_bootstrap', agentToolAccepted: true, firstSpawnAcceptedAt: new Date().toISOString(), }), ], ]), }); run.pendingMemberRestarts.set('alice', { requestedAt: new Date().toISOString(), desired: { name: 'alice', providerId: 'codex', model: 'gpt-5.4-mini', effort: 'medium', }, }); const svc = new TeamProvisioningService(); (svc as any).setMemberSpawnStatus(run, 'alice', 'online', undefined, 'process'); expect(run.pendingMemberRestarts.has('alice')).toBe(false); expect(run.memberSpawnStatuses.get('alice')).toMatchObject({ status: 'online', launchState: 'runtime_pending_bootstrap', runtimeAlive: true, livenessSource: 'process', }); }); it('treats deterministic already_running as a failed restart when a restart is pending', () => { const run = createMemberSpawnRun({ teamName: 'nice-team', expectedMembers: ['alice'], memberSpawnStatuses: new Map([ [ 'alice', createMemberSpawnStatusEntry({ status: 'waiting', launchState: 'runtime_pending_bootstrap', agentToolAccepted: true, firstSpawnAcceptedAt: new Date().toISOString(), }), ], ]), }); run.pendingMemberRestarts.set('alice', { requestedAt: new Date().toISOString(), desired: { name: 'alice', providerId: 'codex', model: 'gpt-5.4-mini', effort: 'medium', }, }); const svc = new TeamProvisioningService(); const handled = (svc as any).handleDeterministicBootstrapEvent(run, { type: 'system', subtype: 'team_bootstrap', event: 'member_spawn_result', member_name: 'alice', outcome: 'already_running', run_id: run.runId, team_name: run.teamName, seq: 1, }); expect(handled).toBe(true); expect(run.pendingMemberRestarts.has('alice')).toBe(false); expect(run.memberSpawnStatuses.get('alice')).toMatchObject({ status: 'error', launchState: 'failed_to_start', hardFailure: true, hardFailureReason: 'Restart for teammate "alice" was skipped because the previous runtime still appears to be active. The requested settings may not have been applied.', }); }); it('clears a pending restart when deterministic spawn reports a hard failure', () => { const run = createMemberSpawnRun({ teamName: 'nice-team', expectedMembers: ['alice'], memberSpawnStatuses: new Map([ [ 'alice', createMemberSpawnStatusEntry({ status: 'waiting', launchState: 'runtime_pending_bootstrap', agentToolAccepted: true, firstSpawnAcceptedAt: new Date().toISOString(), }), ], ]), }); run.pendingMemberRestarts.set('alice', { requestedAt: new Date().toISOString(), desired: { name: 'alice', providerId: 'codex', model: 'gpt-5.4-mini', effort: 'medium', }, }); const svc = new TeamProvisioningService(); const handled = (svc as any).handleDeterministicBootstrapEvent(run, { type: 'system', subtype: 'team_bootstrap', event: 'member_spawn_result', member_name: 'alice', outcome: 'failed', reason: 'spawn failed hard', run_id: run.runId, team_name: run.teamName, seq: 1, }); expect(handled).toBe(true); expect(run.pendingMemberRestarts.has('alice')).toBe(false); expect(run.memberSpawnStatuses.get('alice')).toMatchObject({ status: 'error', launchState: 'failed_to_start', hardFailure: true, hardFailureReason: 'spawn failed hard', }); }); it('clears stale failed_to_start state when live runtime metadata proves the teammate is alive', async () => { const svc = new TeamProvisioningService(); (svc as any).getLiveTeamAgentRuntimeMetadata = vi.fn(async () => new Map([ [ 'bob', { alive: true, model: 'gpt-5.2', }, ], ]) ); const result = await (svc as any).attachLiveRuntimeMetadataToStatuses('beacon-desk-4', { bob: createMemberSpawnStatusEntry({ status: 'error', launchState: 'failed_to_start', error: 'Teammate did not join within the launch grace window.', hardFailure: true, hardFailureReason: 'Teammate did not join within the launch grace window.', }), }); expect(result.bob).toMatchObject({ status: 'online', launchState: 'runtime_pending_bootstrap', runtimeAlive: true, hardFailure: false, hardFailureReason: undefined, error: undefined, runtimeModel: 'gpt-5.2', livenessSource: 'process', }); }); it('does not clear an explicit restart failure just because the old runtime is still alive', async () => { const svc = new TeamProvisioningService(); (svc as any).getLiveTeamAgentRuntimeMetadata = vi.fn(async () => new Map([ [ 'bob', { alive: true, model: 'gpt-5.3-codex', }, ], ]) ); const result = await (svc as any).attachLiveRuntimeMetadataToStatuses('beacon-desk-4', { bob: createMemberSpawnStatusEntry({ status: 'error', launchState: 'failed_to_start', error: 'Restart for teammate "bob" was skipped because the previous runtime still appears to be active. The requested settings may not have been applied.', hardFailure: true, hardFailureReason: 'Restart for teammate "bob" was skipped because the previous runtime still appears to be active. The requested settings may not have been applied.', }), }); expect(result.bob).toMatchObject({ status: 'error', launchState: 'failed_to_start', runtimeAlive: false, hardFailure: true, hardFailureReason: 'Restart for teammate "bob" was skipped because the previous runtime still appears to be active. The requested settings may not have been applied.', error: 'Restart for teammate "bob" was skipped because the previous runtime still appears to be active. The requested settings may not have been applied.', runtimeModel: 'gpt-5.3-codex', }); }); it('does not self-clear a failed launch from stale runtimeAlive state when no live pid exists', async () => { const svc = new TeamProvisioningService(); const run = createMemberSpawnRun({ runId: 'run-self-clear-1', teamName: 'beacon-desk-4', expectedMembers: ['bob'], memberSpawnStatuses: new Map([ [ 'bob', createMemberSpawnStatusEntry({ status: 'error', launchState: 'failed_to_start', runtimeAlive: true, livenessSource: 'process', bootstrapConfirmed: false, hardFailure: true, error: 'Teammate did not join within the launch grace window.', hardFailureReason: 'Teammate did not join within the launch grace window.', }), ], ]), }); (svc as any).runs.set(run.runId, run); (svc as any).provisioningRunByTeam.set(run.teamName, run.runId); (svc as any).configReader = { getConfig: vi.fn(async () => ({ name: 'Beacon Desk', members: [ { name: 'team-lead', agentType: 'team-lead' }, { name: 'bob', agentType: 'general-purpose', providerId: 'codex', model: 'gpt-5.3-codex' }, ], })), }; (svc as any).membersMetaStore = { getMembers: vi.fn(async () => [ { name: 'bob', role: 'Developer', providerId: 'codex', model: 'gpt-5.3-codex', effort: 'medium', agentType: 'general-purpose', }, ]), }; (svc as any).readPersistedRuntimeMembers = vi.fn(() => []); (svc as any).findLiveProcessPidByAgentId = vi.fn(() => new Map()); const result = await (svc as any).attachLiveRuntimeMetadataToStatuses('beacon-desk-4', { bob: createMemberSpawnStatusEntry({ status: 'error', launchState: 'failed_to_start', runtimeAlive: true, livenessSource: 'process', bootstrapConfirmed: false, hardFailure: true, error: 'Teammate did not join within the launch grace window.', hardFailureReason: 'Teammate did not join within the launch grace window.', }), }); expect(result.bob).toMatchObject({ status: 'error', launchState: 'failed_to_start', runtimeAlive: true, hardFailure: true, hardFailureReason: 'Teammate did not join within the launch grace window.', error: 'Teammate did not join within the launch grace window.', runtimeModel: 'gpt-5.3-codex', }); }); it('does not downgrade an already-online teammate when waiting is reported later', () => { const run = createMemberSpawnRun({ memberSpawnStatuses: new Map([ [ 'alice', createMemberSpawnStatusEntry({ status: 'online', launchState: 'confirmed_alive', runtimeAlive: true, livenessSource: 'heartbeat', bootstrapConfirmed: true, lastHeartbeatAt: '2026-04-16T10:00:00.000Z', }), ], ]), }); const svc = new TeamProvisioningService(); (svc as any).setMemberSpawnStatus(run, 'alice', 'waiting'); expect(run.memberSpawnStatuses.get('alice')).toMatchObject({ status: 'online', launchState: 'confirmed_alive', runtimeAlive: true, livenessSource: 'heartbeat', bootstrapConfirmed: true, lastHeartbeatAt: '2026-04-16T10:00:00.000Z', }); }); it('clears stale hard failure state when a new spawn attempt starts', () => { const staleAcceptedAt = '2026-04-16T10:00:00.000Z'; const run = createMemberSpawnRun({ memberSpawnStatuses: new Map([ [ 'alice', createMemberSpawnStatusEntry({ status: 'error', launchState: 'failed_to_start', error: 'Teammate was never spawned during launch.', hardFailure: true, hardFailureReason: 'Teammate was never spawned during launch.', runtimeAlive: true, bootstrapConfirmed: true, livenessSource: 'heartbeat', firstSpawnAcceptedAt: staleAcceptedAt, lastHeartbeatAt: staleAcceptedAt, }), ], ]), }); const svc = new TeamProvisioningService(); (svc as any).setMemberSpawnStatus(run, 'alice', 'spawning'); expect(run.memberSpawnStatuses.get('alice')).toMatchObject({ status: 'spawning', launchState: 'starting', error: undefined, hardFailure: false, hardFailureReason: undefined, agentToolAccepted: false, runtimeAlive: false, bootstrapConfirmed: false, livenessSource: undefined, firstSpawnAcceptedAt: undefined, lastHeartbeatAt: undefined, }); }); it('clears an old member launch grace timer when a new spawn attempt resets acceptance state', () => { vi.useFakeTimers(); const acceptedAt = new Date(Date.now() - 5_000).toISOString(); const run = createMemberSpawnRun({ memberSpawnStatuses: new Map([ [ 'alice', createMemberSpawnStatusEntry({ status: 'waiting', launchState: 'runtime_pending_bootstrap', agentToolAccepted: true, firstSpawnAcceptedAt: acceptedAt, }), ], ]), }); const svc = new TeamProvisioningService(); const timerKey = (svc as any).getMemberLaunchGraceKey(run, 'alice'); (svc as any).syncMemberLaunchGraceCheck(run, 'alice', run.memberSpawnStatuses.get('alice')); expect((svc as any).pendingTimeouts.has(timerKey)).toBe(true); (svc as any).setMemberSpawnStatus(run, 'alice', 'offline'); expect((svc as any).pendingTimeouts.has(timerKey)).toBe(false); (svc as any).setMemberSpawnStatus(run, 'alice', 'spawning'); expect((svc as any).pendingTimeouts.has(timerKey)).toBe(false); expect(run.memberSpawnStatuses.get('alice')).toMatchObject({ firstSpawnAcceptedAt: undefined, lastHeartbeatAt: undefined, error: undefined, hardFailureReason: undefined, livenessSource: undefined, }); }); it('reconciles stale never-spawned failures when bootstrap state proves the teammate was registered', async () => { const teamName = 'registered-bootstrap-team'; const leadSessionId = 'lead-session'; const acceptedAt = new Date(Date.now() - 60_000).toISOString(); writeLaunchConfig(teamName, '/Users/test/proj', leadSessionId, ['alice']); writeLaunchState(teamName, leadSessionId, { alice: { launchState: 'failed_to_start', agentToolAccepted: false, runtimeAlive: false, bootstrapConfirmed: false, hardFailure: true, hardFailureReason: 'Teammate was never spawned during launch.', }, }); writeBootstrapState( teamName, [ { name: 'alice', status: 'registered', lastAttemptAt: Date.parse(acceptedAt), lastObservedAt: Date.parse(acceptedAt), }, ], new Date(Date.now() - 30_000).toISOString() ); const svc = new TeamProvisioningService(); const result = await svc.getMemberSpawnStatuses(teamName); expect(result.statuses.alice).toMatchObject({ status: 'waiting', launchState: 'runtime_pending_bootstrap', hardFailure: false, hardFailureReason: undefined, agentToolAccepted: true, }); }); });