From c033a0cb87b0aad0d7e610900e1f64390f4be0aa Mon Sep 17 00:00:00 2001 From: 777genius Date: Mon, 25 May 2026 14:53:05 +0300 Subject: [PATCH] fix(team): persist incomplete launch state before cleanup --- .../services/team/TeamProvisioningService.ts | 90 ++++++++--- .../team/TeamProvisioningService.test.ts | 151 ++++++++++++++++++ 2 files changed, 215 insertions(+), 26 deletions(-) diff --git a/src/main/services/team/TeamProvisioningService.ts b/src/main/services/team/TeamProvisioningService.ts index e3838b26..9da14501 100644 --- a/src/main/services/team/TeamProvisioningService.ts +++ b/src/main/services/team/TeamProvisioningService.ts @@ -1957,6 +1957,7 @@ interface ProvisioningRun { isLaunch: boolean; launchStateClearedForRun: boolean; deterministicBootstrap: boolean; + launchCleanupStateFinalized?: boolean; workspaceTrustPlan?: WorkspaceTrustFullPlanResult | null; workspaceTrustExecution?: WorkspaceTrustExecutionResult | null; workspaceTrustDiagnostics?: WorkspaceTrustDiagnosticsManifest | null; @@ -33269,6 +33270,65 @@ export class TeamProvisioningService { this.pendingTimeouts.set(key, timer); } + private shouldFinalizeIncompleteLaunchState(run: ProvisioningRun): boolean { + return ( + run.isLaunch && + run.launchStateClearedForRun !== false && + !run.provisioningComplete && + !run.cancelRequested && + run.launchCleanupStateFinalized !== true + ); + } + + private buildIncompleteLaunchCleanupReason( + run: ProvisioningRun, + fallback = 'Launch ended before teammate bootstrap completed.' + ): string { + return typeof run.progress.error === 'string' && run.progress.error.trim() + ? run.progress.error.trim() + : run.progress.state === 'failed' && run.progress.message.trim() + ? run.progress.message.trim() + : fallback; + } + + private markIncompleteLaunchStateFinalized(run: ProvisioningRun, cleanupReason: string): void { + logger.warn(`[${run.teamName}] Launch cleanup finalizing unconfirmed bootstrap members`, { + runId: run.runId, + progressState: run.progress.state, + progressMessage: run.progress.message, + progressError: run.progress.error ?? null, + cleanupReason, + unconfirmedMembers: this.getUnconfirmedBootstrapMemberNames(run), + ...this.buildStdoutCarryDiagnostic(run), + }); + this.markUnconfirmedBootstrapMembersFailed(run, cleanupReason, { + cleanupRequested: true, + preserveExistingFailure: true, + }); + run.launchCleanupStateFinalized = true; + } + + private async finalizeIncompleteLaunchStateBeforeCleanup( + run: ProvisioningRun, + fallbackReason?: string + ): Promise { + if (!this.shouldFinalizeIncompleteLaunchState(run)) { + return; + } + const cleanupReason = this.buildIncompleteLaunchCleanupReason(run, fallbackReason); + this.markIncompleteLaunchStateFinalized(run, cleanupReason); + try { + await this.persistLaunchStateSnapshot(run, 'finished'); + } catch (error) { + run.launchCleanupStateFinalized = false; + logger.warn( + `[${run.teamName}] Failed to finalize launch state before cleanup: ${ + error instanceof Error ? error.message : String(error) + }` + ); + } + } + /** * Remove a run from tracking maps. */ @@ -33281,32 +33341,9 @@ export class TeamProvisioningService { peekAutoResumeService()?.cancelPendingAutoResume(run.teamName); } - if ( - !hasNewerTrackedRun && - run.isLaunch && - run.launchStateClearedForRun !== false && - !run.provisioningComplete && - !run.cancelRequested - ) { - const cleanupReason = - typeof run.progress.error === 'string' && run.progress.error.trim() - ? run.progress.error.trim() - : run.progress.state === 'failed' && run.progress.message.trim() - ? run.progress.message.trim() - : 'Launch ended before teammate bootstrap completed.'; - logger.warn(`[${run.teamName}] Launch cleanup finalizing unconfirmed bootstrap members`, { - runId: run.runId, - progressState: run.progress.state, - progressMessage: run.progress.message, - progressError: run.progress.error ?? null, - cleanupReason, - unconfirmedMembers: this.getUnconfirmedBootstrapMemberNames(run), - ...this.buildStdoutCarryDiagnostic(run), - }); - this.markUnconfirmedBootstrapMembersFailed(run, cleanupReason, { - cleanupRequested: true, - preserveExistingFailure: true, - }); + if (!hasNewerTrackedRun && this.shouldFinalizeIncompleteLaunchState(run)) { + const cleanupReason = this.buildIncompleteLaunchCleanupReason(run); + this.markIncompleteLaunchStateFinalized(run, cleanupReason); void this.persistLaunchStateSnapshot(run, 'finished'); } if ( @@ -33720,6 +33757,7 @@ export class TeamProvisioningService { cliLogsTail: extractCliLogsFromRun(run), } ); + await this.finalizeIncompleteLaunchStateBeforeCleanup(run, warnings[0]); run.onProgress(progress); this.cleanupRun(run); return; diff --git a/test/main/services/team/TeamProvisioningService.test.ts b/test/main/services/team/TeamProvisioningService.test.ts index 17180495..7b43ddb7 100644 --- a/test/main/services/team/TeamProvisioningService.test.ts +++ b/test/main/services/team/TeamProvisioningService.test.ts @@ -19247,6 +19247,157 @@ describe('TeamProvisioningService', () => { expect(progressStates).not.toContain('verifying'); }); + it('clears lead-only bootstrap state before cleanup when deterministic launch process exits', async () => { + allowConsoleLogs(); + const teamName = 'lead-only-launch-exit-clears-bootstrap-state'; + const leadSessionId = 'lead-session-lead-only-exit'; + writeLaunchConfig(teamName, tempClaudeRoot, leadSessionId, []); + + vi.mocked(ClaudeBinaryResolver.resolve).mockResolvedValue('/mock/claude'); + const child = createRunningChild(); + vi.mocked(spawnCli).mockReturnValue(child as any); + + const svc = new TeamProvisioningService(undefined, undefined, undefined, undefined, { + writeConfigFile: vi.fn(async () => '/mock/mcp-config-launch.json'), + removeConfigFile: vi.fn(async () => {}), + } as any); + (svc as any).buildProvisioningEnv = vi.fn(async () => ({ + env: { ANTHROPIC_API_KEY: 'test' }, + authSource: 'anthropic_api_key', + })); + (svc as any).normalizeTeamConfigForLaunch = vi.fn(async () => {}); + (svc as any).assertConfigLeadOnlyForLaunch = vi.fn(async () => {}); + (svc as any).updateConfigProjectPath = vi.fn(async () => {}); + (svc as any).restorePrelaunchConfig = vi.fn(async () => {}); + (svc as any).validateAgentTeamsMcpRuntime = vi.fn(async () => {}); + (svc as any).startFilesystemMonitor = vi.fn(); + (svc as any).writeLaunchFailureArtifactPackBestEffort = vi.fn(); + vi.spyOn(svc as any, 'waitForValidConfig').mockResolvedValue({ + ok: true, + location: 'configured', + configPath: path.join(tempTeamsBase, teamName, 'config.json'), + }); + vi.spyOn(svc as any, 'waitForTeamInList').mockResolvedValue(true); + (svc as any).pathExists = vi.fn(async (targetPath: string) => + targetPath.endsWith(`${leadSessionId}.jsonl`) + ); + + const progressStates: string[] = []; + const { runId } = await svc.launchTeam({ teamName, cwd: tempClaudeRoot }, (progress) => { + progressStates.push(progress.state); + }); + + fs.writeFileSync( + getTeamBootstrapStatePath(teamName), + `${JSON.stringify( + { + version: 1, + runId, + teamName, + ownerPid: child.pid, + startedAt: Date.now(), + updatedAt: Date.now(), + phase: 'auditing_truth', + members: [], + }, + null, + 2 + )}\n`, + 'utf8' + ); + + child.emit('close', 1); + + await vi.waitFor(() => expect(progressStates).toContain('disconnected')); + expect(fs.existsSync(getTeamBootstrapStatePath(teamName))).toBe(false); + expect(fs.existsSync(getTeamLaunchStatePath(teamName))).toBe(false); + expect(fs.existsSync(getTeamLaunchSummaryPath(teamName))).toBe(false); + }); + + it('persists failed member launch state before cleanup when deterministic launch process exits', async () => { + allowConsoleLogs(); + const teamName = 'member-launch-exit-finalizes-before-cleanup'; + const leadSessionId = 'lead-session-member-exit'; + writeLaunchConfig(teamName, tempClaudeRoot, leadSessionId, ['alice']); + + vi.mocked(ClaudeBinaryResolver.resolve).mockResolvedValue('/mock/claude'); + const child = createRunningChild(); + vi.mocked(spawnCli).mockReturnValue(child as any); + + const svc = new TeamProvisioningService(undefined, undefined, undefined, undefined, { + writeConfigFile: vi.fn(async () => '/mock/mcp-config-launch.json'), + removeConfigFile: vi.fn(async () => {}), + } as any); + (svc as any).buildProvisioningEnv = vi.fn(async () => ({ + env: { ANTHROPIC_API_KEY: 'test' }, + authSource: 'anthropic_api_key', + })); + (svc as any).resolveLaunchExpectedMembers = vi.fn(async () => ({ + members: [{ name: 'alice' }], + source: 'members-meta', + warning: undefined, + })); + (svc as any).normalizeTeamConfigForLaunch = vi.fn(async () => {}); + (svc as any).assertConfigLeadOnlyForLaunch = vi.fn(async () => {}); + (svc as any).updateConfigProjectPath = vi.fn(async () => {}); + (svc as any).restorePrelaunchConfig = vi.fn(async () => {}); + (svc as any).validateAgentTeamsMcpRuntime = vi.fn(async () => {}); + (svc as any).startFilesystemMonitor = vi.fn(); + (svc as any).writeLaunchFailureArtifactPackBestEffort = vi.fn(); + vi.spyOn(svc as any, 'waitForValidConfig').mockResolvedValue({ + ok: true, + location: 'configured', + configPath: path.join(tempTeamsBase, teamName, 'config.json'), + }); + vi.spyOn(svc as any, 'waitForTeamInList').mockResolvedValue(true); + (svc as any).pathExists = vi.fn(async (targetPath: string) => { + const basename = path.basename(targetPath); + return basename === `${leadSessionId}.jsonl` || basename === 'alice.json'; + }); + + const progressStates: string[] = []; + const { runId } = await svc.launchTeam({ teamName, cwd: tempClaudeRoot }, (progress) => { + progressStates.push(progress.state); + }); + + fs.writeFileSync( + getTeamBootstrapStatePath(teamName), + `${JSON.stringify( + { + version: 1, + runId, + teamName, + ownerPid: 987654321, + startedAt: Date.now(), + updatedAt: Date.now(), + phase: 'auditing_truth', + members: [{ name: 'alice', status: 'registered', lastAttemptAt: Date.now() }], + }, + null, + 2 + )}\n`, + 'utf8' + ); + + child.emit('close', 1); + + await vi.waitFor(() => expect(progressStates).toContain('disconnected')); + + const persisted = JSON.parse(fs.readFileSync(getTeamLaunchStatePath(teamName), 'utf8')) as { + teamLaunchState?: string; + members?: Record; + }; + expect(persisted.teamLaunchState).toBe('partial_failure'); + expect(persisted.members?.alice?.launchState).toBe('failed_to_start'); + expect(persisted.members?.alice?.hardFailureReason).toContain( + 'team provisioned but not alive' + ); + + const reconciled = await (svc as any).reconcilePersistedLaunchState(teamName); + expect(reconciled.snapshot?.teamLaunchState).toBe('partial_failure'); + expect(reconciled.statuses.alice?.launchState).toBe('failed_to_start'); + }); + it('does not verify provisioning while auth retry is scheduled from final newline-less output', async () => { allowConsoleLogs(); const teamName = 'launch-close-flushes-final-auth-team';