diff --git a/src/main/services/team/TeamProvisioningService.ts b/src/main/services/team/TeamProvisioningService.ts index c5d5edf5..a0a175dc 100644 --- a/src/main/services/team/TeamProvisioningService.ts +++ b/src/main/services/team/TeamProvisioningService.ts @@ -2293,6 +2293,105 @@ function hasRealOpenCodeFailureDiagnostic(text: string): boolean { ); } +const OPEN_CODE_GENERIC_MEMBER_LAUNCH_FAILURE_REASON = + 'OpenCode bridge reported member launch failure'; +const OPEN_CODE_SECRET_FLAG_PATTERN = + /(--(?:api-key|token|password|secret|authorization|auth-token)(?:=|\s+))("[^"]*"|'[^']*'|\S+)/gi; +const OPEN_CODE_BEARER_TOKEN_PATTERN = /\bBearer\s+[A-Za-z0-9._~+/=-]+/gi; +const OPEN_CODE_SECRET_KEY_PATTERN = /\bsk-[A-Za-z0-9_-]{16,}\b/g; + +function normalizeOpenCodePersistedFailureReason(value: string | undefined): string | undefined { + const trimmed = value?.replace(/\s+/g, ' ').trim(); + if (!trimmed) { + return undefined; + } + return trimmed + .replace(OPEN_CODE_SECRET_FLAG_PATTERN, '$1[redacted]') + .replace(OPEN_CODE_BEARER_TOKEN_PATTERN, 'Bearer [redacted]') + .replace(OPEN_CODE_SECRET_KEY_PATTERN, '[redacted-api-key]'); +} + +function isGenericOpenCodePersistedFailureReason(value: string | undefined): boolean { + const normalized = normalizeOpenCodePersistedFailureReason(value); + return ( + normalized === OPEN_CODE_GENERIC_MEMBER_LAUNCH_FAILURE_REASON || + normalized?.startsWith(`${OPEN_CODE_GENERIC_MEMBER_LAUNCH_FAILURE_REASON}:`) === true || + normalized?.startsWith('OpenCode secondary lane timing:') === true || + normalized?.startsWith( + 'OpenCode bridge reported ready without all required durable checkpoints:' + ) === true || + normalized?.startsWith( + 'OpenCode bridge reported ready before all expected members were confirmed:' + ) === true || + normalized?.startsWith( + 'OpenCode bootstrap MCP did not complete required tools before assistant response:' + ) === true || + normalized?.startsWith('info:opencode_launch_member_timing:') === true || + normalized?.startsWith('info:opencode_launch_total_timing:') === true + ); +} + +function selectOpenCodePersistedFailureReasonFromDiagnostics( + member: PersistedTeamLaunchMemberState +): string | undefined { + if (!isPersistedOpenCodeSecondaryLaneMember(member)) { + return undefined; + } + if (member.launchState !== 'failed_to_start' || member.hardFailure !== true) { + return undefined; + } + if (!isGenericOpenCodePersistedFailureReason(member.hardFailureReason)) { + return undefined; + } + for (const value of member.diagnostics ?? []) { + const normalized = normalizeOpenCodePersistedFailureReason(value); + if (!normalized || isGenericOpenCodePersistedFailureReason(normalized)) { + continue; + } + return normalized; + } + return undefined; +} + +function promoteOpenCodePersistedFailureReasonsFromDiagnostics( + snapshot: PersistedTeamLaunchSnapshot | null +): PersistedTeamLaunchSnapshot | null { + if (!snapshot) { + return null; + } + let changed = false; + const members: Record = { ...snapshot.members }; + for (const [memberName, member] of Object.entries(snapshot.members)) { + const promotedReason = selectOpenCodePersistedFailureReasonFromDiagnostics(member); + if (!promotedReason || promotedReason === member.hardFailureReason) { + continue; + } + members[memberName] = { + ...member, + hardFailureReason: promotedReason, + runtimeDiagnostic: + member.runtimeDiagnostic && + !isGenericOpenCodePersistedFailureReason(member.runtimeDiagnostic) + ? member.runtimeDiagnostic + : promotedReason, + runtimeDiagnosticSeverity: member.runtimeDiagnosticSeverity ?? 'error', + }; + changed = true; + } + if (!changed) { + return snapshot; + } + return createPersistedLaunchSnapshot({ + teamName: snapshot.teamName, + expectedMembers: snapshot.expectedMembers, + bootstrapExpectedMembers: snapshot.bootstrapExpectedMembers, + leadSessionId: snapshot.leadSessionId, + launchPhase: snapshot.launchPhase, + members, + updatedAt: nowIso(), + }); +} + function promoteOpenCodeSecondaryMemberFromCommittedBootstrapEvidence(input: { current: PersistedTeamLaunchMemberState; previous: PersistedTeamLaunchMemberState | null; @@ -19770,6 +19869,99 @@ export class TeamProvisioningService { return statuses; } + private async overlayPrimaryBootstrapTruthIntoRunStatusesFromBootstrapState( + run: ProvisioningRun + ): Promise { + if ( + !run.isLaunch || + !Array.isArray(run.mixedSecondaryLanes) || + run.mixedSecondaryLanes.length === 0 + ) { + return; + } + + let bootstrapSnapshot: PersistedTeamLaunchSnapshot | null = null; + try { + bootstrapSnapshot = await readBootstrapLaunchSnapshot(run.teamName); + } catch { + return; + } + if (!bootstrapSnapshot) { + return; + } + + const runStartedAtMs = Date.parse(run.startedAt); + const bootstrapUpdatedAtMs = Date.parse(bootstrapSnapshot.updatedAt); + if ( + !Number.isFinite(runStartedAtMs) || + !Number.isFinite(bootstrapUpdatedAtMs) || + bootstrapUpdatedAtMs < runStartedAtMs + ) { + return; + } + + const primaryMemberNames = new Set( + (run.effectiveMembers ?? []) + .map((member) => member.name?.trim()) + .filter((name): name is string => Boolean(name)) + ); + if (primaryMemberNames.size === 0) { + return; + } + + const updatedAt = nowIso(); + for (const memberName of primaryMemberNames) { + if (this.isOpenCodeSecondaryLaneMemberInRun(run, memberName)) { + continue; + } + const bootstrapMember = bootstrapSnapshot.members[memberName]; + if (bootstrapMember?.bootstrapConfirmed !== true) { + continue; + } + const current = + run.memberSpawnStatuses.get(memberName) ?? createInitialMemberSpawnStatusEntry(); + if (current.launchState === 'skipped_for_launch' || current.skippedForLaunch === true) { + continue; + } + const failureReason = current.hardFailureReason ?? current.error; + if ( + current.launchState === 'failed_to_start' && + !isAutoClearableLaunchFailureReason(failureReason) + ) { + continue; + } + + const observedAt = + bootstrapMember.lastHeartbeatAt ?? + bootstrapMember.lastEvaluatedAt ?? + bootstrapSnapshot.updatedAt ?? + updatedAt; + const next: MemberSpawnStatusEntry = { + ...current, + status: 'online', + updatedAt, + agentToolAccepted: true, + runtimeAlive: true, + bootstrapConfirmed: true, + hardFailure: false, + bootstrapStalled: undefined, + error: undefined, + hardFailureReason: undefined, + livenessSource: current.livenessSource ?? 'heartbeat', + firstSpawnAcceptedAt: + current.firstSpawnAcceptedAt ?? bootstrapMember.firstSpawnAcceptedAt ?? observedAt, + lastHeartbeatAt: isMemberSpawnHeartbeatTimestampNewer(current.lastHeartbeatAt, observedAt) + ? observedAt + : current.lastHeartbeatAt, + livenessLastCheckedAt: updatedAt, + launchState: 'confirmed_alive', + }; + run.memberSpawnStatuses.set(memberName, next); + run.pendingMemberRestarts?.delete(memberName); + this.syncMemberLaunchGraceCheck(run, memberName, next); + } + } + private syncRunMemberSpawnStatusesFromSnapshot( run: ProvisioningRun, snapshot: PersistedTeamLaunchSnapshot @@ -20191,6 +20383,7 @@ export class TeamProvisioningService { run: ProvisioningRun, launchPhase: 'active' | 'finished' | 'reconciled' ): Promise { + await this.overlayPrimaryBootstrapTruthIntoRunStatusesFromBootstrapState(run); const snapshot = this.buildLiveLaunchSnapshotForRun(run, launchPhase); if (!snapshot) { if (run.isLaunch) { @@ -21209,7 +21402,7 @@ export class TeamProvisioningService { metaMembers, }) : null; - const stableRecoveredMixedSnapshot = + const stableRecoveredMixedSnapshotWithCommittedEvidence = overlaidRecoveredMixedSnapshot && this.hasCommittedOpenCodeSecondaryEvidenceOverlayDelta( overlaidRecoveredMixedSnapshot, @@ -21217,6 +21410,14 @@ export class TeamProvisioningService { ) ? await this.writeLaunchStateSnapshot(teamName, overlaidRecoveredMixedSnapshot) : overlaidRecoveredMixedSnapshot; + const promotedRecoveredMixedSnapshot = promoteOpenCodePersistedFailureReasonsFromDiagnostics( + stableRecoveredMixedSnapshotWithCommittedEvidence + ); + const stableRecoveredMixedSnapshot = + promotedRecoveredMixedSnapshot && + promotedRecoveredMixedSnapshot !== stableRecoveredMixedSnapshotWithCommittedEvidence + ? await this.writeLaunchStateSnapshot(teamName, promotedRecoveredMixedSnapshot) + : promotedRecoveredMixedSnapshot; const filteredBootstrapSnapshot = bootstrapSnapshot ? this.filterRemovedMembersFromLaunchSnapshot(bootstrapSnapshot, metaMembers) : null; @@ -21248,15 +21449,46 @@ export class TeamProvisioningService { : null; const shouldPersistCommittedEvidenceOverlay = this.hasCommittedOpenCodeSecondaryEvidenceOverlayDelta(filteredPersisted, persisted); + const promotedPersisted = + promoteOpenCodePersistedFailureReasonsFromDiagnostics(filteredPersisted); + const shouldPersistFailureReasonPromotion = promotedPersisted !== filteredPersisted; const persistedWithCommittedEvidence = - filteredPersisted && shouldPersistCommittedEvidenceOverlay - ? await this.writeLaunchStateSnapshot(teamName, filteredPersisted) - : filteredPersisted; + promotedPersisted && + (shouldPersistCommittedEvidenceOverlay || shouldPersistFailureReasonPromotion) + ? await this.writeLaunchStateSnapshot(teamName, promotedPersisted) + : promotedPersisted; const preferredSnapshot = choosePreferredLaunchSnapshot( overlaidBootstrapSnapshot, persistedWithCommittedEvidence ); - if (preferredSnapshot && preferredSnapshot === overlaidBootstrapSnapshot) { + const bootstrapSelectionWouldCollapseMixedLaunch = + preferredSnapshot && + preferredSnapshot === overlaidBootstrapSnapshot && + preferredSnapshot.teamLaunchState === 'clean_success' && + !this.hasMixedLaunchMetadata(preferredSnapshot) && + this.hasMixedLaunchMetadata(persistedWithCommittedEvidence); + if ( + preferredSnapshot && + preferredSnapshot === overlaidBootstrapSnapshot && + !bootstrapSelectionWouldCollapseMixedLaunch + ) { + if (persistedWithCommittedEvidence) { + if ( + preferredSnapshot.teamLaunchState === 'clean_success' && + !this.hasMixedLaunchMetadata(preferredSnapshot) + ) { + await this.clearPersistedLaunchState(teamName); + return { + snapshot: preferredSnapshot, + statuses: snapshotToMemberSpawnStatuses(preferredSnapshot), + }; + } + const writtenSnapshot = await this.writeLaunchStateSnapshot(teamName, preferredSnapshot); + return { + snapshot: writtenSnapshot, + statuses: snapshotToMemberSpawnStatuses(writtenSnapshot), + }; + } return { snapshot: preferredSnapshot, statuses: snapshotToMemberSpawnStatuses(preferredSnapshot), diff --git a/src/main/services/team/runtime/OpenCodeTeamRuntimeAdapter.ts b/src/main/services/team/runtime/OpenCodeTeamRuntimeAdapter.ts index fab6fbc5..85eaf06d 100644 --- a/src/main/services/team/runtime/OpenCodeTeamRuntimeAdapter.ts +++ b/src/main/services/team/runtime/OpenCodeTeamRuntimeAdapter.ts @@ -78,6 +78,11 @@ const REQUIRED_READY_CHECKPOINTS = new Set([ 'member_ready', 'run_ready', ]); +const GENERIC_OPEN_CODE_MEMBER_FAILURE_REASON = 'OpenCode bridge reported member launch failure'; +const SECRET_FLAG_PATTERN = + /(--(?:api-key|token|password|secret|authorization|auth-token)(?:=|\s+))("[^"]*"|'[^']*'|\S+)/gi; +const BEARER_TOKEN_PATTERN = /\bBearer\s+[A-Za-z0-9._~+/=-]+/gi; +const SECRET_KEY_PATTERN = /\bsk-[A-Za-z0-9_-]{16,}\b/g; export class OpenCodeTeamRuntimeAdapter implements TeamLaunchRuntimeAdapter { readonly providerId = 'opencode' as const; @@ -484,6 +489,23 @@ function mapOpenCodeLaunchDataToRuntimeResult( : data.teamLaunchState === 'failed' ? 'failed' : 'created'; + const checkpointDiagnosticsForMember = [ + ...checkpointDiagnostic, + ...(missingExpectedMembers.includes(member.name) ? incompleteReadyDiagnostic : []), + ]; + const memberDiagnostics = [ + ...(bridgeMember + ? [] + : [ + `OpenCode bridge response did not include ${member.name}; keeping the member pending until lane state materializes.`, + ]), + ...(bridgeMember?.diagnostics ?? []), + ...(bridgeMember?.evidence ?? []).map( + (evidence) => `${evidence.kind} at ${evidence.observedAt}` + ), + ...memberBridgeDiagnostics, + ...checkpointDiagnosticsForMember, + ]; return [ member.name, mapBridgeMemberToRuntimeEvidence( @@ -493,20 +515,13 @@ function mapOpenCodeLaunchDataToRuntimeResult( bridgeMember?.runtimePid, bridgeMember?.pendingPermissionRequestIds, bridgeMember != null, - [ - ...(bridgeMember - ? [] - : [ - `OpenCode bridge response did not include ${member.name}; keeping the member pending until lane state materializes.`, - ]), - ...(bridgeMember?.diagnostics ?? []), - ...(bridgeMember?.evidence ?? []).map( - (evidence) => `${evidence.kind} at ${evidence.observedAt}` - ), - ...memberBridgeDiagnostics, - ...checkpointDiagnostic, - ...(missingExpectedMembers.includes(member.name) ? incompleteReadyDiagnostic : []), - ] + memberDiagnostics, + selectOpenCodeMemberFailureReason({ + memberDiagnostics: bridgeMember?.diagnostics ?? [], + bridgeDiagnostics: data.diagnostics, + checkpointDiagnostics: checkpointDiagnosticsForMember, + fallback: GENERIC_OPEN_CODE_MEMBER_FAILURE_REASON, + }) ), ]; }) @@ -542,7 +557,8 @@ function mapBridgeMemberToRuntimeEvidence( runtimePid: number | undefined, pendingPermissionRequestIds: string[] | undefined, runtimeMaterialized: boolean, - diagnostics: string[] + diagnostics: string[], + selectedHardFailureReason: string ): TeamRuntimeMemberLaunchEvidence { const confirmed = launchState === 'confirmed_alive'; const failed = launchState === 'failed'; @@ -590,7 +606,7 @@ function mapBridgeMemberToRuntimeEvidence( runtimeAlive: confirmed, bootstrapConfirmed: confirmed, hardFailure: failed, - hardFailureReason: failed ? 'OpenCode bridge reported member launch failure' : undefined, + hardFailureReason: failed ? selectedHardFailureReason : undefined, pendingPermissionRequestIds: pendingPermissionRequestIds && pendingPermissionRequestIds.length > 0 ? [...new Set(pendingPermissionRequestIds)] @@ -605,6 +621,83 @@ function mapBridgeMemberToRuntimeEvidence( }; } +function selectOpenCodeMemberFailureReason(input: { + memberDiagnostics: readonly string[]; + bridgeDiagnostics: readonly { + code: string; + severity: 'info' | 'warning' | 'error'; + message: string; + }[]; + checkpointDiagnostics: readonly string[]; + fallback: string; +}): string { + return ( + firstDisplayableOpenCodeFailureMessage(input.memberDiagnostics, { includeGeneric: false }) ?? + firstDisplayableOpenCodeFailureMessage( + input.bridgeDiagnostics + .filter((diagnostic) => diagnostic.severity === 'error') + .map((diagnostic) => diagnostic.message), + { includeGeneric: false } + ) ?? + firstDisplayableOpenCodeFailureMessage(input.memberDiagnostics, { includeGeneric: true }) ?? + firstDisplayableOpenCodeFailureMessage(input.checkpointDiagnostics, { includeGeneric: true }) ?? + firstDisplayableOpenCodeFailureMessage( + input.bridgeDiagnostics + .filter((diagnostic) => diagnostic.severity !== 'info') + .map((diagnostic) => diagnostic.message), + { includeGeneric: true } + ) ?? + normalizeOpenCodeFailureMessage(input.fallback) ?? + GENERIC_OPEN_CODE_MEMBER_FAILURE_REASON + ); +} + +function firstDisplayableOpenCodeFailureMessage( + values: readonly string[], + options: { includeGeneric: boolean } +): string | undefined { + for (const value of values) { + const normalized = normalizeOpenCodeFailureMessage(value); + if (!normalized) { + continue; + } + if (!options.includeGeneric && isGenericOpenCodeFailureMessage(normalized)) { + continue; + } + return normalized; + } + return undefined; +} + +function normalizeOpenCodeFailureMessage(value: string | undefined): string | undefined { + const trimmed = value?.replace(/\s+/g, ' ').trim(); + if (!trimmed) { + return undefined; + } + return trimmed + .replace(SECRET_FLAG_PATTERN, '$1[redacted]') + .replace(BEARER_TOKEN_PATTERN, 'Bearer [redacted]') + .replace(SECRET_KEY_PATTERN, '[redacted-api-key]'); +} + +function isGenericOpenCodeFailureMessage(message: string): boolean { + return ( + message === GENERIC_OPEN_CODE_MEMBER_FAILURE_REASON || + message.startsWith(`${GENERIC_OPEN_CODE_MEMBER_FAILURE_REASON}:`) || + message.startsWith('OpenCode secondary lane timing:') || + message.startsWith( + 'OpenCode bridge reported ready without all required durable checkpoints:' + ) || + message.startsWith( + 'OpenCode bridge reported ready before all expected members were confirmed:' + ) || + message.startsWith( + 'OpenCode bootstrap MCP did not complete required tools before assistant response:' + ) || + isOpenCodeLaunchTimingDiagnostic(message) + ); +} + function extractCheckpointNames(data: OpenCodeLaunchTeamCommandData): Set { const names = new Set(); for (const checkpoint of data.durableCheckpoints ?? []) { diff --git a/test/main/services/team/OpenCodeTeamRuntimeAdapter.test.ts b/test/main/services/team/OpenCodeTeamRuntimeAdapter.test.ts index 4f64f6c5..ef3c6dac 100644 --- a/test/main/services/team/OpenCodeTeamRuntimeAdapter.test.ts +++ b/test/main/services/team/OpenCodeTeamRuntimeAdapter.test.ts @@ -156,6 +156,106 @@ describe('OpenCodeTeamRuntimeAdapter', () => { ); }); + it('uses concrete member diagnostics as failed OpenCode hard failure reasons', async () => { + const concreteReason = + 'Latest assistant message msg_123 failed with APIError - Insufficient credits. Add more using https://openrouter.ai/settings/credits'; + const launchOpenCodeTeam = vi.fn< + NonNullable + >( + async () => + ({ + runId: 'run-1', + teamLaunchState: 'failed', + members: { + alice: { + sessionId: 'oc-session-1', + launchState: 'failed', + model: 'openai/gpt-5.4-mini', + diagnostics: ['OpenCode bridge reported member launch failure', concreteReason], + evidence: [], + }, + }, + warnings: [], + diagnostics: [], + }) satisfies OpenCodeLaunchTeamCommandData + ); + const adapter = new OpenCodeTeamRuntimeAdapter( + bridgePort(readiness({ state: 'ready', launchAllowed: true }), { launchOpenCodeTeam }) + ); + + const result = await adapter.launch(launchInput()); + + expect(result.members.alice).toMatchObject({ + launchState: 'failed_to_start', + hardFailureReason: concreteReason, + }); + }); + + it('falls back to bridge error diagnostics when member failure details are generic', async () => { + const bridgeError = 'Provider runtime returned a concrete launch error'; + const launchOpenCodeTeam = vi.fn< + NonNullable + >( + async () => + ({ + runId: 'run-1', + teamLaunchState: 'failed', + members: { + alice: { + sessionId: 'oc-session-1', + launchState: 'failed', + model: 'openai/gpt-5.4-mini', + diagnostics: ['OpenCode bridge reported member launch failure'], + evidence: [], + }, + }, + warnings: [], + diagnostics: [{ code: 'provider_error', severity: 'error', message: bridgeError }], + }) satisfies OpenCodeLaunchTeamCommandData + ); + const adapter = new OpenCodeTeamRuntimeAdapter( + bridgePort(readiness({ state: 'ready', launchAllowed: true }), { launchOpenCodeTeam }) + ); + + const result = await adapter.launch(launchInput()); + + expect(result.members.alice?.hardFailureReason).toBe(bridgeError); + }); + + it('redacts secret-like values in selected OpenCode failure reasons', async () => { + const launchOpenCodeTeam = vi.fn< + NonNullable + >( + async () => + ({ + runId: 'run-1', + teamLaunchState: 'failed', + members: { + alice: { + sessionId: 'oc-session-1', + launchState: 'failed', + model: 'openai/gpt-5.4-mini', + diagnostics: [ + 'Provider failed with --api-key sk-openroutersecret000000000000 and Bearer abc.def.ghi', + ], + evidence: [], + }, + }, + warnings: [], + diagnostics: [], + }) satisfies OpenCodeLaunchTeamCommandData + ); + const adapter = new OpenCodeTeamRuntimeAdapter( + bridgePort(readiness({ state: 'ready', launchAllowed: true }), { launchOpenCodeTeam }) + ); + + const result = await adapter.launch(launchInput()); + + expect(result.members.alice?.hardFailureReason).toBe( + 'Provider failed with --api-key [redacted] and Bearer [redacted]' + ); + }); + it('rejects non-OpenCode members before readiness or launch bridge dispatch', async () => { const launchOpenCodeTeam = vi.fn(); const bridge = bridgePort(readiness({ state: 'ready', launchAllowed: true }), { diff --git a/test/main/services/team/TeamProvisioningService.test.ts b/test/main/services/team/TeamProvisioningService.test.ts index 2b5ecbe3..daced49e 100644 --- a/test/main/services/team/TeamProvisioningService.test.ts +++ b/test/main/services/team/TeamProvisioningService.test.ts @@ -133,7 +133,10 @@ import { createPersistedLaunchSnapshot, snapshotFromRuntimeMemberStatuses, } from '@main/services/team/TeamLaunchStateEvaluator'; -import { getTeamLaunchStatePath } from '@main/services/team/TeamLaunchStateStore'; +import { + getTeamLaunchStatePath, + getTeamLaunchSummaryPath, +} from '@main/services/team/TeamLaunchStateStore'; import { TeamConfigReader } from '@main/services/team/TeamConfigReader'; import { getOpenCodeLaneScopedRuntimeFilePath, @@ -14200,6 +14203,91 @@ describe('TeamProvisioningService', () => { }); }); + it('clears stale launch-state and launch-summary when bootstrap truth supersedes a persisted failure', async () => { + const teamName = 'bootstrap-supersedes-stale-launch-summary-team'; + const leadSessionId = 'lead-session'; + const staleUpdatedAt = '2026-04-16T09:55:00.000Z'; + const freshObservedAt = '2026-04-16T10:00:00.000Z'; + writeLaunchConfig(teamName, '/Users/test/proj', leadSessionId, ['bob']); + writeLaunchState( + teamName, + leadSessionId, + { + bob: { + launchState: 'failed_to_start', + agentToolAccepted: false, + runtimeAlive: false, + bootstrapConfirmed: false, + hardFailure: true, + hardFailureReason: 'Teammate was never spawned during launch.', + }, + }, + { updatedAt: staleUpdatedAt } + ); + fs.writeFileSync( + getTeamLaunchSummaryPath(teamName), + `${JSON.stringify( + { + version: 1, + teamName, + updatedAt: staleUpdatedAt, + launchPhase: 'finished', + partialLaunchFailure: true, + expectedMemberCount: 1, + confirmedMemberCount: 0, + missingMembers: ['bob'], + teamLaunchState: 'partial_failure', + launchUpdatedAt: staleUpdatedAt, + confirmedCount: 0, + pendingCount: 0, + failedCount: 1, + skippedCount: 0, + runtimeAlivePendingCount: 0, + shellOnlyPendingCount: 0, + runtimeProcessPendingCount: 0, + runtimeCandidatePendingCount: 0, + noRuntimePendingCount: 0, + permissionPendingCount: 0, + }, + null, + 2 + )}\n`, + 'utf8' + ); + writeBootstrapState( + teamName, + [ + { + name: 'bob', + status: 'bootstrap_confirmed', + lastAttemptAt: Date.parse(freshObservedAt), + lastObservedAt: Date.parse(freshObservedAt), + }, + ], + freshObservedAt + ); + + const svc = new TeamProvisioningService(); + const result = await svc.getMemberSpawnStatuses(teamName); + + expect(result.statuses.bob).toMatchObject({ + status: 'online', + launchState: 'confirmed_alive', + bootstrapConfirmed: true, + hardFailure: false, + }); + await expect(fsPromises.readFile(getTeamLaunchStatePath(teamName), 'utf8')).rejects.toMatchObject( + { + code: 'ENOENT', + } + ); + await expect( + fsPromises.readFile(getTeamLaunchSummaryPath(teamName), 'utf8') + ).rejects.toMatchObject({ + code: 'ENOENT', + }); + }); + it('reconciles extra persisted launch members when bootstrap state proves they were registered', async () => { const teamName = 'registered-bootstrap-extra-member-team'; const teamDir = path.join(tempTeamsBase, teamName); @@ -15455,6 +15543,476 @@ describe('TeamProvisioningService', () => { }); }); + it('keeps primary bootstrap-confirmed members alive when OpenCode secondary lanes fail', async () => { + const teamName = 'atlas-hq-source-aware-live'; + const startedAt = '2026-04-23T10:00:00.000Z'; + const exactOpenCodeReason = + 'Latest assistant message msg_alice failed with APIError - Insufficient credits.'; + writeLaunchConfig(teamName, '/Users/test/proj', 'lead-session', ['bob', 'jack']); + writeBootstrapState( + teamName, + [ + { + name: 'bob', + status: 'bootstrap_confirmed', + lastObservedAt: Date.parse('2026-04-23T10:01:00.000Z'), + }, + { + name: 'jack', + status: 'bootstrap_confirmed', + lastObservedAt: Date.parse('2026-04-23T10:01:00.000Z'), + }, + ], + '2026-04-23T10:01:00.000Z' + ); + const run = createMemberSpawnRun({ + teamName, + runId: 'run-atlas-hq-source-aware-live', + startedAt, + expectedMembers: ['bob', 'jack'], + memberSpawnStatuses: new Map([ + [ + 'bob', + createMemberSpawnStatusEntry({ + status: 'error', + launchState: 'failed_to_start', + agentToolAccepted: false, + runtimeAlive: false, + bootstrapConfirmed: false, + hardFailure: true, + error: 'Teammate was never spawned during launch.', + hardFailureReason: 'Teammate was never spawned during launch.', + }), + ], + [ + 'jack', + createMemberSpawnStatusEntry({ + status: 'error', + launchState: 'failed_to_start', + agentToolAccepted: false, + runtimeAlive: false, + bootstrapConfirmed: false, + hardFailure: true, + error: 'Teammate was never spawned during launch.', + hardFailureReason: 'Teammate was never spawned during launch.', + }), + ], + ]), + }); + run.isLaunch = true; + run.request = { + teamName, + cwd: '/Users/test/proj', + providerId: 'codex', + providerBackendId: 'codex-native', + model: 'gpt-5.4', + members: [], + }; + run.effectiveMembers = [ + { name: 'bob', providerId: 'codex', model: 'gpt-5.3-codex' }, + { name: 'jack', providerId: 'codex', model: 'gpt-5.4' }, + ]; + run.mixedSecondaryLanes = [ + { + laneId: 'secondary:opencode:alice', + providerId: 'opencode', + member: { + name: 'alice', + providerId: 'opencode', + model: 'openrouter/z-ai/glm-5.1', + }, + runId: 'lane-run-alice', + state: 'finished', + result: { + runId: 'lane-run-alice', + teamName, + launchPhase: 'finished', + teamLaunchState: 'partial_failure', + members: { + alice: { + memberName: 'alice', + providerId: 'opencode', + launchState: 'failed_to_start', + agentToolAccepted: true, + runtimeAlive: false, + bootstrapConfirmed: false, + hardFailure: true, + hardFailureReason: exactOpenCodeReason, + diagnostics: [exactOpenCodeReason], + }, + }, + warnings: [], + diagnostics: [exactOpenCodeReason], + }, + warnings: [], + diagnostics: [exactOpenCodeReason], + }, + { + laneId: 'secondary:opencode:tom', + providerId: 'opencode', + member: { + name: 'tom', + providerId: 'opencode', + model: 'openrouter/minimax/minimax-m2.5', + }, + runId: 'lane-run-tom', + state: 'finished', + result: { + runId: 'lane-run-tom', + teamName, + launchPhase: 'finished', + teamLaunchState: 'partial_failure', + members: { + tom: { + memberName: 'tom', + providerId: 'opencode', + launchState: 'failed_to_start', + agentToolAccepted: true, + runtimeAlive: false, + bootstrapConfirmed: false, + hardFailure: true, + hardFailureReason: 'Tom provider launch failed.', + diagnostics: ['Tom provider launch failed.'], + }, + }, + warnings: [], + diagnostics: ['Tom provider launch failed.'], + }, + warnings: [], + diagnostics: ['Tom provider launch failed.'], + }, + ]; + run.detectedSessionId = 'lead-session'; + + const svc = new TeamProvisioningService(); + const snapshot = await (svc as any).persistLaunchStateSnapshot(run, 'finished'); + + expect(snapshot.members.bob).toMatchObject({ + launchState: 'confirmed_alive', + bootstrapConfirmed: true, + hardFailure: false, + }); + expect(snapshot.members.jack).toMatchObject({ + launchState: 'confirmed_alive', + bootstrapConfirmed: true, + hardFailure: false, + }); + expect(snapshot.members.alice).toMatchObject({ + launchState: 'failed_to_start', + hardFailureReason: exactOpenCodeReason, + }); + expect(snapshot.members.tom).toMatchObject({ + launchState: 'failed_to_start', + hardFailureReason: 'Tom provider launch failed.', + }); + expect(snapshot.summary.confirmedCount).toBe(2); + expect(snapshot.summary.failedCount).toBe(2); + }); + + it('reconciles persisted mixed launch-state when primary bootstrap members were marked missing', async () => { + const teamName = 'atlas-hq-source-aware-persisted'; + const exactOpenCodeReason = + 'Latest assistant message msg_alice failed with APIError - Insufficient credits.'; + writeTeamMeta(teamName, { + providerId: 'codex', + providerBackendId: 'codex-native', + model: 'gpt-5.4', + }); + writeMembersMeta(teamName, [ + { name: 'bob', providerId: 'codex', model: 'gpt-5.3-codex' }, + { name: 'jack', providerId: 'codex', model: 'gpt-5.4' }, + { name: 'alice', providerId: 'opencode', model: 'openrouter/z-ai/glm-5.1' }, + { name: 'tom', providerId: 'opencode', model: 'openrouter/minimax/minimax-m2.5' }, + ]); + writeLaunchConfig(teamName, '/Users/test/proj', 'lead-session', ['bob', 'jack']); + writeBootstrapState( + teamName, + [ + { + name: 'bob', + status: 'bootstrap_confirmed', + lastObservedAt: Date.parse('2026-04-23T10:01:00.000Z'), + }, + { + name: 'jack', + status: 'bootstrap_confirmed', + lastObservedAt: Date.parse('2026-04-23T10:01:00.000Z'), + }, + ], + '2026-04-23T10:01:00.000Z' + ); + fs.writeFileSync( + getTeamLaunchStatePath(teamName), + `${JSON.stringify( + createPersistedLaunchSnapshot({ + teamName, + leadSessionId: 'lead-session', + launchPhase: 'finished', + expectedMembers: ['bob', 'jack', 'alice', 'tom'], + bootstrapExpectedMembers: ['bob', 'jack'], + members: { + bob: { + name: 'bob', + providerId: 'codex', + laneId: 'primary', + laneKind: 'primary', + laneOwnerProviderId: 'codex', + launchState: 'failed_to_start', + agentToolAccepted: false, + runtimeAlive: false, + bootstrapConfirmed: false, + hardFailure: true, + hardFailureReason: 'Teammate was never spawned during launch.', + lastEvaluatedAt: '2026-04-23T10:02:00.000Z', + }, + jack: { + name: 'jack', + providerId: 'codex', + laneId: 'primary', + laneKind: 'primary', + laneOwnerProviderId: 'codex', + launchState: 'failed_to_start', + agentToolAccepted: false, + runtimeAlive: false, + bootstrapConfirmed: false, + hardFailure: true, + hardFailureReason: 'Teammate was never spawned during launch.', + lastEvaluatedAt: '2026-04-23T10:02:00.000Z', + }, + alice: { + name: 'alice', + providerId: 'opencode', + model: 'openrouter/z-ai/glm-5.1', + laneId: 'secondary:opencode:alice', + laneKind: 'secondary', + laneOwnerProviderId: 'opencode', + launchState: 'failed_to_start', + agentToolAccepted: true, + runtimeAlive: false, + bootstrapConfirmed: false, + hardFailure: true, + hardFailureReason: 'OpenCode bridge reported member launch failure', + diagnostics: [exactOpenCodeReason], + lastEvaluatedAt: '2026-04-23T10:02:00.000Z', + }, + tom: { + name: 'tom', + providerId: 'opencode', + model: 'openrouter/minimax/minimax-m2.5', + laneId: 'secondary:opencode:tom', + laneKind: 'secondary', + laneOwnerProviderId: 'opencode', + launchState: 'failed_to_start', + agentToolAccepted: true, + runtimeAlive: false, + bootstrapConfirmed: false, + hardFailure: true, + hardFailureReason: 'OpenCode bridge reported member launch failure', + diagnostics: ['Tom provider launch failed.'], + lastEvaluatedAt: '2026-04-23T10:02:00.000Z', + }, + }, + updatedAt: '2026-04-23T10:02:00.000Z', + }), + null, + 2 + )}\n`, + 'utf8' + ); + + const svc = new TeamProvisioningService(); + const result = await svc.getMemberSpawnStatuses(teamName); + + expect(result.statuses.bob).toMatchObject({ + status: 'online', + launchState: 'confirmed_alive', + bootstrapConfirmed: true, + hardFailure: false, + }); + expect(result.statuses.jack).toMatchObject({ + status: 'online', + launchState: 'confirmed_alive', + bootstrapConfirmed: true, + hardFailure: false, + }); + expect(result.statuses.alice).toMatchObject({ + status: 'error', + launchState: 'failed_to_start', + hardFailureReason: exactOpenCodeReason, + }); + expect(result.statuses.tom).toMatchObject({ + status: 'error', + launchState: 'failed_to_start', + hardFailureReason: 'Tom provider launch failed.', + }); + const summary = JSON.parse(await fsPromises.readFile(getTeamLaunchSummaryPath(teamName), 'utf8')); + expect(summary).toMatchObject({ + teamLaunchState: 'partial_failure', + confirmedCount: 2, + failedCount: 2, + missingMembers: ['alice', 'tom'], + }); + }); + + it('does not collapse persisted mixed secondary failures when primary bootstrap snapshot is clean and richer', async () => { + const teamName = 'mixed-clean-bootstrap-does-not-collapse-secondary-failure'; + writeMembersMeta(teamName, [ + { name: 'bob', providerId: 'codex', model: 'gpt-5.4' }, + { name: 'alice', providerId: 'opencode', model: 'openrouter/z-ai/glm-5.1' }, + ]); + writeLaunchConfig(teamName, '/Users/test/proj', 'lead-session', ['bob']); + writeBootstrapState( + teamName, + [ + { name: 'bob', status: 'bootstrap_confirmed' }, + { name: 'jack', status: 'bootstrap_confirmed' }, + { name: 'nova', status: 'bootstrap_confirmed' }, + { name: 'sam', status: 'bootstrap_confirmed' }, + { name: 'kim', status: 'bootstrap_confirmed' }, + ], + '2026-04-23T10:03:00.000Z' + ); + const exactOpenCodeReason = + 'Latest assistant message msg_alice failed with APIError - Insufficient credits.'; + fs.writeFileSync( + getTeamLaunchStatePath(teamName), + `${JSON.stringify( + createPersistedLaunchSnapshot({ + teamName, + leadSessionId: 'lead-session', + launchPhase: 'finished', + expectedMembers: ['bob', 'alice'], + bootstrapExpectedMembers: ['bob'], + members: { + bob: { + name: 'bob', + providerId: 'codex', + laneId: 'primary', + laneKind: 'primary', + laneOwnerProviderId: 'codex', + launchState: 'failed_to_start', + agentToolAccepted: false, + runtimeAlive: false, + bootstrapConfirmed: false, + hardFailure: true, + hardFailureReason: 'Teammate was never spawned during launch.', + lastEvaluatedAt: '2026-04-23T10:02:00.000Z', + }, + alice: { + name: 'alice', + providerId: 'opencode', + laneId: 'secondary:opencode:alice', + laneKind: 'secondary', + laneOwnerProviderId: 'opencode', + launchState: 'failed_to_start', + agentToolAccepted: true, + runtimeAlive: false, + bootstrapConfirmed: false, + hardFailure: true, + hardFailureReason: 'OpenCode bridge reported member launch failure', + diagnostics: [exactOpenCodeReason], + lastEvaluatedAt: '2026-04-23T10:02:00.000Z', + }, + }, + updatedAt: '2026-04-23T10:02:00.000Z', + }), + null, + 2 + )}\n`, + 'utf8' + ); + + const svc = new TeamProvisioningService(); + const result = await svc.getMemberSpawnStatuses(teamName); + + expect(result.teamLaunchState).toBe('partial_failure'); + expect(result.statuses.bob).toMatchObject({ + status: 'online', + launchState: 'confirmed_alive', + bootstrapConfirmed: true, + hardFailure: false, + }); + expect(result.statuses.alice).toMatchObject({ + status: 'error', + launchState: 'failed_to_start', + hardFailureReason: exactOpenCodeReason, + }); + const persisted = JSON.parse(await fsPromises.readFile(getTeamLaunchStatePath(teamName), 'utf8')); + expect(persisted.members.alice).toMatchObject({ + laneId: 'secondary:opencode:alice', + launchState: 'failed_to_start', + hardFailureReason: exactOpenCodeReason, + }); + }); + + it('does not revive primary members from stale bootstrap-state during mixed projection', async () => { + const teamName = 'atlas-hq-stale-bootstrap-live'; + writeLaunchConfig(teamName, '/Users/test/proj', 'lead-session', ['bob']); + writeBootstrapState( + teamName, + [ + { + name: 'bob', + status: 'bootstrap_confirmed', + lastObservedAt: Date.parse('2026-04-23T09:59:00.000Z'), + }, + ], + '2026-04-23T09:59:00.000Z' + ); + const run = createMemberSpawnRun({ + teamName, + runId: 'run-atlas-hq-stale-bootstrap-live', + startedAt: '2026-04-23T10:00:00.000Z', + expectedMembers: ['bob'], + memberSpawnStatuses: new Map([ + [ + 'bob', + createMemberSpawnStatusEntry({ + status: 'error', + launchState: 'failed_to_start', + agentToolAccepted: false, + runtimeAlive: false, + bootstrapConfirmed: false, + hardFailure: true, + error: 'Teammate was never spawned during launch.', + hardFailureReason: 'Teammate was never spawned during launch.', + }), + ], + ]), + }); + run.isLaunch = true; + run.request = { + teamName, + cwd: '/Users/test/proj', + providerId: 'codex', + providerBackendId: 'codex-native', + model: 'gpt-5.4', + members: [], + }; + run.effectiveMembers = [{ name: 'bob', providerId: 'codex', model: 'gpt-5.3-codex' }]; + run.mixedSecondaryLanes = [ + { + laneId: 'secondary:opencode:alice', + providerId: 'opencode', + member: { name: 'alice', providerId: 'opencode', model: 'openrouter/model' }, + runId: 'lane-run-alice', + state: 'finished', + result: null, + warnings: [], + diagnostics: [], + }, + ]; + + const svc = new TeamProvisioningService(); + const snapshot = await (svc as any).persistLaunchStateSnapshot(run, 'finished'); + + expect(snapshot.members.bob).toMatchObject({ + launchState: 'failed_to_start', + bootstrapConfirmed: false, + hardFailure: true, + }); + }); + it('includes queued OpenCode secondary lanes in live spawn statuses during createTeam runs', async () => { const svc = new TeamProvisioningService(); vi.spyOn(svc as any, 'refreshMemberSpawnStatusesFromLeadInbox').mockResolvedValue(undefined);