From ebcc0e717f057c110013bcc1f53ce45a4393ba47 Mon Sep 17 00:00:00 2001 From: infiniti <52129260+developerInfiniti@users.noreply.github.com> Date: Wed, 27 May 2026 12:16:41 +0300 Subject: [PATCH] fix(team): reconcile provisioned-but-not-alive bootstrap state --- .../renderer/adapters/TeamGraphAdapter.ts | 42 +- .../renderer/ui/GraphNodePopover.tsx | 4 + .../buildMixedPersistedLaunchSnapshot.test.ts | 207 +++++ .../buildMixedPersistedLaunchSnapshot.ts | 84 +- .../team/TeamLaunchSummaryProjection.ts | 227 ++++- .../services/team/TeamProvisioningService.ts | 207 ++++- .../TeamProvisioningLaunchDiagnostics.ts | 31 +- .../TeamProvisioningLaunchFailurePolicy.ts | 30 +- .../TeamProvisioningPromptBuilders.ts | 72 +- .../components/team/members/MemberCard.tsx | 39 +- .../team/members/MemberDetailDialog.tsx | 18 +- .../team/members/MemberDetailHeader.tsx | 13 + .../team/members/MemberHoverCard.tsx | 6 + .../components/team/members/MemberList.tsx | 58 +- .../components/team/provisioningSteps.ts | 73 +- .../components/team/teamRuntimeDisplayRows.ts | 60 +- src/renderer/store/slices/teamSlice.ts | 21 + src/renderer/utils/memberHelpers.ts | 223 +++-- src/renderer/utils/memberLaunchDiagnostics.ts | 129 ++- .../utils/teamProvisioningPresentation.ts | 24 +- src/shared/utils/teamLaunchFailureReason.ts | 142 +++ .../services/team/TeamConfigReader.test.ts | 148 ++- .../team/TeamLaunchSummaryProjection.test.ts | 422 +++++++++ .../TeamProvisioningLaunchDiagnostics.test.ts | 135 ++- ...eamProvisioningLaunchFailurePolicy.test.ts | 98 +- .../TeamProvisioningPromptBuilders.test.ts | 59 ++ .../team/TeamProvisioningService.test.ts | 870 +++++++++++++++--- .../team/members/MemberCard.test.ts | 49 + .../team/members/MemberDetailDialog.test.ts | 61 ++ .../team/members/MemberList.test.ts | 156 +++- .../components/team/provisioningSteps.test.ts | 329 +++++++ .../team/teamRuntimeDisplayRows.test.ts | 272 +++++- .../agent-graph/TeamGraphAdapter.test.ts | 148 +++ test/renderer/store/teamSlice.test.ts | 90 ++ test/renderer/utils/memberHelpers.test.ts | 360 +++++++- .../utils/memberLaunchDiagnostics.test.ts | 355 +++++++ .../teamProvisioningPresentation.test.ts | 121 +++ .../utils/teamLaunchFailureReason.test.ts | 149 +++ 38 files changed, 5114 insertions(+), 418 deletions(-) create mode 100644 src/shared/utils/teamLaunchFailureReason.ts create mode 100644 test/main/services/team/TeamProvisioningPromptBuilders.test.ts create mode 100644 test/shared/utils/teamLaunchFailureReason.test.ts diff --git a/src/features/agent-graph/renderer/adapters/TeamGraphAdapter.ts b/src/features/agent-graph/renderer/adapters/TeamGraphAdapter.ts index 25d11a8b..ce88d46c 100644 --- a/src/features/agent-graph/renderer/adapters/TeamGraphAdapter.ts +++ b/src/features/agent-graph/renderer/adapters/TeamGraphAdapter.ts @@ -38,6 +38,10 @@ import { import { isInboxNoiseMessage } from '@shared/utils/inboxNoise'; import { isLeadMember } from '@shared/utils/leadDetection'; import { buildOrderedVisibleTeamGraphOwnerIds } from '@shared/utils/teamGraphDefaultLayout'; +import { + hasUnsafeProvisionedButNotAliveRuntimeEvidenceWithSpawnContext, + isBootstrapConfirmedProvisionedButNotAliveFailure, +} from '@shared/utils/teamLaunchFailureReason'; import { isTeamTaskActivelyWorked, isTeamTaskNeedsFixActionable, @@ -560,6 +564,7 @@ export class TeamGraphAdapter { member.runtimeAdvisory, member.providerId, spawn, + runtimeEntry, pendingApprovalAgents?.has(member.name) ?? false ); const currentTask = member.currentTaskId @@ -581,7 +586,11 @@ export class TeamGraphAdapter { spawnBootstrapStalled: spawn?.bootstrapStalled, spawnAgentToolAccepted: spawn?.agentToolAccepted, spawnHardFailure: spawn?.hardFailure, + spawnHardFailureReason: spawn?.hardFailureReason, + spawnError: spawn?.error, + spawnRuntimeDiagnostic: spawn?.runtimeDiagnostic, spawnLivenessKind: spawn?.livenessKind, + spawnRuntimeDiagnosticSeverity: spawn?.runtimeDiagnosticSeverity, spawnFirstSpawnAcceptedAt: spawn?.firstSpawnAcceptedAt, spawnUpdatedAt: spawn?.updatedAt, runtimeEntry, @@ -599,7 +608,7 @@ export class TeamGraphAdapter { ? 'terminated' : hasRunningTool ? 'tool_calling' - : TeamGraphAdapter.#mapMemberStatus(member.status, spawn), + : TeamGraphAdapter.#mapMemberStatus(member.status, spawn, runtimeEntry), color: isTeamVisualOnline ? (member.color ?? undefined) : undefined, role: member.role ?? undefined, runtimeLabel: TeamGraphAdapter.#getRuntimeLabel( @@ -1269,9 +1278,15 @@ export class TeamGraphAdapter { runtimeAdvisory: ResolvedTeamMember['runtimeAdvisory'], providerId: ResolvedTeamMember['providerId'], spawn: MemberSpawnStatusEntry | undefined, + runtimeEntry: TeamAgentRuntimeEntry | undefined, pendingApproval: boolean ): Pick | undefined { - if (spawn?.launchState === 'failed_to_start' || spawn?.status === 'error') { + const hasUnsuppressedSpawnFailure = + TeamGraphAdapter.#hasUnsuppressedProvisionedButNotAliveFailure(spawn, runtimeEntry); + if ( + hasUnsuppressedSpawnFailure && + (spawn?.launchState === 'failed_to_start' || spawn?.status === 'error') + ) { return { exceptionTone: 'error', exceptionLabel: 'spawn failed' }; } if (pendingApproval || spawn?.launchState === 'runtime_pending_permission') { @@ -1290,10 +1305,19 @@ export class TeamGraphAdapter { return undefined; } - static #mapMemberStatus(status: string, spawn?: MemberSpawnStatusEntry): GraphNodeState { + static #mapMemberStatus( + status: string, + spawn?: MemberSpawnStatusEntry, + runtimeEntry?: TeamAgentRuntimeEntry + ): GraphNodeState { if (spawn?.launchState === 'runtime_pending_permission') return 'waiting'; if (spawn?.status === 'spawning') return 'thinking'; - if (spawn?.status === 'error') return 'error'; + if ( + spawn?.status === 'error' && + TeamGraphAdapter.#hasUnsuppressedProvisionedButNotAliveFailure(spawn, runtimeEntry) + ) { + return 'error'; + } if (spawn?.status === 'waiting') return 'waiting'; switch (status) { case 'active': @@ -1307,6 +1331,16 @@ export class TeamGraphAdapter { } } + static #hasUnsuppressedProvisionedButNotAliveFailure( + spawn: MemberSpawnStatusEntry | undefined, + runtimeEntry: TeamAgentRuntimeEntry | undefined + ): boolean { + return ( + !isBootstrapConfirmedProvisionedButNotAliveFailure(spawn) || + hasUnsafeProvisionedButNotAliveRuntimeEvidenceWithSpawnContext(spawn, runtimeEntry) + ); + } + static #mapTaskStatus(status: string): GraphNodeState { switch (status) { case 'pending': diff --git a/src/features/agent-graph/renderer/ui/GraphNodePopover.tsx b/src/features/agent-graph/renderer/ui/GraphNodePopover.tsx index 6ea2258d..3bdce9f2 100644 --- a/src/features/agent-graph/renderer/ui/GraphNodePopover.tsx +++ b/src/features/agent-graph/renderer/ui/GraphNodePopover.tsx @@ -373,7 +373,11 @@ const MemberPopoverContent = ({ spawnBootstrapStalled: spawnEntry?.bootstrapStalled, spawnAgentToolAccepted: spawnEntry?.agentToolAccepted, spawnHardFailure: spawnEntry?.hardFailure, + spawnHardFailureReason: spawnEntry?.hardFailureReason, + spawnError: spawnEntry?.error, + spawnRuntimeDiagnostic: spawnEntry?.runtimeDiagnostic, spawnLivenessKind: spawnEntry?.livenessKind, + spawnRuntimeDiagnosticSeverity: spawnEntry?.runtimeDiagnosticSeverity, spawnFirstSpawnAcceptedAt: spawnEntry?.firstSpawnAcceptedAt, spawnUpdatedAt: spawnEntry?.updatedAt, runtimeEntry, diff --git a/src/features/team-runtime-lanes/core/domain/__tests__/buildMixedPersistedLaunchSnapshot.test.ts b/src/features/team-runtime-lanes/core/domain/__tests__/buildMixedPersistedLaunchSnapshot.test.ts index dc6fb0f7..d7cd1d51 100644 --- a/src/features/team-runtime-lanes/core/domain/__tests__/buildMixedPersistedLaunchSnapshot.test.ts +++ b/src/features/team-runtime-lanes/core/domain/__tests__/buildMixedPersistedLaunchSnapshot.test.ts @@ -258,6 +258,213 @@ describe('buildMixedPersistedLaunchSnapshot', () => { expect(snapshot.teamLaunchState).toBe('partial_failure'); }); + it('heals bootstrap-confirmed provisioned-but-not-alive primary status while building snapshots', () => { + const snapshot = buildMixedPersistedLaunchSnapshot({ + teamName: 'signal-ops', + launchPhase: 'finished', + updatedAt: '2026-05-25T20:14:02.147Z', + leadDefaults: { + providerId: 'anthropic', + providerBackendId: null, + selectedFastMode: 'off', + resolvedFastMode: false, + launchIdentity: null, + }, + primaryMembers: [{ name: 'tom', providerId: 'anthropic', model: 'sonnet', effort: 'medium' }], + primaryStatuses: { + tom: { + status: 'error', + launchState: 'failed_to_start', + agentToolAccepted: true, + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: 'CLI process exited (code 1) \u2014 team provisioned but not alive', + livenessKind: 'confirmed_bootstrap', + runtimeDiagnostic: + 'runtime pid could not be verified because process table is unavailable', + runtimeDiagnosticSeverity: 'warning', + firstSpawnAcceptedAt: '2026-05-25T20:13:46.326Z', + lastHeartbeatAt: '2026-05-25T20:13:56.110Z', + updatedAt: '2026-05-25T20:14:02.147Z', + } as never, + }, + secondaryMembers: [], + }); + + expect(snapshot.members.tom).toMatchObject({ + launchState: 'confirmed_alive', + bootstrapConfirmed: true, + runtimeAlive: true, + hardFailure: false, + hardFailureReason: undefined, + livenessKind: 'confirmed_bootstrap', + }); + expect(snapshot.summary).toMatchObject({ + confirmedCount: 1, + failedCount: 0, + pendingCount: 0, + }); + expect(snapshot.teamLaunchState).toBe('clean_success'); + }); + + it('heals Windows process-table-unavailable provisioned-but-not-alive primary metadata', () => { + const snapshot = buildMixedPersistedLaunchSnapshot({ + teamName: 'signal-ops', + launchPhase: 'finished', + updatedAt: '2026-05-25T20:14:02.147Z', + leadDefaults: { + providerId: 'anthropic', + providerBackendId: null, + selectedFastMode: 'off', + resolvedFastMode: false, + launchIdentity: null, + }, + primaryMembers: [{ name: 'tom', providerId: 'anthropic', model: 'sonnet', effort: 'medium' }], + primaryStatuses: { + tom: { + status: 'error', + launchState: 'failed_to_start', + agentToolAccepted: true, + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: + 'CLI process exited (code 1) - team provisioned but not alive; process table unavailable', + livenessKind: 'stale_metadata', + runtimeDiagnostic: + 'runtime pid could not be verified because process table is unavailable', + runtimeDiagnosticSeverity: 'warning', + firstSpawnAcceptedAt: '2026-05-25T20:13:46.326Z', + lastHeartbeatAt: '2026-05-25T20:13:56.110Z', + updatedAt: '2026-05-25T20:14:02.147Z', + } as never, + }, + secondaryMembers: [], + }); + + expect(snapshot.members.tom).toMatchObject({ + launchState: 'confirmed_alive', + bootstrapConfirmed: true, + runtimeAlive: true, + hardFailure: false, + }); + expect(snapshot.summary).toMatchObject({ + confirmedCount: 1, + failedCount: 0, + }); + }); + + it('keeps bootstrap-confirmed provisioned-but-not-alive primary status failed when diagnostics are errors', () => { + const snapshot = buildMixedPersistedLaunchSnapshot({ + teamName: 'signal-ops', + launchPhase: 'finished', + updatedAt: '2026-05-25T20:14:02.147Z', + leadDefaults: { + providerId: 'anthropic', + providerBackendId: null, + selectedFastMode: 'off', + resolvedFastMode: false, + launchIdentity: null, + }, + primaryMembers: [{ name: 'tom', providerId: 'anthropic', model: 'sonnet', effort: 'medium' }], + primaryStatuses: { + tom: { + status: 'error', + launchState: 'failed_to_start', + agentToolAccepted: true, + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: 'CLI process exited (code 1) - team provisioned but not alive', + livenessKind: 'confirmed_bootstrap', + runtimeDiagnostic: 'Runtime process crashed', + runtimeDiagnosticSeverity: 'error', + updatedAt: '2026-05-25T20:14:02.147Z', + } as never, + }, + secondaryMembers: [], + }); + + expect(snapshot.members.tom).toMatchObject({ + launchState: 'failed_to_start', + bootstrapConfirmed: true, + runtimeAlive: false, + hardFailure: true, + hardFailureReason: 'CLI process exited (code 1) - team provisioned but not alive', + runtimeDiagnostic: 'Runtime process crashed', + runtimeDiagnosticSeverity: 'error', + }); + expect(snapshot.summary).toMatchObject({ + confirmedCount: 0, + failedCount: 1, + pendingCount: 0, + }); + expect(snapshot.teamLaunchState).toBe('partial_failure'); + }); + + it('keeps bootstrap-confirmed provisioned-but-not-alive secondary status failed when liveness is stopped', () => { + const snapshot = buildMixedPersistedLaunchSnapshot({ + teamName: 'mixed-team', + launchPhase: 'finished', + updatedAt: '2026-05-25T20:14:02.147Z', + leadDefaults: { + providerId: 'codex', + providerBackendId: 'codex-native', + selectedFastMode: 'off', + resolvedFastMode: false, + launchIdentity: null, + }, + primaryMembers: [], + primaryStatuses: {}, + secondaryMembers: [ + { + laneId: 'secondary:opencode:tom', + member: { + name: 'tom', + providerId: 'opencode', + model: 'minimax-m2.5-free', + effort: 'medium', + }, + leadDefaults: { + providerId: 'codex', + providerBackendId: 'codex-native', + selectedFastMode: 'off', + resolvedFastMode: false, + launchIdentity: null, + }, + evidence: { + status: 'error', + launchState: 'failed_to_start', + agentToolAccepted: true, + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: 'CLI process exited (code 1) - team provisioned but not alive', + livenessKind: 'not_found', + runtimeDiagnostic: 'Runtime is no longer registered', + runtimeDiagnosticSeverity: 'warning', + } as never, + }, + ], + }); + + expect(snapshot.members.tom).toMatchObject({ + laneKind: 'secondary', + launchState: 'failed_to_start', + runtimeAlive: false, + hardFailure: true, + livenessKind: 'not_found', + runtimeDiagnosticSeverity: 'warning', + }); + expect(snapshot.summary).toMatchObject({ + confirmedCount: 0, + failedCount: 1, + pendingCount: 0, + }); + expect(snapshot.teamLaunchState).toBe('partial_failure'); + }); + it('preserves permission-blocked side-lane members as runtime_pending_permission', () => { const snapshot = buildMixedPersistedLaunchSnapshot({ teamName: 'mixed-team', diff --git a/src/features/team-runtime-lanes/core/domain/buildMixedPersistedLaunchSnapshot.ts b/src/features/team-runtime-lanes/core/domain/buildMixedPersistedLaunchSnapshot.ts index 5eae2e40..11568134 100644 --- a/src/features/team-runtime-lanes/core/domain/buildMixedPersistedLaunchSnapshot.ts +++ b/src/features/team-runtime-lanes/core/domain/buildMixedPersistedLaunchSnapshot.ts @@ -1,10 +1,13 @@ import { isLeadMember } from '@shared/utils/leadDetection'; import { migrateProviderBackendId } from '@shared/utils/providerBackend'; +import { + hasUnsafeProvisionedButNotAliveRuntimeEvidence, + isBootstrapConfirmedProvisionedButNotAliveFailure, +} from '@shared/utils/teamLaunchFailureReason'; import { normalizeOptionalTeamProviderId } from '@shared/utils/teamProvider'; import type { MemberLaunchState, - MemberSpawnLivenessSource, MemberSpawnStatusEntry, OpenCodeAppManagedBootstrapCandidate, OpenCodeBootstrapEvidenceSource, @@ -95,6 +98,20 @@ function preservesStrongRuntimeAlive(value: { ); } +function canHealBootstrapConfirmedProvisionedButNotAliveFailure( + entry: + | (Parameters[0] & { + runtimeDiagnosticSeverity?: TeamAgentRuntimeDiagnosticSeverity; + livenessKind?: TeamAgentRuntimeLivenessKind; + }) + | undefined +): boolean { + return ( + isBootstrapConfirmedProvisionedButNotAliveFailure(entry) && + !hasUnsafeProvisionedButNotAliveRuntimeEvidence(entry) + ); +} + function hasMaterializedOpenCodeRuntimeMarker(value: { runtimeAlive?: boolean; runtimePid?: number; @@ -233,16 +250,22 @@ function createPrimaryLaneMemberState(params: { const runtime = params.status; const strongRuntimeAlive = preservesStrongRuntimeAlive(runtime ?? {}); const sources = runtime ? createSourcesFromStatus(runtime) : undefined; - const launchState = - runtime?.launchState ?? - deriveMemberLaunchState({ - hardFailure: runtime?.hardFailure, - bootstrapConfirmed: runtime?.bootstrapConfirmed, - runtimeAlive: strongRuntimeAlive, - agentToolAccepted: runtime?.agentToolAccepted, - pendingPermissionRequestIds: runtime?.pendingPermissionRequestIds, - }); - const hardFailure = runtime?.hardFailure === true || launchState === 'failed_to_start'; + const healBootstrapConfirmedProvisionedButNotAlive = + canHealBootstrapConfirmedProvisionedButNotAliveFailure(runtime); + const runtimeAlive = healBootstrapConfirmedProvisionedButNotAlive || strongRuntimeAlive; + const launchState = healBootstrapConfirmedProvisionedButNotAlive + ? 'confirmed_alive' + : (runtime?.launchState ?? + deriveMemberLaunchState({ + hardFailure: runtime?.hardFailure, + bootstrapConfirmed: runtime?.bootstrapConfirmed, + runtimeAlive: strongRuntimeAlive, + agentToolAccepted: runtime?.agentToolAccepted, + pendingPermissionRequestIds: runtime?.pendingPermissionRequestIds, + })); + const hardFailure = + !healBootstrapConfirmedProvisionedButNotAlive && + (runtime?.hardFailure === true || launchState === 'failed_to_start'); const base: PersistedTeamLaunchMemberState = { name: params.member.name.trim(), providerId, @@ -272,7 +295,7 @@ function createPrimaryLaneMemberState(params: { : undefined, launchState, agentToolAccepted: runtime?.agentToolAccepted === true, - runtimeAlive: strongRuntimeAlive, + runtimeAlive, bootstrapConfirmed: runtime?.bootstrapConfirmed === true, hardFailure, hardFailureReason: hardFailure ? (runtime?.hardFailureReason ?? runtime?.error) : undefined, @@ -285,7 +308,7 @@ function createPrimaryLaneMemberState(params: { firstSpawnAcceptedAt: runtime?.firstSpawnAcceptedAt, lastHeartbeatAt: runtime?.lastHeartbeatAt, runtimeLastSeenAt: runtime?.livenessLastCheckedAt, - lastRuntimeAliveAt: preservesStrongRuntimeAlive(runtime ?? {}) ? params.updatedAt : undefined, + lastRuntimeAliveAt: runtimeAlive ? params.updatedAt : undefined, lastEvaluatedAt: runtime?.updatedAt ?? params.updatedAt, sources, diagnostics: undefined, @@ -301,16 +324,22 @@ function createSecondaryLaneMemberState( normalizeOptionalTeamProviderId(params.member.providerId) ?? params.leadDefaults.providerId; const evidence = params.evidence; const strongRuntimeAlive = preservesStrongRuntimeAlive(evidence ?? {}); - const launchState = - evidence?.launchState ?? - deriveMemberLaunchState({ - hardFailure: evidence?.hardFailure, - bootstrapConfirmed: evidence?.bootstrapConfirmed, - runtimeAlive: strongRuntimeAlive, - agentToolAccepted: evidence?.agentToolAccepted, - pendingPermissionRequestIds: evidence?.pendingPermissionRequestIds, - }); - const hardFailure = evidence?.hardFailure === true || launchState === 'failed_to_start'; + const healBootstrapConfirmedProvisionedButNotAlive = + canHealBootstrapConfirmedProvisionedButNotAliveFailure(evidence ?? undefined); + const runtimeAlive = healBootstrapConfirmedProvisionedButNotAlive || strongRuntimeAlive; + const launchState = healBootstrapConfirmedProvisionedButNotAlive + ? 'confirmed_alive' + : (evidence?.launchState ?? + deriveMemberLaunchState({ + hardFailure: evidence?.hardFailure, + bootstrapConfirmed: evidence?.bootstrapConfirmed, + runtimeAlive: strongRuntimeAlive, + agentToolAccepted: evidence?.agentToolAccepted, + pendingPermissionRequestIds: evidence?.pendingPermissionRequestIds, + })); + const hardFailure = + !healBootstrapConfirmedProvisionedButNotAlive && + (evidence?.hardFailure === true || launchState === 'failed_to_start'); const hardFailureReason = hardFailure ? evidence?.hardFailureReason : undefined; const firstSpawnAcceptedAt = evidence ? resolveOpenCodeSecondaryFirstSpawnAcceptedAt(evidence, params.updatedAt) @@ -340,7 +369,7 @@ function createSecondaryLaneMemberState( laneOwnerProviderId: providerId, launchState, agentToolAccepted: evidence?.agentToolAccepted === true, - runtimeAlive: strongRuntimeAlive, + runtimeAlive, bootstrapConfirmed: evidence?.bootstrapConfirmed === true, hardFailure, hardFailureReason, @@ -373,7 +402,7 @@ function createSecondaryLaneMemberState( firstSpawnAcceptedAt, lastHeartbeatAt: evidence?.bootstrapConfirmed ? params.updatedAt : undefined, runtimeLastSeenAt: strongRuntimeAlive ? params.updatedAt : undefined, - lastRuntimeAliveAt: strongRuntimeAlive ? params.updatedAt : undefined, + lastRuntimeAliveAt: runtimeAlive ? params.updatedAt : undefined, lastEvaluatedAt: params.updatedAt, sources: strongRuntimeAlive ? { @@ -412,7 +441,10 @@ function summarizeMembers( pendingCount += 1; continue; } - if (entry.launchState === 'confirmed_alive') { + if ( + entry.launchState === 'confirmed_alive' || + canHealBootstrapConfirmedProvisionedButNotAliveFailure(entry) + ) { confirmedCount += 1; continue; } diff --git a/src/main/services/team/TeamLaunchSummaryProjection.ts b/src/main/services/team/TeamLaunchSummaryProjection.ts index ac29deef..3dd6befb 100644 --- a/src/main/services/team/TeamLaunchSummaryProjection.ts +++ b/src/main/services/team/TeamLaunchSummaryProjection.ts @@ -1,10 +1,26 @@ import { isMixedOpenCodeSideLanePlan, planTeamRuntimeLanes } from '@features/team-runtime-lanes'; +import { + hasBootstrapConfirmationProofForLaunchFailure, + hasUnsafeProvisionedButNotAliveRuntimeEvidence, + isProvisionedButNotAliveLaunchFailure, +} from '@shared/utils/teamLaunchFailureReason'; import { normalizeOptionalTeamProviderId } from '@shared/utils/teamProvider'; +import { isBootstrapMemberEvidenceCurrentForMember } from './provisioning/TeamProvisioningOpenCodeRuntimeEvidencePolicy'; import { shouldIgnoreTerminalBootstrapOnlyPendingSnapshot } from './TeamBootstrapStateReader'; -import { hasMixedPersistedLaunchMetadata } from './TeamLaunchStateEvaluator'; +import { + deriveTeamLaunchAggregateState, + hasMixedPersistedLaunchMetadata, + summarizePersistedLaunchMembers, +} from './TeamLaunchStateEvaluator'; -import type { PersistedTeamLaunchSnapshot, TeamProviderId, TeamSummary } from '@shared/types'; +import type { + PersistedTeamLaunchMemberState, + PersistedTeamLaunchSnapshot, + PersistedTeamLaunchSummary, + TeamProviderId, + TeamSummary, +} from '@shared/types'; export const TEAM_LAUNCH_SUMMARY_FILE = 'launch-summary.json'; const STALE_PENDING_SUMMARY_GRACE_MS = 5 * 60 * 1000; @@ -41,6 +57,71 @@ function getPersistedLaunchMemberNames(snapshot: PersistedTeamLaunchSnapshot): s return Array.from(new Set([...snapshot.expectedMembers, ...Object.keys(snapshot.members)])); } +function hasBootstrapConfirmationProof( + member: PersistedTeamLaunchMemberState, + bootstrapMember: PersistedTeamLaunchMemberState | undefined +): boolean { + if (hasBootstrapConfirmationProofForLaunchFailure(member)) { + return true; + } + return ( + bootstrapMember != null && + hasBootstrapConfirmationProofForLaunchFailure(bootstrapMember) && + isBootstrapMemberEvidenceCurrentForMember(member, bootstrapMember, 'confirmation') + ); +} + +function shouldProjectProvisionedButNotAliveAsConfirmed(params: { + member: PersistedTeamLaunchMemberState | undefined; + bootstrapMember?: PersistedTeamLaunchMemberState; +}): params is { member: PersistedTeamLaunchMemberState } { + const member = params.member; + if (member?.launchState !== 'failed_to_start' || member.hardFailure !== true) { + return false; + } + if ( + hasUnsafeProvisionedButNotAliveRuntimeEvidence(member) || + hasUnsafeProvisionedButNotAliveRuntimeEvidence(params.bootstrapMember) + ) { + return false; + } + return ( + isProvisionedButNotAliveLaunchFailure(member) && + hasBootstrapConfirmationProof(member, params.bootstrapMember) + ); +} + +function buildProjectedMembersForSummary( + snapshot: PersistedTeamLaunchSnapshot, + bootstrapSnapshot?: PersistedTeamLaunchSnapshot | null +): Record | null { + let changed = false; + const projectedMembers: Record = {}; + for (const [memberName, member] of Object.entries(snapshot.members)) { + if ( + shouldProjectProvisionedButNotAliveAsConfirmed({ + member, + bootstrapMember: bootstrapSnapshot?.members[memberName], + }) + ) { + changed = true; + projectedMembers[memberName] = { + ...member, + launchState: 'confirmed_alive', + runtimeAlive: true, + bootstrapConfirmed: true, + hardFailure: false, + hardFailureReason: undefined, + runtimeDiagnostic: undefined, + runtimeDiagnosticSeverity: undefined, + }; + continue; + } + projectedMembers[memberName] = member; + } + return changed ? projectedMembers : null; +} + function normalizeIsoDate(value: unknown): string | null { if (typeof value !== 'string') { return null; @@ -57,42 +138,47 @@ function toMillis(value: string | undefined | null): number { } export function createLaunchStateSummary( - snapshot: PersistedTeamLaunchSnapshot + snapshot: PersistedTeamLaunchSnapshot, + options: { bootstrapSnapshot?: PersistedTeamLaunchSnapshot | null } = {} ): LaunchStateSummary { const persistedMemberNames = getPersistedLaunchMemberNames(snapshot); + const projectedMembers = buildProjectedMembersForSummary(snapshot, options.bootstrapSnapshot); + const members = projectedMembers ?? snapshot.members; + const summary = projectedMembers + ? summarizePersistedLaunchMembers(snapshot.expectedMembers, projectedMembers) + : snapshot.summary; + const teamLaunchState = projectedMembers + ? deriveTeamLaunchAggregateState(summary) + : snapshot.teamLaunchState; const missingMembers = persistedMemberNames.filter((name) => { - const member = snapshot.members[name]; + const member = members[name]; return member?.launchState === 'failed_to_start'; }); const skippedMembers = persistedMemberNames.filter((name) => { - const member = snapshot.members[name]; + const member = members[name]; return member?.launchState === 'skipped_for_launch' || member?.skippedForLaunch === true; }); return { - ...(snapshot.teamLaunchState === 'partial_failure' - ? { partialLaunchFailure: true as const } - : {}), + ...(teamLaunchState === 'partial_failure' ? { partialLaunchFailure: true as const } : {}), ...(persistedMemberNames.length > 0 ? { expectedMemberCount: persistedMemberNames.length } : {}), - ...(snapshot.summary.confirmedCount > 0 - ? { confirmedMemberCount: snapshot.summary.confirmedCount } - : {}), + ...(summary.confirmedCount > 0 ? { confirmedMemberCount: summary.confirmedCount } : {}), ...(missingMembers.length > 0 ? { missingMembers } : {}), ...(skippedMembers.length > 0 ? { skippedMembers } : {}), - teamLaunchState: snapshot.teamLaunchState, + teamLaunchState, launchUpdatedAt: snapshot.updatedAt, - confirmedCount: snapshot.summary.confirmedCount, - pendingCount: snapshot.summary.pendingCount, - failedCount: snapshot.summary.failedCount, - skippedCount: snapshot.summary.skippedCount, - runtimeAlivePendingCount: snapshot.summary.runtimeAlivePendingCount, - shellOnlyPendingCount: snapshot.summary.shellOnlyPendingCount, - runtimeProcessPendingCount: snapshot.summary.runtimeProcessPendingCount, - runtimeCandidatePendingCount: snapshot.summary.runtimeCandidatePendingCount, - noRuntimePendingCount: snapshot.summary.noRuntimePendingCount, - permissionPendingCount: snapshot.summary.permissionPendingCount, + confirmedCount: summary.confirmedCount, + pendingCount: summary.pendingCount, + failedCount: summary.failedCount, + skippedCount: summary.skippedCount, + runtimeAlivePendingCount: summary.runtimeAlivePendingCount, + shellOnlyPendingCount: summary.shellOnlyPendingCount, + runtimeProcessPendingCount: summary.runtimeProcessPendingCount, + runtimeCandidatePendingCount: summary.runtimeCandidatePendingCount, + noRuntimePendingCount: summary.noRuntimePendingCount, + permissionPendingCount: summary.permissionPendingCount, }; } @@ -242,6 +328,83 @@ function shouldIgnoreStalePendingLaunchSnapshotSummary( return Number.isFinite(updatedAtMs) && nowMs - updatedAtMs >= STALE_PENDING_SUMMARY_GRACE_MS; } +function reconcileSummaryProjectionWithBootstrap( + projection: PersistedTeamLaunchSummaryProjection, + bootstrapSnapshot: PersistedTeamLaunchSnapshot +): PersistedTeamLaunchSummaryProjection { + const missingMembers = projection.missingMembers ?? []; + if (missingMembers.length === 0) { + return projection; + } + + const projectionBoundary = projection.launchUpdatedAt ?? projection.updatedAt; + const healedMembers = missingMembers.filter((memberName) => { + const bootstrapMember = bootstrapSnapshot.members[memberName]; + return ( + bootstrapMember != null && + hasBootstrapConfirmationProofForLaunchFailure(bootstrapMember) && + !hasUnsafeProvisionedButNotAliveRuntimeEvidence(bootstrapMember) && + isBootstrapMemberEvidenceCurrentForMember( + { firstSpawnAcceptedAt: projectionBoundary, lastEvaluatedAt: projectionBoundary }, + bootstrapMember, + 'confirmation' + ) + ); + }); + if (healedMembers.length === 0) { + return projection; + } + + const healedMemberNames = new Set(healedMembers); + const nextMissingMembers = missingMembers.filter( + (memberName) => !healedMemberNames.has(memberName) + ); + const summary: PersistedTeamLaunchSummary = { + confirmedCount: + (projection.confirmedCount ?? projection.confirmedMemberCount ?? 0) + healedMembers.length, + pendingCount: projection.pendingCount ?? 0, + failedCount: Math.max( + 0, + (projection.failedCount ?? missingMembers.length) - healedMembers.length + ), + skippedCount: projection.skippedCount ?? projection.skippedMembers?.length ?? 0, + runtimeAlivePendingCount: projection.runtimeAlivePendingCount ?? 0, + shellOnlyPendingCount: projection.shellOnlyPendingCount, + runtimeProcessPendingCount: projection.runtimeProcessPendingCount, + runtimeCandidatePendingCount: projection.runtimeCandidatePendingCount, + noRuntimePendingCount: projection.noRuntimePendingCount, + permissionPendingCount: projection.permissionPendingCount, + }; + const teamLaunchState = deriveTeamLaunchAggregateState(summary); + + const reconciled: PersistedTeamLaunchSummaryProjection = { + ...projection, + teamLaunchState, + confirmedMemberCount: summary.confirmedCount, + confirmedCount: summary.confirmedCount, + pendingCount: summary.pendingCount, + failedCount: summary.failedCount, + skippedCount: summary.skippedCount, + runtimeAlivePendingCount: summary.runtimeAlivePendingCount, + shellOnlyPendingCount: summary.shellOnlyPendingCount, + runtimeProcessPendingCount: summary.runtimeProcessPendingCount, + runtimeCandidatePendingCount: summary.runtimeCandidatePendingCount, + noRuntimePendingCount: summary.noRuntimePendingCount, + permissionPendingCount: summary.permissionPendingCount, + }; + if (nextMissingMembers.length > 0) { + reconciled.missingMembers = nextMissingMembers; + } else { + delete reconciled.missingMembers; + } + if (teamLaunchState === 'partial_failure') { + reconciled.partialLaunchFailure = true; + } else { + delete reconciled.partialLaunchFailure; + } + return reconciled; +} + export function choosePreferredLaunchStateSummary(params: { bootstrapSnapshot?: PersistedTeamLaunchSnapshot | null; launchSnapshot?: PersistedTeamLaunchSnapshot | null; @@ -252,7 +415,9 @@ export function choosePreferredLaunchStateSummary(params: { ? null : (params.launchSnapshot ?? null); if (launchSnapshot) { - return createLaunchStateSummary(launchSnapshot); + return createLaunchStateSummary(launchSnapshot, { + bootstrapSnapshot: params.bootstrapSnapshot ?? null, + }); } const bootstrapSnapshot = params.bootstrapSnapshot ?? null; @@ -271,22 +436,28 @@ export function choosePreferredLaunchStateSummary(params: { return createLaunchStateSummary(bootstrapSnapshot); } + const reconciledProjection = reconcileSummaryProjectionWithBootstrap( + projection, + bootstrapSnapshot + ); const bootstrapMixedAware = hasMixedPersistedLaunchMetadata(bootstrapSnapshot); - const projectionMixedAware = projection.mixedAware === true; + const projectionMixedAware = reconciledProjection.mixedAware === true; if (projectionMixedAware !== bootstrapMixedAware) { - return projectionMixedAware ? projection : createLaunchStateSummary(bootstrapSnapshot); + return projectionMixedAware + ? reconciledProjection + : createLaunchStateSummary(bootstrapSnapshot); } - const projectionUpdatedAtMs = toMillis(projection.updatedAt); + const projectionUpdatedAtMs = toMillis(reconciledProjection.updatedAt); const bootstrapUpdatedAtMs = toMillis(bootstrapSnapshot.updatedAt); if (!Number.isFinite(bootstrapUpdatedAtMs)) { - return projection; + return reconciledProjection; } if (!Number.isFinite(projectionUpdatedAtMs)) { return createLaunchStateSummary(bootstrapSnapshot); } return projectionUpdatedAtMs >= bootstrapUpdatedAtMs - ? projection + ? reconciledProjection : createLaunchStateSummary(bootstrapSnapshot); } diff --git a/src/main/services/team/TeamProvisioningService.ts b/src/main/services/team/TeamProvisioningService.ts index 1199b685..f20418c9 100644 --- a/src/main/services/team/TeamProvisioningService.ts +++ b/src/main/services/team/TeamProvisioningService.ts @@ -122,6 +122,7 @@ import { isTeamInternalControlMessageText, stripExactInternalControlEchoPrefix, } from '@shared/utils/teamInternalControlMessages'; +import { hasUnsafeProvisionedButNotAliveRuntimeEvidence } from '@shared/utils/teamLaunchFailureReason'; import { parseAllTeammateMessages, type ParsedTeammateContent, @@ -231,6 +232,7 @@ import { isAutoClearableLaunchFailureReason, isCliProvisionedButNotAliveFailureReason, isNeverSpawnedDuringLaunchReason, + isProvisionedButNotAliveFailureReason, } from './provisioning/TeamProvisioningLaunchFailurePolicy'; import { isOpenCodeOverlayMemberRemoved, @@ -959,9 +961,16 @@ function isConfirmedBootstrapStaleRuntimeDiagnostic(reason?: string): boolean { return text === 'persisted runtime pid is not alive'; } +function isBootstrapProofClearableLaunchFailureReason(reason?: string): boolean { + return ( + isAutoClearableLaunchFailureReason(reason) || isProvisionedButNotAliveFailureReason(reason) + ); +} + function shouldClearRuntimeDiagnosticAfterBootstrapConfirmation(reason?: string): boolean { return ( - isAutoClearableLaunchFailureReason(reason) || isConfirmedBootstrapStaleRuntimeDiagnostic(reason) + isBootstrapProofClearableLaunchFailureReason(reason) || + isConfirmedBootstrapStaleRuntimeDiagnostic(reason) ); } @@ -13666,13 +13675,14 @@ export class TeamProvisioningService { status: 'online', updatedAt, agentToolAccepted: true, - runtimeAlive: source === 'runtime-proof' ? true : prev.runtimeAlive === true, + runtimeAlive: source === 'runtime-proof' ? true : prev.runtimeAlive, bootstrapConfirmed: true, hardFailure: false, bootstrapStalled: undefined, error: undefined, hardFailureReason: undefined, - livenessSource: prev.livenessSource ?? 'process', + livenessSource: + source === 'runtime-proof' ? (prev.livenessSource ?? 'process') : prev.livenessSource, firstSpawnAcceptedAt: prev.firstSpawnAcceptedAt ?? observedAt, lastHeartbeatAt: isMemberSpawnHeartbeatTimestampNewer(prev.lastHeartbeatAt, observedAt) ? observedAt @@ -17128,7 +17138,7 @@ export class TeamProvisioningService { const canClearFailedBootstrap = current?.launchState === 'failed_to_start' && current.agentToolAccepted === true && - isAutoClearableLaunchFailureReason(failureReason); + isBootstrapProofClearableLaunchFailureReason(failureReason); if ( !current || (current.launchState === 'failed_to_start' && !canClearFailedBootstrap) || @@ -24243,6 +24253,39 @@ export class TeamProvisioningService { current.bootstrapConfirmed === true || current.launchState === 'confirmed_alive'; const shouldSuppressWeakRuntimeMetadataForConfirmedBootstrap = hasConfirmedBootstrap && !hasStrongEvidence; + const failureReason = current.hardFailureReason ?? current.error ?? current.runtimeDiagnostic; + const bootstrapProofClearableFailure = + isBootstrapProofClearableLaunchFailureReason(failureReason); + const metadataRuntimeDiagnosticForUnsafe = buildRuntimeDiagnosticForSpawn(metadata); + const unsafeRuntimeDiagnosticEvidence = + metadataRuntimeDiagnosticForUnsafe && + current.runtimeDiagnostic && + metadataRuntimeDiagnosticForUnsafe !== current.runtimeDiagnostic + ? `${metadataRuntimeDiagnosticForUnsafe}; ${current.runtimeDiagnostic}` + : (metadataRuntimeDiagnosticForUnsafe ?? current.runtimeDiagnostic); + const hasUnsafeProvisionedButNotAliveFailure = + isProvisionedButNotAliveFailureReason(failureReason) && + hasUnsafeProvisionedButNotAliveRuntimeEvidence({ + ...current, + runtimeDiagnostic: unsafeRuntimeDiagnosticEvidence, + runtimeDiagnosticSeverity: + metadata.runtimeDiagnosticSeverity ?? current.runtimeDiagnosticSeverity, + livenessKind: metadata.livenessKind ?? current.livenessKind, + }); + const shouldPreserveConfirmedBootstrapRuntimeError = + hasConfirmedBootstrap && + metadata.alive === false && + metadata.runtimeDiagnosticSeverity === 'error'; + const shouldPreserveUnsafeMetadataLivenessKind = + hasUnsafeProvisionedButNotAliveFailure && + (metadata.livenessKind === 'not_found' || + metadata.livenessKind === 'shell_only' || + metadata.livenessKind === 'runtime_process_candidate' || + ((metadata.livenessKind === 'registered_only' || + metadata.livenessKind === 'stale_metadata') && + (metadata.runtimeDiagnosticSeverity ?? current.runtimeDiagnosticSeverity) !== 'error' && + !mentionsProcessTableUnavailable(unsafeRuntimeDiagnosticEvidence) && + !mentionsProcessTableUnavailable(failureReason))); let runtimeDiagnostic: string | undefined; let runtimeDiagnosticSeverity: TeamAgentRuntimeDiagnosticSeverity | undefined; if (shouldPreserveProcessBootstrapTransportDiagnostic) { @@ -24256,7 +24299,7 @@ export class TeamProvisioningService { runtimeDiagnostic = current.runtimeDiagnostic; runtimeDiagnosticSeverity = current.runtimeDiagnosticSeverity; } else { - const metadataRuntimeDiagnostic = buildRuntimeDiagnosticForSpawn(metadata); + const metadataRuntimeDiagnostic = metadataRuntimeDiagnosticForUnsafe; if ( metadataRuntimeDiagnostic && !shouldClearRuntimeDiagnosticAfterBootstrapConfirmation(metadataRuntimeDiagnostic) @@ -24271,7 +24314,9 @@ export class TeamProvisioningService { } const metadataLivenessKind = hasConfirmedBootstrap ? metadata.livenessKind === 'runtime_process' || - metadata.livenessKind === 'confirmed_bootstrap' + metadata.livenessKind === 'confirmed_bootstrap' || + shouldPreserveConfirmedBootstrapRuntimeError || + shouldPreserveUnsafeMetadataLivenessKind ? metadata.livenessKind : current.livenessKind === 'stale_metadata' || current.livenessKind === 'registered_only' ? 'confirmed_bootstrap' @@ -24291,7 +24336,6 @@ export class TeamProvisioningService { : {}), livenessLastCheckedAt: nowIso(), }; - const failureReason = current.hardFailureReason ?? current.error; const hasWeakEvidence = metadata.livenessKind != null && !hasStrongEvidence && current.bootstrapConfirmed !== true; if ( @@ -24335,7 +24379,8 @@ export class TeamProvisioningService { if ( hasStrongEvidence && current.launchState === 'failed_to_start' && - isAutoClearableLaunchFailureReason(failureReason) + bootstrapProofClearableFailure && + !hasUnsafeProvisionedButNotAliveFailure ) { nextEntry.status = 'online'; nextEntry.agentToolAccepted = true; @@ -24346,7 +24391,34 @@ export class TeamProvisioningService { nextEntry.livenessSource = current.bootstrapConfirmed ? current.livenessSource : 'process'; nextEntry.launchState = deriveMemberLaunchState(nextEntry); } - if (hasWeakEvidence) { + if ( + hasConfirmedBootstrap && + current.hardFailure === true && + bootstrapProofClearableFailure && + !hasUnsafeProvisionedButNotAliveFailure + ) { + nextEntry.status = 'online'; + nextEntry.agentToolAccepted = true; + nextEntry.runtimeAlive = true; + nextEntry.bootstrapConfirmed = true; + nextEntry.hardFailure = false; + nextEntry.hardFailureReason = undefined; + nextEntry.error = undefined; + nextEntry.bootstrapStalled = undefined; + nextEntry.launchState = deriveMemberLaunchState(nextEntry); + } + const healedConfirmedBootstrapFailure = + hasConfirmedBootstrap && + current.hardFailure === true && + bootstrapProofClearableFailure && + !hasUnsafeProvisionedButNotAliveFailure; + if (shouldPreserveConfirmedBootstrapRuntimeError) { + nextEntry.runtimeAlive = false; + if (nextEntry.livenessSource === 'process') { + nextEntry.livenessSource = undefined; + } + } + if (hasWeakEvidence && !healedConfirmedBootstrapFailure) { nextEntry.runtimeAlive = false; if (nextEntry.livenessSource === 'process') { nextEntry.livenessSource = undefined; @@ -26714,10 +26786,17 @@ export class TeamProvisioningService { if (current.launchState === 'skipped_for_launch' || current.skippedForLaunch === true) { continue; } - const failureReason = current.hardFailureReason ?? current.error; + const failureReason = current.hardFailureReason ?? current.error ?? current.runtimeDiagnostic; + const provisionedButNotAliveFailure = isProvisionedButNotAliveFailureReason(failureReason); + if ( + provisionedButNotAliveFailure && + hasUnsafeProvisionedButNotAliveRuntimeEvidence(current) + ) { + continue; + } if ( current.launchState === 'failed_to_start' && - !isAutoClearableLaunchFailureReason(failureReason) + !isBootstrapProofClearableLaunchFailureReason(failureReason) ) { continue; } @@ -26827,12 +26906,19 @@ export class TeamProvisioningService { : undefined; const failureReason = current.hardFailureReason ?? persistedError ?? current.runtimeDiagnostic; + const provisionedButNotAliveFailure = isProvisionedButNotAliveFailureReason(failureReason); + if ( + provisionedButNotAliveFailure && + hasUnsafeProvisionedButNotAliveRuntimeEvidence(current) + ) { + continue; + } const hasFailure = current.launchState === 'failed_to_start' || current.hardFailure === true || typeof current.hardFailureReason === 'string' || typeof persistedError === 'string'; - if (hasFailure && !isAutoClearableLaunchFailureReason(failureReason)) { + if (hasFailure && !isBootstrapProofClearableLaunchFailureReason(failureReason)) { continue; } @@ -26845,14 +26931,21 @@ export class TeamProvisioningService { ...current, launchState: 'confirmed_alive', agentToolAccepted: true, - runtimeAlive: current.runtimeAlive === true || bootstrapMember.runtimeAlive === true, + runtimeAlive: + current.runtimeAlive === true || + bootstrapMember.runtimeAlive === true || + provisionedButNotAliveFailure, bootstrapConfirmed: true, hardFailure: false, hardFailureReason: undefined, - runtimeDiagnostic: isAutoClearableLaunchFailureReason(current.runtimeDiagnostic) + runtimeDiagnostic: shouldClearRuntimeDiagnosticAfterBootstrapConfirmation( + current.runtimeDiagnostic + ) ? undefined : current.runtimeDiagnostic, - runtimeDiagnosticSeverity: isAutoClearableLaunchFailureReason(current.runtimeDiagnostic) + runtimeDiagnosticSeverity: shouldClearRuntimeDiagnosticAfterBootstrapConfirmation( + current.runtimeDiagnostic + ) ? undefined : current.runtimeDiagnosticSeverity, bootstrapStalled: undefined, @@ -28638,7 +28731,9 @@ export class TeamProvisioningService { current.firstSpawnAcceptedAt != null ? Date.parse(current.firstSpawnAcceptedAt) : NaN; if ( current.launchState !== 'failed_to_start' || - isAutoClearableLaunchFailureReason(current.hardFailureReason ?? current.runtimeDiagnostic) + isBootstrapProofClearableLaunchFailureReason( + current.hardFailureReason ?? current.runtimeDiagnostic + ) ) { const runtimeProofObservedAt = await this.findBootstrapRuntimeProofObservedAt( snapshot.teamName, @@ -29053,9 +29148,16 @@ export class TeamProvisioningService { continue; } const failureReason = current.hardFailureReason ?? current.runtimeDiagnostic; + const provisionedButNotAliveFailure = isProvisionedButNotAliveFailureReason(failureReason); + if ( + provisionedButNotAliveFailure && + hasUnsafeProvisionedButNotAliveRuntimeEvidence(current) + ) { + continue; + } const canClearFailedBootstrap = current.launchState !== 'failed_to_start' || - isAutoClearableLaunchFailureReason(failureReason); + isBootstrapProofClearableLaunchFailureReason(failureReason); if (!canClearFailedBootstrap) { continue; } @@ -29083,7 +29185,9 @@ export class TeamProvisioningService { ...current, agentToolAccepted: true, bootstrapConfirmed: true, - runtimeAlive: runtimeProofObservedAt ? true : current.runtimeAlive === true, + runtimeAlive: runtimeProofObservedAt + ? true + : current.runtimeAlive === true || provisionedButNotAliveFailure, hardFailure: false, hardFailureReason: undefined, lastHeartbeatAt: current.lastHeartbeatAt ?? observedAt, @@ -29140,7 +29244,7 @@ export class TeamProvisioningService { const failureReason = current.hardFailureReason ?? current.runtimeDiagnostic; const hasAutoClearableFailure = (current.launchState === 'failed_to_start' || current.hardFailure === true) && - isAutoClearableLaunchFailureReason(failureReason); + isBootstrapProofClearableLaunchFailureReason(failureReason); if (!currentConfirmed || hasAutoClearableFailure) { return true; } @@ -29537,12 +29641,58 @@ export class TeamProvisioningService { const hadAutoClearableFailure = isAutoClearableLaunchFailureReason(initialFailureReason); const requiresConfirmedBootstrapToClearFailure = isCliProvisionedButNotAliveFailureReason(initialFailureReason); + const metadataRuntimeDiagnostic = runtimeMetadata?.[1].runtimeDiagnostic; + const metadataRuntimeDiagnosticSeverity = runtimeMetadata?.[1].runtimeDiagnosticSeverity; + const metadataLivenessKind = runtimeMetadata?.[1].livenessKind; + const refreshedRuntimeDiagnosticEvidence = + metadataRuntimeDiagnostic && + current.runtimeDiagnostic && + metadataRuntimeDiagnostic !== current.runtimeDiagnostic + ? `${metadataRuntimeDiagnostic}; ${current.runtimeDiagnostic}` + : (metadataRuntimeDiagnostic ?? current.runtimeDiagnostic); + const hasUnsafeProvisionedButNotAliveFailure = + requiresConfirmedBootstrapToClearFailure && + hasUnsafeProvisionedButNotAliveRuntimeEvidence({ + ...current, + runtimeDiagnostic: refreshedRuntimeDiagnosticEvidence, + runtimeDiagnosticSeverity: + metadataRuntimeDiagnosticSeverity ?? current.runtimeDiagnosticSeverity, + livenessKind: metadataLivenessKind ?? current.livenessKind, + }); + const shouldPreserveUnsafeMetadataLivenessKind = + hasUnsafeProvisionedButNotAliveFailure && + (metadataLivenessKind === 'not_found' || + metadataLivenessKind === 'shell_only' || + metadataLivenessKind === 'runtime_process_candidate' || + ((metadataLivenessKind === 'registered_only' || + metadataLivenessKind === 'stale_metadata') && + (metadataRuntimeDiagnosticSeverity ?? current.runtimeDiagnosticSeverity) !== 'error' && + !mentionsProcessTableUnavailable(refreshedRuntimeDiagnosticEvidence) && + !mentionsProcessTableUnavailable(initialFailureReason))); + const nextLivenessKind = current.bootstrapConfirmed + ? metadataLivenessKind === 'runtime_process' || + metadataLivenessKind === 'confirmed_bootstrap' || + shouldPreserveUnsafeMetadataLivenessKind + ? metadataLivenessKind + : current.livenessKind === 'stale_metadata' || current.livenessKind === 'registered_only' + ? 'confirmed_bootstrap' + : (current.livenessKind ?? 'confirmed_bootstrap') + : (metadataLivenessKind ?? current.livenessKind); current.runtimeAlive = observedRuntimeAlive; current.lastRuntimeAliveAt = observedRuntimeAlive ? now : current.lastRuntimeAliveAt; - current.livenessKind = runtimeMetadata?.[1].livenessKind; + current.livenessKind = nextLivenessKind; current.pidSource = runtimeMetadata?.[1].pidSource; - current.runtimeDiagnostic = runtimeMetadata?.[1].runtimeDiagnostic; - current.runtimeDiagnosticSeverity = runtimeMetadata?.[1].runtimeDiagnosticSeverity; + const shouldKeepUnsafeRuntimeDiagnostic = + hasUnsafeProvisionedButNotAliveFailure && + (metadataRuntimeDiagnostic == null || + (current.runtimeDiagnosticSeverity === 'error' && + metadataRuntimeDiagnosticSeverity !== 'error')); + current.runtimeDiagnostic = shouldKeepUnsafeRuntimeDiagnostic + ? current.runtimeDiagnostic + : metadataRuntimeDiagnostic; + current.runtimeDiagnosticSeverity = shouldKeepUnsafeRuntimeDiagnostic + ? current.runtimeDiagnosticSeverity + : metadataRuntimeDiagnosticSeverity; current.sources = { ...(current.sources ?? {}), processAlive: observedRuntimeAlive || undefined, @@ -29572,8 +29722,12 @@ export class TeamProvisioningService { if ( current.bootstrapConfirmed && !isOpenCodeSecondaryLaneMember && - isAutoClearableLaunchFailureReason(current.hardFailureReason) + !hasUnsafeProvisionedButNotAliveFailure && + isBootstrapProofClearableLaunchFailureReason(current.hardFailureReason) ) { + if (isProvisionedButNotAliveFailureReason(current.hardFailureReason)) { + current.runtimeAlive = true; + } current.hardFailure = false; current.hardFailureReason = undefined; if (current.sources) { @@ -29592,9 +29746,10 @@ export class TeamProvisioningService { } const canApplyBootstrapSuccess = !heartbeatReason && + !hasUnsafeProvisionedButNotAliveFailure && (current.launchState !== 'failed_to_start' || hadAutoClearableFailure || - isAutoClearableLaunchFailureReason( + isBootstrapProofClearableLaunchFailureReason( current.hardFailureReason ?? current.runtimeDiagnostic )); if (!current.bootstrapConfirmed && canApplyBootstrapSuccess) { @@ -29614,7 +29769,9 @@ export class TeamProvisioningService { if (bootstrapObservedAt && !isOpenCodeSecondaryLaneMember) { current.bootstrapConfirmed = true; current.lastHeartbeatAt = current.lastHeartbeatAt ?? bootstrapObservedAt; - current.runtimeAlive = runtimeProofObservedAt ? true : current.runtimeAlive === true; + current.runtimeAlive = runtimeProofObservedAt + ? true + : current.runtimeAlive === true || requiresConfirmedBootstrapToClearFailure; current.lastRuntimeAliveAt = runtimeProofObservedAt ? (current.lastRuntimeAliveAt ?? bootstrapObservedAt) : current.lastRuntimeAliveAt; diff --git a/src/main/services/team/provisioning/TeamProvisioningLaunchDiagnostics.ts b/src/main/services/team/provisioning/TeamProvisioningLaunchDiagnostics.ts index 4eb1290f..ccac46c0 100644 --- a/src/main/services/team/provisioning/TeamProvisioningLaunchDiagnostics.ts +++ b/src/main/services/team/provisioning/TeamProvisioningLaunchDiagnostics.ts @@ -1,6 +1,14 @@ +import { + hasUnsafeProvisionedButNotAliveRuntimeEvidence, + isBootstrapConfirmedProvisionedButNotAliveFailure, + mentionsProcessTableUnavailable, +} from '@shared/utils/teamLaunchFailureReason'; + import type { WorkspaceTrustExecutionResult } from '@features/workspace-trust/main'; import type { MemberSpawnStatusEntry, TeamLaunchDiagnosticItem } from '@shared/types'; +export { mentionsProcessTableUnavailable }; + export interface TeamProvisioningLaunchDiagnosticsRun { isLaunch: boolean; memberSpawnStatuses?: ReadonlyMap | null; @@ -12,10 +20,6 @@ interface LaunchDiagnosticsClockOptions { const defaultNowIso = (): string => new Date().toISOString(); -export function mentionsProcessTableUnavailable(value: string | undefined): boolean { - return /\bprocess table\b.*\bunavailable\b/i.test(value ?? ''); -} - export function buildLaunchDiagnosticsFromRun( run: TeamProvisioningLaunchDiagnosticsRun, options: LaunchDiagnosticsClockOptions = {} @@ -28,7 +32,24 @@ export function buildLaunchDiagnosticsFromRun( const observedAt = (options.nowIso ?? defaultNowIso)(); const items: TeamLaunchDiagnosticItem[] = []; for (const [memberName, entry] of memberSpawnStatuses.entries()) { - if (entry.launchState === 'confirmed_alive') { + const bootstrapConfirmedProvisionedButNotAlive = + isBootstrapConfirmedProvisionedButNotAliveFailure(entry); + if ( + bootstrapConfirmedProvisionedButNotAlive && + hasUnsafeProvisionedButNotAliveRuntimeEvidence(entry) + ) { + items.push({ + id: `${memberName}:bootstrap_stalled`, + memberName, + severity: 'error', + code: 'bootstrap_stalled', + label: `${memberName} - launch diagnostic error`, + detail: entry.runtimeDiagnostic ?? entry.hardFailureReason ?? entry.error, + observedAt, + }); + continue; + } + if (entry.launchState === 'confirmed_alive' || bootstrapConfirmedProvisionedButNotAlive) { items.push({ id: `${memberName}:bootstrap_confirmed`, memberName, diff --git a/src/main/services/team/provisioning/TeamProvisioningLaunchFailurePolicy.ts b/src/main/services/team/provisioning/TeamProvisioningLaunchFailurePolicy.ts index 2f6366c0..be073191 100644 --- a/src/main/services/team/provisioning/TeamProvisioningLaunchFailurePolicy.ts +++ b/src/main/services/team/provisioning/TeamProvisioningLaunchFailurePolicy.ts @@ -1,6 +1,17 @@ +import { + isProvisionedButNotAliveFailureReason, + stripProcessTableUnavailableDiagnosticSuffix, +} from '@shared/utils/teamLaunchFailureReason'; + import { mentionsProcessTableUnavailable } from './TeamProvisioningLaunchDiagnostics'; import { isBootstrapInstructionPrompt } from './TeamProvisioningPromptBuilders'; +export { + isCliProvisionedButNotAliveFailureReason, + isProvisionedButNotAliveFailureReason, + stripProcessTableUnavailableDiagnosticSuffix, +} from '@shared/utils/teamLaunchFailureReason'; + import type { MemberLaunchState } from '@shared/types'; export function isNeverSpawnedDuringLaunchReason(reason?: string): boolean { @@ -37,22 +48,6 @@ export function isProcessTableUnavailableFailureReason(reason?: string): boolean ); } -export function isCliProvisionedButNotAliveFailureReason(reason?: string): boolean { - const text = reason?.trim(); - if (!text) { - return false; - } - return /^CLI process exited \(code (?:unknown|\d+|\?)\) [\u2014-] team provisioned but not alive$/i.test( - text - ); -} - -export function stripProcessTableUnavailableDiagnosticSuffix(reason: string): string | null { - const match = /^(.*?);\s*process table (?:is )?unavailable$/i.exec(reason.trim()); - const baseReason = match?.[1]?.trim(); - return baseReason && baseReason.length > 0 ? baseReason : null; -} - function isBaseAutoClearableLaunchFailureReason(reason?: string): boolean { return ( isNeverSpawnedDuringLaunchReason(reason) || @@ -63,8 +58,7 @@ function isBaseAutoClearableLaunchFailureReason(reason?: string): boolean { isBootstrapMcpResourceReadFailureReason(reason) || isBootstrapCheckInTimeoutFailureReason(reason) || isBootstrapInstructionPromptFailureReason(reason) || - isLaunchCleanupBootstrapIncompleteFailureReason(reason) || - isCliProvisionedButNotAliveFailureReason(reason) + isLaunchCleanupBootstrapIncompleteFailureReason(reason) ); } diff --git a/src/main/services/team/provisioning/TeamProvisioningPromptBuilders.ts b/src/main/services/team/provisioning/TeamProvisioningPromptBuilders.ts index a4ce7a4c..4bc8c574 100644 --- a/src/main/services/team/provisioning/TeamProvisioningPromptBuilders.ts +++ b/src/main/services/team/provisioning/TeamProvisioningPromptBuilders.ts @@ -2,6 +2,10 @@ import { resolveTeamProviderId } from '@main/services/runtime/providerRuntimeEnv import { AGENT_BLOCK_CLOSE, AGENT_BLOCK_OPEN, wrapAgentBlock } from '@shared/constants/agentBlocks'; import { CROSS_TEAM_PREFIX_TAG } from '@shared/constants/crossTeam'; import { formatTaskDisplayLabel } from '@shared/utils/taskIdentity'; +import { + hasUnsafeProvisionedButNotAliveRuntimeEvidence, + isBootstrapConfirmedProvisionedButNotAliveFailure, +} from '@shared/utils/teamLaunchFailureReason'; import { getTeamTaskWorkflowColumn, isTeamTaskActivelyWorked, @@ -44,6 +48,57 @@ interface CanonicalSendMessageExample { const SEND_MESSAGE_CANONICAL_FIELDS = ['to', 'summary', 'message'] as const; const SEND_MESSAGE_FORBIDDEN_ALIAS_FIELDS = ['recipient', 'content'] as const; +function isUnsafeProvisionedButNotAliveStatus(status: MemberSpawnStatusEntry | undefined) { + return ( + isBootstrapConfirmedProvisionedButNotAliveFailure(status) && + hasUnsafeProvisionedButNotAliveRuntimeEvidence(status) + ); +} + +function isSafelyHealedProvisionedButNotAliveStatus(status: MemberSpawnStatusEntry | undefined) { + return ( + isBootstrapConfirmedProvisionedButNotAliveFailure(status) && + !isUnsafeProvisionedButNotAliveStatus(status) + ); +} + +function formatFailedLaunchStatus(status: MemberSpawnStatusEntry): string { + return `failed to start${status.hardFailureReason ? ` - ${status.hardFailureReason}` : status.error ? ` - ${status.error}` : ''}`; +} + +function buildTeammateLaunchStatusLabel(status: MemberSpawnStatusEntry | undefined): string { + if (!status) { + return 'runtime state unclear'; + } + if ( + status.launchState === 'failed_to_start' && + !isSafelyHealedProvisionedButNotAliveStatus(status) + ) { + return formatFailedLaunchStatus(status); + } + if ( + status.launchState === 'confirmed_alive' || + isSafelyHealedProvisionedButNotAliveStatus(status) + ) { + return 'bootstrap confirmed'; + } + if (status.launchState === 'runtime_pending_permission') { + return status.runtimeAlive + ? 'runtime online and waiting for permission approval' + : 'waiting for permission approval'; + } + if (status.runtimeAlive) { + return 'runtime online and ready for instructions'; + } + if (status.launchState === 'runtime_pending_bootstrap') { + return 'spawn accepted, runtime not confirmed yet'; + } + if (status.status === 'spawning') { + return 'spawn in progress'; + } + return 'runtime state unclear'; +} + export function buildCanonicalSendMessageExample(example: CanonicalSendMessageExample): string { return `{ ${SEND_MESSAGE_CANONICAL_FIELDS.map((field) => `${field}: "${example[field]}"`).join(', ')} }`; } @@ -1037,22 +1092,7 @@ export function buildGeminiPostLaunchHydrationPrompt( ? `Current teammate launch status:\n${members .map((member) => { const status = run.memberSpawnStatuses.get(member.name); - const label = - status?.launchState === 'failed_to_start' - ? `failed to start${status.hardFailureReason ? ` - ${status.hardFailureReason}` : status.error ? ` - ${status.error}` : ''}` - : status?.launchState === 'confirmed_alive' - ? 'bootstrap confirmed' - : status?.launchState === 'runtime_pending_permission' - ? status?.runtimeAlive - ? 'runtime online and waiting for permission approval' - : 'waiting for permission approval' - : status?.runtimeAlive - ? 'runtime online and ready for instructions' - : status?.launchState === 'runtime_pending_bootstrap' - ? 'spawn accepted, runtime not confirmed yet' - : status?.status === 'spawning' - ? 'spawn in progress' - : 'runtime state unclear'; + const label = buildTeammateLaunchStatusLabel(status); return `- @${member.name}: ${label}`; }) .join('\n')}\n` diff --git a/src/renderer/components/team/members/MemberCard.tsx b/src/renderer/components/team/members/MemberCard.tsx index 9cfd5332..034b0373 100644 --- a/src/renderer/components/team/members/MemberCard.tsx +++ b/src/renderer/components/team/members/MemberCard.tsx @@ -28,6 +28,10 @@ import { import { getRuntimeMemorySourceLabel } from '@renderer/utils/memberRuntimeSummary'; import { isLeadMember } from '@shared/utils/leadDetection'; import { deriveTaskDisplayId } from '@shared/utils/taskIdentity'; +import { + hasUnsafeProvisionedButNotAliveRuntimeEvidenceWithSpawnContext, + isBootstrapConfirmedProvisionedButNotAliveFailure, +} from '@shared/utils/teamLaunchFailureReason'; import { Activity, AlertTriangle, @@ -661,12 +665,20 @@ export const MemberCard = memo(function MemberCard({ selectedTeamName ? selectResolvedMembersForTeamName(s, selectedTeamName) : [] ); const avatarMap = useMemo(() => buildMemberAvatarMap(teamMembers), [teamMembers]); + const bootstrapConfirmedProvisionedButNotAlive = + isBootstrapConfirmedProvisionedButNotAliveFailure(spawnEntry); + const hasUnsafeBootstrapConfirmedProvisionedButNotAlive = + bootstrapConfirmedProvisionedButNotAlive && + hasUnsafeProvisionedButNotAliveRuntimeEvidenceWithSpawnContext(spawnEntry, runtimeEntry); + const effectiveSpawnStatus = spawnStatus; + const effectiveSpawnLaunchState = spawnLaunchState; const showTaskActivity = shouldDisplayMemberCurrentTask({ member, isTeamAlive, - spawnStatus, - spawnLaunchState, + spawnStatus: effectiveSpawnStatus, + spawnLaunchState: effectiveSpawnLaunchState, spawnRuntimeAlive, + spawnEntry, runtimeEntry, }); const visibleCurrentTask = showTaskActivity ? currentTask : null; @@ -680,15 +692,19 @@ export const MemberCard = memo(function MemberCard({ : member; const launchPresentation = buildMemberLaunchPresentation({ member: presentationMember, - spawnStatus, - spawnLaunchState, + spawnStatus: effectiveSpawnStatus, + spawnLaunchState: effectiveSpawnLaunchState, spawnLivenessSource, spawnRuntimeAlive, spawnBootstrapConfirmed: spawnEntry?.bootstrapConfirmed, spawnBootstrapStalled: spawnEntry?.bootstrapStalled, spawnAgentToolAccepted: spawnEntry?.agentToolAccepted, spawnHardFailure: spawnEntry?.hardFailure, + spawnHardFailureReason: spawnEntry?.hardFailureReason, + spawnError: spawnEntry?.error, + spawnRuntimeDiagnostic: spawnEntry?.runtimeDiagnostic, spawnLivenessKind: spawnEntry?.livenessKind, + spawnRuntimeDiagnosticSeverity: spawnEntry?.runtimeDiagnosticSeverity, spawnFirstSpawnAcceptedAt: spawnEntry?.firstSpawnAcceptedAt, spawnUpdatedAt: spawnEntry?.updatedAt, runtimeEntry, @@ -844,7 +860,7 @@ export const MemberCard = memo(function MemberCard({ const showStartingSkeleton = !isRemoved && presenceLabel === 'starting' && - spawnLaunchState !== 'failed_to_start' && + effectiveSpawnLaunchState !== 'failed_to_start' && !activityTask && !runtimeSummary; const usesLaunchSkeletonSurface = spawnCardClass.includes('member-waiting-shimmer'); @@ -869,8 +885,8 @@ export const MemberCard = memo(function MemberCard({ runId: runtimeRunId, memberName: member.name, member, - spawnStatus, - launchState: spawnLaunchState, + spawnStatus: effectiveSpawnStatus, + launchState: effectiveSpawnLaunchState, livenessSource: spawnLivenessSource, spawnEntry, runtimeEntry, @@ -886,9 +902,9 @@ export const MemberCard = memo(function MemberCard({ runtimeRunId, selectedTeamName, spawnEntry, - spawnLaunchState, + effectiveSpawnLaunchState, spawnLivenessSource, - spawnStatus, + effectiveSpawnStatus, ] ); const showCopyDiagnostics = @@ -900,7 +916,10 @@ export const MemberCard = memo(function MemberCard({ Boolean(runtimeAdvisoryLabel) && runtimeAdvisoryTone === 'error' && hasMemberLaunchDiagnosticsDetails(launchDiagnosticsPayload); - const isFailedLaunch = spawnStatus === 'error' || spawnLaunchState === 'failed_to_start'; + const isFailedLaunch = + (!bootstrapConfirmedProvisionedButNotAlive || + hasUnsafeBootstrapConfirmedProvisionedButNotAlive) && + (spawnStatus === 'error' || spawnLaunchState === 'failed_to_start'); const isSkippedLaunch = spawnStatus === 'skipped' || spawnLaunchState === 'skipped_for_launch' || diff --git a/src/renderer/components/team/members/MemberDetailDialog.tsx b/src/renderer/components/team/members/MemberDetailDialog.tsx index 0a1ee0ec..fa80c2b8 100644 --- a/src/renderer/components/team/members/MemberDetailDialog.tsx +++ b/src/renderer/components/team/members/MemberDetailDialog.tsx @@ -24,6 +24,10 @@ import { } from '@renderer/utils/memberRuntimeSummary'; import { isDisplayableCurrentTask } from '@renderer/utils/teamTaskDisplayState'; import { isLeadMember } from '@shared/utils/leadDetection'; +import { + hasUnsafeProvisionedButNotAliveRuntimeEvidenceWithSpawnContext, + isBootstrapConfirmedProvisionedButNotAliveFailure, +} from '@shared/utils/teamLaunchFailureReason'; import { isTeamTaskFinishedForDependency } from '@shared/utils/teamTaskState'; import { BarChart3, @@ -83,7 +87,14 @@ function isOpenCodeNoRuntimeEvidenceFailure( spawnEntry: MemberSpawnStatusEntry | undefined, runtimeEntry: TeamAgentRuntimeEntry | undefined ): boolean { - const failed = spawnEntry?.launchState === 'failed_to_start' || spawnEntry?.status === 'error'; + const bootstrapConfirmedProvisionedButNotAlive = + isBootstrapConfirmedProvisionedButNotAliveFailure(spawnEntry); + const unsafeProvisionedButNotAlive = + bootstrapConfirmedProvisionedButNotAlive && + hasUnsafeProvisionedButNotAliveRuntimeEvidenceWithSpawnContext(spawnEntry, runtimeEntry); + const failed = + (!bootstrapConfirmedProvisionedButNotAlive || unsafeProvisionedButNotAlive) && + (spawnEntry?.launchState === 'failed_to_start' || spawnEntry?.status === 'error'); return member.providerId === 'opencode' && failed && !hasOpenCodeRuntimeEvidence(runtimeEntry); } @@ -180,6 +191,7 @@ export const MemberDetailDialog = ({ spawnStatus: spawnEntry?.status, spawnLaunchState: spawnEntry?.launchState, spawnRuntimeAlive: spawnEntry?.runtimeAlive, + spawnEntry, runtimeEntry, }); const displayableCurrentTask = @@ -303,7 +315,11 @@ export const MemberDetailDialog = ({ spawnBootstrapStalled={spawnEntry?.bootstrapStalled} spawnAgentToolAccepted={spawnEntry?.agentToolAccepted} spawnHardFailure={spawnEntry?.hardFailure} + spawnHardFailureReason={spawnEntry?.hardFailureReason} + spawnError={spawnEntry?.error} + spawnRuntimeDiagnostic={spawnEntry?.runtimeDiagnostic} spawnLivenessKind={spawnEntry?.livenessKind} + spawnRuntimeDiagnosticSeverity={spawnEntry?.runtimeDiagnosticSeverity} spawnFirstSpawnAcceptedAt={spawnEntry?.firstSpawnAcceptedAt} spawnUpdatedAt={spawnEntry?.updatedAt} runtimeEntry={runtimeEntry} diff --git a/src/renderer/components/team/members/MemberDetailHeader.tsx b/src/renderer/components/team/members/MemberDetailHeader.tsx index b1c9acb7..e61d3a17 100644 --- a/src/renderer/components/team/members/MemberDetailHeader.tsx +++ b/src/renderer/components/team/members/MemberDetailHeader.tsx @@ -25,6 +25,7 @@ import type { MemberSpawnLivenessSource, MemberSpawnStatus, ResolvedTeamMember, + TeamAgentRuntimeDiagnosticSeverity, TeamAgentRuntimeEntry, } from '@shared/types'; @@ -43,7 +44,11 @@ interface MemberDetailHeaderProps { spawnBootstrapStalled?: boolean; spawnAgentToolAccepted?: boolean; spawnHardFailure?: boolean; + spawnHardFailureReason?: string; + spawnError?: string; + spawnRuntimeDiagnostic?: string; spawnLivenessKind?: TeamAgentRuntimeEntry['livenessKind']; + spawnRuntimeDiagnosticSeverity?: TeamAgentRuntimeDiagnosticSeverity; spawnFirstSpawnAcceptedAt?: string; spawnUpdatedAt?: string; isLaunchSettling?: boolean; @@ -66,7 +71,11 @@ export const MemberDetailHeader = ({ spawnBootstrapStalled, spawnAgentToolAccepted, spawnHardFailure, + spawnHardFailureReason, + spawnError, + spawnRuntimeDiagnostic, spawnLivenessKind, + spawnRuntimeDiagnosticSeverity, spawnFirstSpawnAcceptedAt, spawnUpdatedAt, isLaunchSettling, @@ -99,7 +108,11 @@ export const MemberDetailHeader = ({ spawnBootstrapStalled, spawnAgentToolAccepted, spawnHardFailure, + spawnHardFailureReason, + spawnError, + spawnRuntimeDiagnostic, spawnLivenessKind, + spawnRuntimeDiagnosticSeverity, spawnFirstSpawnAcceptedAt, spawnUpdatedAt, runtimeEntry, diff --git a/src/renderer/components/team/members/MemberHoverCard.tsx b/src/renderer/components/team/members/MemberHoverCard.tsx index 0bc94f54..8957e3d3 100644 --- a/src/renderer/components/team/members/MemberHoverCard.tsx +++ b/src/renderer/components/team/members/MemberHoverCard.tsx @@ -147,6 +147,7 @@ export const MemberHoverCard = memo(function MemberHoverCard({ spawnStatus: spawnEntry?.status, spawnLaunchState: spawnEntry?.launchState, spawnRuntimeAlive: spawnEntry?.runtimeAlive, + spawnEntry, runtimeEntry, }) ? currentTaskCandidate @@ -168,7 +169,11 @@ export const MemberHoverCard = memo(function MemberHoverCard({ spawnBootstrapStalled: spawnEntry?.bootstrapStalled, spawnAgentToolAccepted: spawnEntry?.agentToolAccepted, spawnHardFailure: spawnEntry?.hardFailure, + spawnHardFailureReason: spawnEntry?.hardFailureReason, + spawnError: spawnEntry?.error, + spawnRuntimeDiagnostic: spawnEntry?.runtimeDiagnostic, spawnLivenessKind: spawnEntry?.livenessKind, + spawnRuntimeDiagnosticSeverity: spawnEntry?.runtimeDiagnosticSeverity, spawnFirstSpawnAcceptedAt: spawnEntry?.firstSpawnAcceptedAt, spawnUpdatedAt: spawnEntry?.updatedAt, runtimeEntry, @@ -226,6 +231,7 @@ export const MemberHoverCard = memo(function MemberHoverCard({ spawnStatus: spawnEntry?.status, spawnLaunchState: spawnEntry?.launchState, spawnRuntimeAlive: spawnEntry?.runtimeAlive, + spawnEntry, runtimeEntry, }) ? reviewTaskCandidate diff --git a/src/renderer/components/team/members/MemberList.tsx b/src/renderer/components/team/members/MemberList.tsx index 5e3732a9..5f6ea327 100644 --- a/src/renderer/components/team/members/MemberList.tsx +++ b/src/renderer/components/team/members/MemberList.tsx @@ -11,6 +11,10 @@ import { buildMemberColorMap, shouldDisplayMemberCurrentTask } from '@renderer/u import { resolveMemberRuntimeSummary } from '@renderer/utils/memberRuntimeSummary'; import { isDisplayableCurrentTask } from '@renderer/utils/teamTaskDisplayState'; import { isLeadMember } from '@shared/utils/leadDetection'; +import { + hasUnsafeProvisionedButNotAliveRuntimeEvidenceWithSpawnContext, + isBootstrapConfirmedProvisionedButNotAliveFailure, +} from '@shared/utils/teamLaunchFailureReason'; import { getTeamTaskWorkflowColumn } from '@shared/utils/teamTaskState'; import { MemberCard, type RuntimeTelemetryScale } from './MemberCard'; @@ -785,6 +789,7 @@ export const MemberList = memo(function MemberList({ spawnStatus: spawnEntry?.status, spawnLaunchState: spawnEntry?.launchState, spawnRuntimeAlive: spawnEntry?.runtimeAlive, + spawnEntry, runtimeEntry, }); }, @@ -837,6 +842,7 @@ export const MemberList = memo(function MemberList({ spawnStatus: spawnEntry?.status, spawnLaunchState: spawnEntry?.launchState, spawnRuntimeAlive: spawnEntry?.runtimeAlive, + spawnEntry, runtimeEntry, }); syncMemberActivityTimer({ @@ -924,6 +930,32 @@ export const MemberList = memo(function MemberList({ {activeMembers.map((member) => { const spawnEntry = memberSpawnStatuses?.get(member.name); const runtimeEntry = memberRuntimeEntries?.get(member.name); + const bootstrapConfirmedProvisionedButNotAlive = + isBootstrapConfirmedProvisionedButNotAliveFailure(spawnEntry); + const hasUnsafeProvisionedButNotAliveEvidence = + bootstrapConfirmedProvisionedButNotAlive && + hasUnsafeProvisionedButNotAliveRuntimeEvidenceWithSpawnContext( + spawnEntry, + runtimeEntry + ); + const canPromoteBootstrapConfirmedVisualState = + bootstrapConfirmedProvisionedButNotAlive && + spawnEntry?.runtimeDiagnosticSeverity !== 'error' && + runtimeEntry?.runtimeDiagnosticSeverity !== 'error' && + !hasUnsafeProvisionedButNotAliveEvidence; + const effectiveSpawnStatus = canPromoteBootstrapConfirmedVisualState + ? 'online' + : spawnEntry?.status; + const effectiveSpawnLaunchState = canPromoteBootstrapConfirmedVisualState + ? 'confirmed_alive' + : spawnEntry?.launchState; + const useBootstrapConfirmedRuntimeAlive = + canPromoteBootstrapConfirmedVisualState && + runtimeEntry?.runtimeDiagnosticSeverity !== 'error' && + spawnEntry?.runtimeDiagnosticSeverity !== 'error'; + const effectiveSpawnRuntimeAlive = useBootstrapConfirmedRuntimeAlive + ? true + : spawnEntry?.runtimeAlive; const currentTaskCandidate = member.currentTaskId && taskMap ? (taskMap.get(member.currentTaskId) ?? null) : null; const currentTask = @@ -931,9 +963,10 @@ export const MemberList = memo(function MemberList({ shouldDisplayMemberCurrentTask({ member, isTeamAlive, - spawnStatus: spawnEntry?.status, - spawnLaunchState: spawnEntry?.launchState, - spawnRuntimeAlive: spawnEntry?.runtimeAlive, + spawnStatus: effectiveSpawnStatus, + spawnLaunchState: effectiveSpawnLaunchState, + spawnRuntimeAlive: effectiveSpawnRuntimeAlive, + spawnEntry, runtimeEntry, }) ? currentTaskCandidate @@ -945,9 +978,10 @@ export const MemberList = memo(function MemberList({ shouldDisplayMemberCurrentTask({ member, isTeamAlive, - spawnStatus: spawnEntry?.status, - spawnLaunchState: spawnEntry?.launchState, - spawnRuntimeAlive: spawnEntry?.runtimeAlive, + spawnStatus: effectiveSpawnStatus, + spawnLaunchState: effectiveSpawnLaunchState, + spawnRuntimeAlive: effectiveSpawnRuntimeAlive, + spawnEntry, runtimeEntry, }) ? reviewCandidate @@ -995,12 +1029,16 @@ export const MemberList = memo(function MemberList({ runtimeSummary={buildRuntimeSummary(member, spawnEntry, runtimeEntry)} runtimeEntry={runtimeEntry} runtimeRunId={runtimeRunId} - spawnStatus={spawnEntry?.status} + spawnStatus={effectiveSpawnStatus} spawnEntry={spawnEntry} - spawnError={spawnEntry?.error ?? spawnEntry?.hardFailureReason} + spawnError={ + canPromoteBootstrapConfirmedVisualState + ? undefined + : (spawnEntry?.error ?? spawnEntry?.hardFailureReason) + } spawnLivenessSource={spawnEntry?.livenessSource} - spawnLaunchState={spawnEntry?.launchState} - spawnRuntimeAlive={spawnEntry?.runtimeAlive} + spawnLaunchState={effectiveSpawnLaunchState} + spawnRuntimeAlive={effectiveSpawnRuntimeAlive} isTeamAlive={isTeamAlive} isTeamProvisioning={isTeamProvisioning} leadActivity={leadActivity} diff --git a/src/renderer/components/team/provisioningSteps.ts b/src/renderer/components/team/provisioningSteps.ts index a8fb62bb..3f34dd2d 100644 --- a/src/renderer/components/team/provisioningSteps.ts +++ b/src/renderer/components/team/provisioningSteps.ts @@ -1,4 +1,10 @@ import { isLeadMember } from '@shared/utils/leadDetection'; +import { + hasUnsafeProvisionedButNotAliveRuntimeEvidence, + hasUnsafeProvisionedButNotAliveRuntimeEvidenceWithSpawnContext, + isBootstrapConfirmedProvisionedButNotAliveFailure, + mentionsProcessTableUnavailable, +} from '@shared/utils/teamLaunchFailureReason'; import type { MemberSpawnStatusEntry, @@ -80,6 +86,9 @@ function parseStatusUpdatedAtMs(value: string | undefined): number | null { } function isFailedSpawnEntry(entry: MemberSpawnStatusEntry | undefined): boolean { + if (isBootstrapConfirmedProvisionedButNotAliveFailure(entry)) { + return hasUnsafeProvisionedButNotAliveRuntimeEvidence(entry); + } return entry?.launchState === 'failed_to_start' || entry?.status === 'error'; } @@ -92,13 +101,62 @@ function isStrongRuntimeProcessSpawnEntry(entry: MemberSpawnStatusEntry): boolea } function isConfirmedSpawnEntry(entry: MemberSpawnStatusEntry): boolean { + if (isBootstrapConfirmedProvisionedButNotAliveFailure(entry)) { + return !isFailedSpawnEntry(entry); + } return entry.launchState === 'confirmed_alive' || entry.bootstrapConfirmed === true; } +function spawnEntryContradictsConfirmedJoin(entry: MemberSpawnStatusEntry): boolean { + if (!isConfirmedSpawnEntry(entry) || entry.runtimeAlive !== false) { + return false; + } + if (entry.runtimeDiagnosticSeverity === 'error') { + return true; + } + if ( + entry.livenessKind === 'not_found' || + entry.livenessKind === 'shell_only' || + entry.livenessKind === 'permission_blocked' || + entry.livenessKind === 'runtime_process_candidate' + ) { + return true; + } + const hasProcessTableUnavailableMarker = + mentionsProcessTableUnavailable(entry.runtimeDiagnostic) || + mentionsProcessTableUnavailable(entry.hardFailureReason) || + mentionsProcessTableUnavailable(entry.error); + if (!entry.livenessKind) { + return !hasProcessTableUnavailableMarker; + } + if (entry.livenessKind !== 'registered_only' && entry.livenessKind !== 'stale_metadata') { + return false; + } + return !hasProcessTableUnavailableMarker; +} + function runtimeEntryContradictsConfirmedJoin( + entry: MemberSpawnStatusEntry, runtimeEntry: TeamAgentRuntimeEntry | undefined ): boolean { - return runtimeEntry?.alive === false; + if (runtimeEntry?.alive !== false || runtimeEntry.livenessKind === 'confirmed_bootstrap') { + return false; + } + if ( + isBootstrapConfirmedProvisionedButNotAliveFailure(entry) && + !hasUnsafeProvisionedButNotAliveRuntimeEvidence(entry) && + !hasUnsafeProvisionedButNotAliveRuntimeEvidenceWithSpawnContext(entry, runtimeEntry) && + (runtimeEntry.livenessKind == null || + runtimeEntry.livenessKind === 'registered_only' || + runtimeEntry.livenessKind === 'stale_metadata') && + (mentionsProcessTableUnavailable(runtimeEntry.runtimeDiagnostic) || + mentionsProcessTableUnavailable(entry.runtimeDiagnostic) || + mentionsProcessTableUnavailable(entry.hardFailureReason) || + mentionsProcessTableUnavailable(entry.error)) + ) { + return false; + } + return true; } function shouldPreferSnapshotEntryOverLive( @@ -159,7 +217,7 @@ function summarizeLiveLaunchJoinMilestones(params: { continue; } observedTeammateCount += 1; - if (entry.launchState === 'failed_to_start') { + if (isFailedSpawnEntry(entry)) { failedSpawnCount += 1; continue; } @@ -167,14 +225,21 @@ function summarizeLiveLaunchJoinMilestones(params: { skippedSpawnCount += 1; continue; } + if (spawnEntryContradictsConfirmedJoin(entry)) { + pendingSpawnCount += 1; + continue; + } if ( isConfirmedSpawnEntry(entry) && - runtimeEntryContradictsConfirmedJoin(getRuntimeEntry(params.memberRuntimeEntries, memberName)) + runtimeEntryContradictsConfirmedJoin( + entry, + getRuntimeEntry(params.memberRuntimeEntries, memberName) + ) ) { pendingSpawnCount += 1; continue; } - if (entry.launchState === 'confirmed_alive') { + if (isConfirmedSpawnEntry(entry)) { heartbeatConfirmedCount += 1; continue; } diff --git a/src/renderer/components/team/teamRuntimeDisplayRows.ts b/src/renderer/components/team/teamRuntimeDisplayRows.ts index 0737cbf0..65367a15 100644 --- a/src/renderer/components/team/teamRuntimeDisplayRows.ts +++ b/src/renderer/components/team/teamRuntimeDisplayRows.ts @@ -1,3 +1,9 @@ +import { + hasUnsafeProvisionedButNotAliveRuntimeEvidence, + hasUnsafeProvisionedButNotAliveRuntimeEvidenceWithSpawnContext, + isBootstrapConfirmedProvisionedButNotAliveFailure, +} from '@shared/utils/teamLaunchFailureReason'; + import type { MemberSpawnStatusEntry, TeamAgentRuntimeDiagnosticSeverity, @@ -139,15 +145,29 @@ function buildRuntimeBackedDisplayRow( spawn?: MemberSpawnStatusEntry ): TeamRuntimeDisplayRow { const hasErrorDiagnostic = runtime.runtimeDiagnosticSeverity === 'error'; + const bootstrapConfirmedProvisionedButNotAlive = + isBootstrapConfirmedProvisionedButNotAliveFailure(spawn); const spawnDegradation = getSpawnDegradation(spawn); + const unsafeRuntimeEvidence = hasUnsafeProvisionedButNotAliveRuntimeEvidenceWithSpawnContext( + spawn, + runtime + ); + const useBootstrapConfirmedState = + bootstrapConfirmedProvisionedButNotAlive && + !hasErrorDiagnostic && + !unsafeRuntimeEvidence && + spawnDegradation == null; const spawnStoppedEvidence = spawnDegradation ? null : getSpawnStoppedEvidence(runtime, spawn); - const state = spawnStoppedEvidence - ? 'stopped' - : getRuntimeBackedState(runtime, hasErrorDiagnostic, spawnDegradation != null); + const state = useBootstrapConfirmedState + ? 'running' + : spawnStoppedEvidence + ? 'stopped' + : getRuntimeBackedState(runtime, hasErrorDiagnostic, spawnDegradation != null); const degradedReason = spawnDegradation ? withLiveProcessContext(spawnDegradation.reason, runtime) : undefined; const stateReason = + (useBootstrapConfirmedState ? 'Bootstrap confirmed' : undefined) ?? degradedReason ?? spawnStoppedEvidence?.reason ?? runtime.runtimeDiagnostic ?? @@ -181,6 +201,17 @@ function buildRuntimeBackedDisplayRow( function getSpawnDegradation(spawn?: MemberSpawnStatusEntry): SpawnDegradation | null { if (!spawn) return null; + if (isBootstrapConfirmedProvisionedButNotAliveFailure(spawn)) { + if (!hasUnsafeProvisionedButNotAliveRuntimeEvidence(spawn)) { + return null; + } + const reason = spawn.runtimeDiagnostic ?? 'Runtime launch status needs attention'; + return { + reason, + diagnostic: spawn.runtimeDiagnostic ?? reason, + diagnosticSeverity: spawn.runtimeDiagnosticSeverity === 'error' ? 'error' : 'warning', + }; + } if (spawn.status === 'error' || spawn.hardFailure === true) { const reason = @@ -226,7 +257,10 @@ function getSpawnStoppedEvidence( runtime: TeamAgentRuntimeEntry, spawn?: MemberSpawnStatusEntry ): SpawnStoppedEvidence | null { - if (!spawn || spawn.runtimeAlive !== false || runtime.livenessKind !== 'confirmed_bootstrap') { + if (isBootstrapConfirmedProvisionedButNotAliveFailure(spawn)) { + return null; + } + if (spawn?.runtimeAlive !== false || runtime.livenessKind !== 'confirmed_bootstrap') { return null; } if (spawn.status !== 'online' && spawn.launchState !== 'confirmed_alive') { @@ -267,6 +301,23 @@ function buildSpawnBackedDisplayRow( memberName: string, spawn: MemberSpawnStatusEntry ): TeamRuntimeDisplayRow { + if ( + isBootstrapConfirmedProvisionedButNotAliveFailure(spawn) && + !hasUnsafeProvisionedButNotAliveRuntimeEvidence(spawn) + ) { + return { + memberName, + state: 'running', + stateReason: 'Bootstrap confirmed', + source: 'spawn-status', + updatedAt: spawn.livenessLastCheckedAt ?? spawn.lastHeartbeatAt ?? spawn.updatedAt, + runtimeModel: spawn.runtimeModel, + diagnostic: spawn.runtimeDiagnostic, + diagnosticSeverity: spawn.runtimeDiagnosticSeverity, + actionsAllowed: false, + }; + } + const spawnDegradation = getSpawnDegradation(spawn); if (spawnDegradation) { return { @@ -359,6 +410,7 @@ function buildSpawnBackedDisplayRow( } function getSpawnOnlyStoppedEvidence(spawn: MemberSpawnStatusEntry): SpawnStoppedEvidence | null { + if (isBootstrapConfirmedProvisionedButNotAliveFailure(spawn)) return null; if (spawn.runtimeAlive !== false) return null; if (spawn.status !== 'online' && spawn.launchState !== 'confirmed_alive') return null; diff --git a/src/renderer/store/slices/teamSlice.ts b/src/renderer/store/slices/teamSlice.ts index 23063056..a2189160 100644 --- a/src/renderer/store/slices/teamSlice.ts +++ b/src/renderer/store/slices/teamSlice.ts @@ -4117,6 +4117,27 @@ export const createTeamSlice: StateCreator = (set, operation: 'fetchTeams', }); void get().fetchTeams(); + const terminalRefreshState = get(); + if (isVisibleInActiveTeamSurface(terminalRefreshState, progress.teamName)) { + noteTeamRefreshFanout({ + teamName: progress.teamName, + surface: 'provisioning-progress', + phase: 'scheduled', + reason: terminalReason, + operation: 'fetchMemberSpawnStatuses', + visible: true, + }); + void terminalRefreshState.fetchMemberSpawnStatuses(progress.teamName); + noteTeamRefreshFanout({ + teamName: progress.teamName, + surface: 'provisioning-progress', + phase: 'scheduled', + reason: terminalReason, + operation: 'fetchTeamAgentRuntime', + visible: true, + }); + void terminalRefreshState.fetchTeamAgentRuntime(progress.teamName); + } if (hydratedVisibleTeam) { noteTeamRefreshFanout({ teamName: progress.teamName, diff --git a/src/renderer/utils/memberHelpers.ts b/src/renderer/utils/memberHelpers.ts index 85ea1d2a..2f5b1d00 100644 --- a/src/renderer/utils/memberHelpers.ts +++ b/src/renderer/utils/memberHelpers.ts @@ -1,4 +1,8 @@ import { isLeadMember } from '@shared/utils/leadDetection'; +import { + hasUnsafeProvisionedButNotAliveRuntimeEvidenceWithSpawnContext, + isBootstrapConfirmedProvisionedButNotAliveFailure, +} from '@shared/utils/teamLaunchFailureReason'; import { buildTeamMemberColorMap } from '@shared/utils/teamMemberColors'; import { @@ -17,6 +21,7 @@ import type { MemberSpawnStatusEntry, MemberStatus, ResolvedTeamMember, + TeamAgentRuntimeDiagnosticSeverity, TeamAgentRuntimeEntry, TeamProviderId, TeamReviewState, @@ -394,7 +399,7 @@ const OPENCODE_SESSION_REFRESH_REASON_PATTERN = /\b(?:resolved_behavior_changed|opencode_app_mcp_transport_changed):[-a-z0-9._~/=]+->[-a-z0-9._~/=]+/i; const OPENCODE_SESSION_REFRESH_FAILURE_PATTERN = // eslint-disable-next-line sonarjs/regex-complexity -- Keyword taxonomy is kept literal to preserve diagnostic behavior. - /(?:^|[_\s:;.\/()-])(?:permission[_\s-]?denied|permission[_\s-]?blocked|access[_\s-]?denied|auth[_\s-]?unavailable|authentication[_\s-]?failed|unauthorized|forbidden|401|403|login[_\s-]?required|not\s+logged\s+in|missing\s+credentials?|invalid\s+credentials?|credentials?[_\s-]?required|credentials?[_\s-]?unavailable|no auth available|authorization|auth(?:entication)?(?:[_\s-]?(?:failed|unavailable))?|invalid api[_\s-]?key|api[_\s-]?key|does not have access|quota|rate[_\s-]?(?:limit|limited)|too many requests|429|model cooldown|cooling down|enospc|no space left|disk is full|capacity exceeded|quota exhausted|usage exceeded|free usage exceeded|key limit exceeded|total limit|insufficient credits|subscribe to go|error|failed|failure|timeout|timed\s+out|network|connection|unable\s+to\s+connect|connect\s+failed|econn[a-z_]*|enotfound|fetch[_\s-]?failed|connection[_\s-]?(?:refused|reset)|aborted|cancel(?:ed|led)|interrupted|service[_\s-]?unavailable|temporarily\s+unavailable|overloaded|visible[_\s-]?reply(?:[_\s-][a-z0-9]+)*|task[_\s-]?refs|relayofmessageid|relay[_\s-]?of[_\s-]?message[_\s-]?id|message[_\s-]?send|non[_\s-]?visible[_\s-]?tool(?:[_\s-][a-z0-9]+)*|protocol[_\s-]?proof)(?=$|[_\s:;.\/(),-])/i; + /(?:^|[_\s:;./()-])(?:permission[_\s-]?denied|permission[_\s-]?blocked|access[_\s-]?denied|auth[_\s-]?unavailable|authentication[_\s-]?failed|unauthorized|forbidden|401|403|login[_\s-]?required|not\s+logged\s+in|missing\s+credentials?|invalid\s+credentials?|credentials?[_\s-]?required|credentials?[_\s-]?unavailable|no auth available|authorization|auth(?:entication)?(?:[_\s-]?(?:failed|unavailable))?|invalid api[_\s-]?key|api[_\s-]?key|does not have access|quota|rate[_\s-]?(?:limit|limited)|too many requests|429|model cooldown|cooling down|enospc|no space left|disk is full|capacity exceeded|quota exhausted|usage exceeded|free usage exceeded|key limit exceeded|total limit|insufficient credits|subscribe to go|error|failed|failure|timeout|timed\s+out|network|connection|unable\s+to\s+connect|connect\s+failed|econn[a-z_]*|enotfound|fetch[_\s-]?failed|connection[_\s-]?(?:refused|reset)|aborted|cancel(?:ed|led)|interrupted|service[_\s-]?unavailable|temporarily\s+unavailable|overloaded|visible[_\s-]?reply(?:[_\s-][a-z0-9]+)*|task[_\s-]?refs|relayofmessageid|relay[_\s-]?of[_\s-]?message[_\s-]?id|message[_\s-]?send|non[_\s-]?visible[_\s-]?tool(?:[_\s-][a-z0-9]+)*|protocol[_\s-]?proof)(?=$|[_\s:;./(),-])/i; const OPENCODE_SESSION_REFRESH_ANY_REASON_PATTERN = /\b(?:resolved_behavior_changed|opencode_app_mcp_transport_changed):[-a-z0-9._~/=]+->[-a-z0-9._~/=]+/gi; const OPENCODE_SESSION_REFRESH_SAFE_MARKER_STATE_PATTERN = @@ -983,6 +988,17 @@ function getCurrentRuntimeOfflineVisualState( return null; } +function hasStoppedRuntimeLivenessKind( + livenessKind: TeamAgentRuntimeEntry['livenessKind'] | undefined +): boolean { + return ( + livenessKind === 'not_found' || + livenessKind === 'registered_only' || + livenessKind === 'shell_only' || + livenessKind === 'stale_metadata' + ); +} + function isCodexNativeProcessTeammate(member: ResolvedTeamMember): boolean { if (isLeadMember(member)) { return false; @@ -1076,6 +1092,7 @@ export function shouldDisplayMemberCurrentTask({ spawnStatus, spawnLaunchState, spawnRuntimeAlive, + spawnEntry, runtimeEntry, }: { member: ResolvedTeamMember; @@ -1083,36 +1100,64 @@ export function shouldDisplayMemberCurrentTask({ spawnStatus?: MemberSpawnStatus; spawnLaunchState?: MemberLaunchState; spawnRuntimeAlive?: boolean; + spawnEntry?: MemberSpawnStatusEntry; runtimeEntry?: TeamAgentRuntimeEntry; }): boolean { + const bootstrapConfirmedProvisionedButNotAlive = + isBootstrapConfirmedProvisionedButNotAliveFailure(spawnEntry); + const unsafeProvisionedButNotAliveEvidence = + bootstrapConfirmedProvisionedButNotAlive && + hasUnsafeProvisionedButNotAliveRuntimeEvidenceWithSpawnContext(spawnEntry, runtimeEntry); + const useBootstrapConfirmedVisualState = + bootstrapConfirmedProvisionedButNotAlive && !unsafeProvisionedButNotAliveEvidence; + const effectiveSpawnStatus = useBootstrapConfirmedVisualState ? 'online' : spawnStatus; + const effectiveSpawnLaunchState = useBootstrapConfirmedVisualState + ? 'confirmed_alive' + : spawnLaunchState; + const effectiveSpawnRuntimeAlive = useBootstrapConfirmedVisualState ? true : spawnRuntimeAlive; if (member.removedAt || member.status === 'terminated') { return false; } if (isTeamAlive === false) { return false; } - if (spawnStatus === 'offline' || spawnStatus === 'error' || spawnStatus === 'skipped') { - return false; - } if ( - spawnLaunchState === 'failed_to_start' || - spawnLaunchState === 'skipped_for_launch' || - spawnLaunchState === 'runtime_pending_permission' + effectiveSpawnStatus === 'offline' || + effectiveSpawnStatus === 'error' || + effectiveSpawnStatus === 'skipped' ) { return false; } if ( - runtimeEntry?.livenessKind === 'shell_only' || - runtimeEntry?.livenessKind === 'registered_only' || - runtimeEntry?.livenessKind === 'stale_metadata' || - runtimeEntry?.livenessKind === 'not_found' + effectiveSpawnLaunchState === 'failed_to_start' || + effectiveSpawnLaunchState === 'skipped_for_launch' || + effectiveSpawnLaunchState === 'runtime_pending_permission' ) { return false; } - if (runtimeEntry?.alive === false) { + if ( + !useBootstrapConfirmedVisualState && + (runtimeEntry?.livenessKind === 'shell_only' || + spawnEntry?.livenessKind === 'shell_only' || + runtimeEntry?.livenessKind === 'registered_only' || + spawnEntry?.livenessKind === 'registered_only' || + runtimeEntry?.livenessKind === 'stale_metadata' || + spawnEntry?.livenessKind === 'stale_metadata' || + runtimeEntry?.livenessKind === 'not_found' || + spawnEntry?.livenessKind === 'not_found') + ) { return false; } - if (spawnRuntimeAlive === false) { + if (runtimeEntry?.runtimeDiagnosticSeverity === 'error') { + return false; + } + if (spawnEntry?.runtimeDiagnosticSeverity === 'error') { + return false; + } + if (runtimeEntry?.alive === false && !useBootstrapConfirmedVisualState) { + return false; + } + if (effectiveSpawnRuntimeAlive === false) { return false; } if (isCodexNativeProcessTeammate(member) && !hasLiveRuntimeProcessEvidence(runtimeEntry)) { @@ -1228,6 +1273,9 @@ export function isOpenCodeRelaunchActionable({ runtimeEntry?.livenessKind === 'stale_metadata' ); } + if (isBootstrapConfirmedProvisionedButNotAliveFailure(spawnEntry)) { + return hasUnsafeProvisionedButNotAliveRuntimeEvidenceWithSpawnContext(spawnEntry, runtimeEntry); + } if ( spawnEntry?.launchState === 'failed_to_start' || spawnEntry?.launchState === 'skipped_for_launch' || @@ -1280,7 +1328,11 @@ export function buildMemberLaunchPresentation({ spawnBootstrapStalled, spawnAgentToolAccepted, spawnHardFailure, + spawnHardFailureReason, + spawnError, + spawnRuntimeDiagnostic, spawnLivenessKind, + spawnRuntimeDiagnosticSeverity, spawnFirstSpawnAcceptedAt, spawnUpdatedAt, runtimeAdvisory, @@ -1300,7 +1352,11 @@ export function buildMemberLaunchPresentation({ spawnBootstrapStalled?: boolean; spawnAgentToolAccepted?: boolean; spawnHardFailure?: boolean; + spawnHardFailureReason?: string; + spawnError?: string; + spawnRuntimeDiagnostic?: string; spawnLivenessKind?: TeamAgentRuntimeEntry['livenessKind']; + spawnRuntimeDiagnosticSeverity?: TeamAgentRuntimeDiagnosticSeverity; spawnFirstSpawnAcceptedAt?: string; spawnUpdatedAt?: string; runtimeAdvisory: MemberRuntimeAdvisory | undefined; @@ -1311,46 +1367,105 @@ export function buildMemberLaunchPresentation({ leadActivity?: LeadActivityState; nowMs?: number; }): MemberLaunchPresentation { + const bootstrapConfirmedProvisionedButNotAlive = + isBootstrapConfirmedProvisionedButNotAliveFailure({ + status: spawnStatus, + launchState: spawnLaunchState, + hardFailure: spawnHardFailure, + hardFailureReason: spawnHardFailureReason, + error: spawnError, + runtimeDiagnostic: spawnRuntimeDiagnostic, + runtimeDiagnosticSeverity: spawnRuntimeDiagnosticSeverity, + bootstrapConfirmed: spawnBootstrapConfirmed, + livenessKind: spawnLivenessKind ?? runtimeEntry?.livenessKind, + }); + const hasSpawnRuntimeErrorDiagnostic = spawnRuntimeDiagnosticSeverity === 'error'; + const hasRuntimeErrorDiagnostic = runtimeEntry?.runtimeDiagnosticSeverity === 'error'; + const hasUnsafeProvisionedButNotAliveEvidence = + bootstrapConfirmedProvisionedButNotAlive && + hasUnsafeProvisionedButNotAliveRuntimeEvidenceWithSpawnContext( + { + status: spawnStatus, + launchState: spawnLaunchState, + hardFailure: spawnHardFailure, + hardFailureReason: spawnHardFailureReason, + error: spawnError, + runtimeDiagnostic: spawnRuntimeDiagnostic, + runtimeDiagnosticSeverity: spawnRuntimeDiagnosticSeverity, + bootstrapConfirmed: spawnBootstrapConfirmed, + livenessKind: spawnLivenessKind, + }, + runtimeEntry + ); + const allowBootstrapConfirmedVisualPromotion = + bootstrapConfirmedProvisionedButNotAlive && + !hasSpawnRuntimeErrorDiagnostic && + !hasRuntimeErrorDiagnostic && + !hasUnsafeProvisionedButNotAliveEvidence; + const useBootstrapConfirmedRuntimeAlive = + allowBootstrapConfirmedVisualPromotion && !hasRuntimeErrorDiagnostic; + const suppressConfirmedLaunchRuntimeAlivePromotion = + bootstrapConfirmedProvisionedButNotAlive && !useBootstrapConfirmedRuntimeAlive; + const visualSpawnStatus = allowBootstrapConfirmedVisualPromotion ? 'online' : spawnStatus; + const visualSpawnLaunchState = allowBootstrapConfirmedVisualPromotion + ? 'confirmed_alive' + : spawnLaunchState; + const visualSpawnRuntimeAlive = useBootstrapConfirmedRuntimeAlive ? true : spawnRuntimeAlive; + const visualSpawnBootstrapConfirmed = allowBootstrapConfirmedVisualPromotion + ? true + : spawnBootstrapConfirmed; + const visualSpawnHardFailure = allowBootstrapConfirmedVisualPromotion ? false : spawnHardFailure; + const visualSpawnLivenessKind = allowBootstrapConfirmedVisualPromotion + ? 'confirmed_bootstrap' + : spawnLivenessKind; + const visualRuntimeEntry = + useBootstrapConfirmedRuntimeAlive && runtimeEntry + ? ({ + ...runtimeEntry, + alive: true, + livenessKind: 'confirmed_bootstrap', + } satisfies TeamAgentRuntimeEntry) + : runtimeEntry; const currentRuntimeOfflineVisualState = getCurrentRuntimeOfflineVisualState( member, - runtimeEntry, - spawnStatus, - spawnLaunchState, - spawnRuntimeAlive, - spawnBootstrapConfirmed, + visualRuntimeEntry, + visualSpawnStatus, + visualSpawnLaunchState, + visualSpawnRuntimeAlive, + visualSpawnBootstrapConfirmed, isTeamProvisioning ); const hasConfirmedSpawnLaunch = - spawnLaunchState === 'confirmed_alive' && spawnBootstrapConfirmed === true; + visualSpawnLaunchState === 'confirmed_alive' && visualSpawnBootstrapConfirmed === true; const suppressOpenCodeAppMcpAdvisory = isHealthyOpenCodeAppMcpConnectivityAdvisory({ providerId: member.providerId, runtimeAdvisory, - spawnStatus, - launchState: spawnLaunchState, - runtimeAlive: spawnRuntimeAlive, - bootstrapConfirmed: spawnBootstrapConfirmed, + spawnStatus: visualSpawnStatus, + launchState: visualSpawnLaunchState, + runtimeAlive: visualSpawnRuntimeAlive, + bootstrapConfirmed: visualSpawnBootstrapConfirmed, agentToolAccepted: spawnAgentToolAccepted, - hardFailure: spawnHardFailure, - livenessKind: spawnLivenessKind ?? runtimeEntry?.livenessKind, - runtimeEntry, + hardFailure: visualSpawnHardFailure, + livenessKind: visualSpawnLivenessKind ?? visualRuntimeEntry?.livenessKind, + runtimeEntry: visualRuntimeEntry, }); const displayRuntimeAdvisory = suppressOpenCodeAppMcpAdvisory ? undefined : runtimeAdvisory; const effectiveSpawnStatus = hasConfirmedSpawnLaunch && currentRuntimeOfflineVisualState == null && - (spawnStatus === 'waiting' || spawnStatus === 'spawning') + (visualSpawnStatus === 'waiting' || visualSpawnStatus === 'spawning') ? 'online' - : spawnStatus; + : visualSpawnStatus; const effectiveSpawnRuntimeAlive = currentRuntimeOfflineVisualState != null ? false - : hasConfirmedSpawnLaunch + : hasConfirmedSpawnLaunch && !suppressConfirmedLaunchRuntimeAlivePromotion ? true - : spawnRuntimeAlive; + : visualSpawnRuntimeAlive; const presenceLabel = getLaunchAwarePresenceLabel( member, effectiveSpawnStatus, - spawnLaunchState, + visualSpawnLaunchState, spawnLivenessSource, effectiveSpawnRuntimeAlive, displayRuntimeAdvisory, @@ -1362,7 +1477,7 @@ export function buildMemberLaunchPresentation({ const baseDotClass = getSpawnAwareDotClass( member, effectiveSpawnStatus, - spawnLaunchState, + visualSpawnLaunchState, effectiveSpawnRuntimeAlive, isLaunchSettling, isTeamAlive, @@ -1371,7 +1486,7 @@ export function buildMemberLaunchPresentation({ ); const cardClass = getSpawnCardClass( effectiveSpawnStatus, - spawnLaunchState, + visualSpawnLaunchState, effectiveSpawnRuntimeAlive, isLaunchSettling, isTeamAlive, @@ -1393,8 +1508,8 @@ export function buildMemberLaunchPresentation({ const startingIsStale = !hasConfirmedSpawnLaunch && isMemberStartingStale({ - spawnStatus, - spawnLaunchState, + spawnStatus: visualSpawnStatus, + spawnLaunchState: visualSpawnLaunchState, spawnFirstSpawnAcceptedAt, spawnUpdatedAt, nowMs, @@ -1402,19 +1517,19 @@ export function buildMemberLaunchPresentation({ let launchVisualState: MemberLaunchVisualState = null; if (isTeamAlive !== false || isTeamProvisioning) { - if (spawnLaunchState === 'failed_to_start' || spawnStatus === 'error') { + if (visualSpawnLaunchState === 'failed_to_start' || visualSpawnStatus === 'error') { launchVisualState = 'error'; - } else if (spawnLaunchState === 'skipped_for_launch' || spawnStatus === 'skipped') { + } else if (visualSpawnLaunchState === 'skipped_for_launch' || visualSpawnStatus === 'skipped') { launchVisualState = 'skipped'; - } else if (spawnLaunchState === 'runtime_pending_permission') { + } else if (visualSpawnLaunchState === 'runtime_pending_permission') { launchVisualState = 'permission_pending'; } else if (spawnBootstrapStalled === true) { launchVisualState = 'bootstrap_stalled'; } else if (currentRuntimeOfflineVisualState != null) { launchVisualState = currentRuntimeOfflineVisualState; - } else if (runtimeEntry?.livenessKind === 'shell_only') { + } else if (visualRuntimeEntry?.livenessKind === 'shell_only') { launchVisualState = 'shell_only'; - } else if (runtimeEntry?.livenessKind === 'runtime_process_candidate') { + } else if (visualRuntimeEntry?.livenessKind === 'runtime_process_candidate') { launchVisualState = 'runtime_candidate'; } else if (!hasConfirmedSpawnLaunch && startingIsStale) { launchVisualState = 'starting_stale'; @@ -1422,9 +1537,9 @@ export function buildMemberLaunchPresentation({ !hasConfirmedSpawnLaunch && isQueuedOpenCodeLaunch( member, - spawnStatus, - spawnLaunchState, - runtimeEntry, + visualSpawnStatus, + visualSpawnLaunchState, + visualRuntimeEntry, isLaunchSettling, isTeamProvisioning ) @@ -1433,21 +1548,21 @@ export function buildMemberLaunchPresentation({ } else if ( !hasConfirmedSpawnLaunch && isLaunchStillStarting( - spawnStatus, - spawnLaunchState, - spawnRuntimeAlive, + visualSpawnStatus, + visualSpawnLaunchState, + visualSpawnRuntimeAlive, keepLaunchSettlingVisuals ) ) { - launchVisualState = spawnStatus === 'spawning' ? 'spawning' : 'waiting'; + launchVisualState = visualSpawnStatus === 'spawning' ? 'spawning' : 'waiting'; } else if ( !hasConfirmedSpawnLaunch && - spawnLaunchState === 'runtime_pending_bootstrap' && - (runtimeEntry?.livenessKind === 'runtime_process' || - (spawnStatus === 'online' && spawnRuntimeAlive === true)) + visualSpawnLaunchState === 'runtime_pending_bootstrap' && + (visualRuntimeEntry?.livenessKind === 'runtime_process' || + (visualSpawnStatus === 'online' && visualSpawnRuntimeAlive === true)) ) { launchVisualState = 'runtime_pending'; - } else if (isLaunchSettling && spawnLaunchState === 'confirmed_alive') { + } else if (isLaunchSettling && visualSpawnLaunchState === 'confirmed_alive') { launchVisualState = 'settling'; } } @@ -1471,12 +1586,12 @@ export function buildMemberLaunchPresentation({ ? (launchStatusLabel ?? presenceLabel) : presenceLabel; const spawnBadgeLabel = - spawnStatus && spawnStatus !== 'online' - ? spawnStatus === 'waiting' || spawnStatus === 'spawning' + effectiveSpawnStatus && effectiveSpawnStatus !== 'online' + ? effectiveSpawnStatus === 'waiting' || effectiveSpawnStatus === 'spawning' ? startingIsStale ? 'starting stale' : 'starting' - : spawnStatus + : effectiveSpawnStatus : null; return { diff --git a/src/renderer/utils/memberLaunchDiagnostics.ts b/src/renderer/utils/memberLaunchDiagnostics.ts index fe316b72..e4efbeeb 100644 --- a/src/renderer/utils/memberLaunchDiagnostics.ts +++ b/src/renderer/utils/memberLaunchDiagnostics.ts @@ -1,3 +1,9 @@ +import { + hasUnsafeProvisionedButNotAliveRuntimeEvidence, + hasUnsafeProvisionedButNotAliveRuntimeEvidenceWithSpawnContext, + isBootstrapConfirmedProvisionedButNotAliveFailure, +} from '@shared/utils/teamLaunchFailureReason'; + import { isHealthyOpenCodeAppMcpConnectivityAdvisory } from './openCodeAdvisoryHealth'; import type { @@ -87,6 +93,15 @@ const SECRET_ENV_KEY_PARTS = [ 'PASSWORD', 'AUTHORIZATION', ]; + +function hasStoppedRuntimeLivenessKind(livenessKind: TeamAgentRuntimeLivenessKind | undefined) { + return ( + livenessKind === 'not_found' || + livenessKind === 'registered_only' || + livenessKind === 'shell_only' || + livenessKind === 'stale_metadata' + ); +} const OPENCODE_SESSION_REFRESH_REASON_MARKERS = [ 'resolved_behavior_changed', 'opencode_app_mcp_transport_changed', @@ -94,7 +109,7 @@ const OPENCODE_SESSION_REFRESH_REASON_MARKERS = [ const OPENCODE_SESSION_REFRESH_REASON_CHARS = 'abcdefghijklmnopqrstuvwxyz0123456789._~/=->'; const OPENCODE_SESSION_REFRESH_FAILURE_PATTERN = // eslint-disable-next-line sonarjs/regex-complexity -- Keyword taxonomy is kept literal to preserve diagnostic behavior. - /(?:^|[_\s:;.\/()-])(?:permission[_\s-]?denied|permission[_\s-]?blocked|access[_\s-]?denied|auth[_\s-]?unavailable|authentication[_\s-]?failed|unauthorized|forbidden|401|403|login[_\s-]?required|not\s+logged\s+in|missing\s+credentials?|invalid\s+credentials?|credentials?[_\s-]?required|credentials?[_\s-]?unavailable|no auth available|authorization|auth(?:entication)?(?:[_\s-]?(?:failed|unavailable))?|invalid api[_\s-]?key|api[_\s-]?key|does not have access|quota|rate[_\s-]?(?:limit|limited)|too many requests|429|model cooldown|cooling down|enospc|no space left|disk is full|capacity exceeded|quota exhausted|usage exceeded|free usage exceeded|key limit exceeded|total limit|insufficient credits|subscribe to go|error|failed|failure|timeout|timed\s+out|network|connection|unable\s+to\s+connect|connect\s+failed|econn[a-z_]*|enotfound|fetch[_\s-]?failed|connection[_\s-]?(?:refused|reset)|aborted|cancel(?:ed|led)|interrupted|service[_\s-]?unavailable|temporarily\s+unavailable|overloaded|visible[_\s-]?reply(?:[_\s-][a-z0-9]+)*|task[_\s-]?refs|relayofmessageid|relay[_\s-]?of[_\s-]?message[_\s-]?id|message[_\s-]?send|non[_\s-]?visible[_\s-]?tool(?:[_\s-][a-z0-9]+)*|protocol[_\s-]?proof)(?=$|[_\s:;.\/(),-])/i; + /(?:^|[_\s:;./()-])(?:permission[_\s-]?denied|permission[_\s-]?blocked|access[_\s-]?denied|auth[_\s-]?unavailable|authentication[_\s-]?failed|unauthorized|forbidden|401|403|login[_\s-]?required|not\s+logged\s+in|missing\s+credentials?|invalid\s+credentials?|credentials?[_\s-]?required|credentials?[_\s-]?unavailable|no auth available|authorization|auth(?:entication)?(?:[_\s-]?(?:failed|unavailable))?|invalid api[_\s-]?key|api[_\s-]?key|does not have access|quota|rate[_\s-]?(?:limit|limited)|too many requests|429|model cooldown|cooling down|enospc|no space left|disk is full|capacity exceeded|quota exhausted|usage exceeded|free usage exceeded|key limit exceeded|total limit|insufficient credits|subscribe to go|error|failed|failure|timeout|timed\s+out|network|connection|unable\s+to\s+connect|connect\s+failed|econn[a-z_]*|enotfound|fetch[_\s-]?failed|connection[_\s-]?(?:refused|reset)|aborted|cancel(?:ed|led)|interrupted|service[_\s-]?unavailable|temporarily\s+unavailable|overloaded|visible[_\s-]?reply(?:[_\s-][a-z0-9]+)*|task[_\s-]?refs|relayofmessageid|relay[_\s-]?of[_\s-]?message[_\s-]?id|message[_\s-]?send|non[_\s-]?visible[_\s-]?tool(?:[_\s-][a-z0-9]+)*|protocol[_\s-]?proof)(?=$|[_\s:;./(),-])/i; const OPENCODE_SESSION_REFRESH_SAFE_MARKER_STATE_PATTERN = /\b(?:not_observed|pending|prompt_not_indexed|responded_tool_call|responded_visible_message|responded_non_visible_tool|responded_plain_text|permission_blocked|tool_error|empty_assistant_turn|prompt_delivered_no_assistant_message|session_stale|session_error|reconcile_failed)\b/g; @@ -527,9 +542,48 @@ export function buildMemberLaunchDiagnosticsPayload(params: { const providerBackendId = runtimeEntry?.providerBackendId ?? params.member?.providerBackendId; const laneId = runtimeEntry?.laneId ?? params.member?.laneId; const laneKind = runtimeEntry?.laneKind ?? params.member?.laneKind; - const livenessKind = spawnEntry?.livenessKind ?? runtimeEntry?.livenessKind; - const launchState = spawnEntry?.launchState ?? params.launchState; - const spawnStatus = spawnEntry?.status ?? params.spawnStatus; + const livenessKind = hasStoppedRuntimeLivenessKind(runtimeEntry?.livenessKind) + ? runtimeEntry?.livenessKind + : (spawnEntry?.livenessKind ?? runtimeEntry?.livenessKind); + const bootstrapConfirmedProvisionedButNotAlive = + isBootstrapConfirmedProvisionedButNotAliveFailure(spawnEntry); + const hasUnsafeSpawnProvisionedButNotAliveEvidence = + bootstrapConfirmedProvisionedButNotAlive && + hasUnsafeProvisionedButNotAliveRuntimeEvidence(spawnEntry); + const hasUnsafeRuntimeProvisionedButNotAliveEvidence = + bootstrapConfirmedProvisionedButNotAlive && + !hasUnsafeSpawnProvisionedButNotAliveEvidence && + hasUnsafeProvisionedButNotAliveRuntimeEvidenceWithSpawnContext(spawnEntry, runtimeEntry); + const hasUnsafeProvisionedButNotAliveEvidence = + bootstrapConfirmedProvisionedButNotAlive && + (hasUnsafeSpawnProvisionedButNotAliveEvidence || + hasUnsafeRuntimeProvisionedButNotAliveEvidence); + const useBootstrapConfirmedVisualState = + bootstrapConfirmedProvisionedButNotAlive && + spawnEntry?.runtimeDiagnosticSeverity !== 'error' && + runtimeEntry?.runtimeDiagnosticSeverity !== 'error' && + !hasUnsafeProvisionedButNotAliveEvidence; + const useBootstrapConfirmedRuntimeAlive = + useBootstrapConfirmedVisualState && + runtimeEntry?.runtimeDiagnosticSeverity !== 'error' && + spawnEntry?.runtimeDiagnosticSeverity !== 'error'; + const runtimeEntryDiagnostic = boundedString(runtimeEntry?.runtimeDiagnostic); + const hasRuntimeDiagnosticEvidence = + runtimeEntryDiagnostic != null || runtimeEntry?.runtimeDiagnosticSeverity != null; + const useSpawnDiagnosticsForHealedEntry = + bootstrapConfirmedProvisionedButNotAlive && !hasRuntimeDiagnosticEvidence; + const keepSpawnFailureDiagnostics = + useSpawnDiagnosticsForHealedEntry || + hasUnsafeSpawnProvisionedButNotAliveEvidence || + spawnEntry?.runtimeDiagnosticSeverity === 'error'; + const launchState = useBootstrapConfirmedVisualState + ? 'confirmed_alive' + : (spawnEntry?.launchState ?? params.launchState); + const spawnStatus = useBootstrapConfirmedVisualState + ? 'online' + : (spawnEntry?.status ?? params.spawnStatus); + const spawnRuntimeAlive = useBootstrapConfirmedRuntimeAlive ? true : spawnEntry?.runtimeAlive; + const spawnHardFailure = useBootstrapConfirmedVisualState ? false : spawnEntry?.hardFailure; const runtimeAdvisoryTitle = boundedString(params.runtimeAdvisoryTitle); const runtimeAdvisoryLabel = boundedString(params.runtimeAdvisoryLabel ?? undefined); const runtimeAdvisoryMessage = boundedString(runtimeAdvisory?.message); @@ -541,10 +595,10 @@ export function buildMemberLaunchDiagnosticsPayload(params: { runtimeAdvisoryMessage, spawnStatus, launchState, - runtimeAlive: spawnEntry?.runtimeAlive, + runtimeAlive: spawnRuntimeAlive, bootstrapConfirmed: spawnEntry?.bootstrapConfirmed, agentToolAccepted: spawnEntry?.agentToolAccepted, - hardFailure: spawnEntry?.hardFailure, + hardFailure: spawnHardFailure, livenessKind, runtimeEntry, }); @@ -553,9 +607,17 @@ export function buildMemberLaunchDiagnosticsPayload(params: { ? (runtimeAdvisoryTitle ?? runtimeAdvisoryLabel ?? runtimeAdvisoryMessage) : undefined; const runtimeDiagnosticSeverity = - spawnEntry?.runtimeDiagnosticSeverity ?? runtimeEntry?.runtimeDiagnosticSeverity; + spawnEntry?.runtimeDiagnosticSeverity === 'error' + ? spawnEntry.runtimeDiagnosticSeverity + : bootstrapConfirmedProvisionedButNotAlive + ? (runtimeEntry?.runtimeDiagnosticSeverity ?? + (useSpawnDiagnosticsForHealedEntry ? spawnEntry?.runtimeDiagnosticSeverity : undefined)) + : (spawnEntry?.runtimeDiagnosticSeverity ?? runtimeEntry?.runtimeDiagnosticSeverity); const spawnRuntimeDiagnosticCardError = isRuntimeDiagnosticCardError({ - runtimeDiagnostic: spawnEntry?.runtimeDiagnostic, + runtimeDiagnostic: + bootstrapConfirmedProvisionedButNotAlive && !keepSpawnFailureDiagnostics + ? undefined + : spawnEntry?.runtimeDiagnostic, runtimeDiagnosticSeverity: spawnEntry?.runtimeDiagnosticSeverity, launchState: spawnEntry?.launchState, spawnStatus: spawnEntry?.status, @@ -564,6 +626,10 @@ export function buildMemberLaunchDiagnosticsPayload(params: { }) ? spawnEntry?.runtimeDiagnostic : undefined; + const healedSpawnFailureCardError = + keepSpawnFailureDiagnostics && spawnEntry?.runtimeDiagnosticSeverity === 'error' + ? (spawnRuntimeDiagnosticCardError ?? spawnEntry?.error ?? spawnEntry?.hardFailureReason) + : undefined; const runtimeEntryDiagnosticCardError = isRuntimeDiagnosticCardError({ runtimeDiagnostic: runtimeEntry?.runtimeDiagnostic, runtimeDiagnosticSeverity: runtimeEntry?.runtimeDiagnosticSeverity, @@ -572,19 +638,24 @@ export function buildMemberLaunchDiagnosticsPayload(params: { ? runtimeEntry?.runtimeDiagnostic : undefined; const runtimeDiagnostic = - boundedString(spawnEntry?.runtimeDiagnostic) ?? - boundedString(runtimeEntry?.runtimeDiagnostic) ?? - boundedString(spawnEntry?.hardFailureReason) ?? - boundedString(spawnEntry?.error) ?? + (bootstrapConfirmedProvisionedButNotAlive && !keepSpawnFailureDiagnostics + ? undefined + : boundedString(spawnEntry?.runtimeDiagnostic)) ?? + runtimeEntryDiagnostic ?? + (bootstrapConfirmedProvisionedButNotAlive && !keepSpawnFailureDiagnostics + ? undefined + : (boundedString(spawnEntry?.hardFailureReason) ?? boundedString(spawnEntry?.error))) ?? runtimeAdvisoryMessage; const memberCardError = firstMemberCardFailureReason({ - candidates: [ - spawnEntry?.error, - spawnEntry?.hardFailureReason, - spawnRuntimeDiagnosticCardError, - runtimeEntryDiagnosticCardError, - runtimeAdvisoryCardError, - ], + candidates: bootstrapConfirmedProvisionedButNotAlive + ? [healedSpawnFailureCardError, runtimeEntryDiagnosticCardError, runtimeAdvisoryCardError] + : [ + spawnEntry?.error, + spawnEntry?.hardFailureReason, + spawnRuntimeDiagnosticCardError, + runtimeEntryDiagnosticCardError, + runtimeAdvisoryCardError, + ], evidence: [ spawnEntry?.runtimeDiagnostic, runtimeEntry?.runtimeDiagnostic, @@ -601,8 +672,13 @@ export function buildMemberLaunchDiagnosticsPayload(params: { runtimeAdvisoryTitle ? [runtimeAdvisoryTitle] : undefined, runtimeAdvisoryLabel ? [runtimeAdvisoryLabel] : undefined, runtimeAdvisoryMessage ? [runtimeAdvisoryMessage] : undefined, - spawnEntry?.hardFailureReason ? [spawnEntry.hardFailureReason] : undefined, - spawnEntry?.error ? [spawnEntry.error] : undefined, + (!bootstrapConfirmedProvisionedButNotAlive || keepSpawnFailureDiagnostics) && + spawnEntry?.hardFailureReason + ? [spawnEntry.hardFailureReason] + : undefined, + (!bootstrapConfirmedProvisionedButNotAlive || keepSpawnFailureDiagnostics) && spawnEntry?.error + ? [spawnEntry.error] + : undefined, runtimeEntry?.diagnostics ); const runId = boundedString(params.runId ?? undefined); @@ -648,18 +724,14 @@ export function buildMemberLaunchDiagnosticsPayload(params: { ...(typeof runtimeEntry?.restartable === 'boolean' ? { restartable: runtimeEntry.restartable } : {}), - ...(typeof spawnEntry?.runtimeAlive === 'boolean' - ? { runtimeAlive: spawnEntry.runtimeAlive } - : {}), + ...(typeof spawnRuntimeAlive === 'boolean' ? { runtimeAlive: spawnRuntimeAlive } : {}), ...(typeof spawnEntry?.bootstrapConfirmed === 'boolean' ? { bootstrapConfirmed: spawnEntry.bootstrapConfirmed } : {}), ...(typeof spawnEntry?.agentToolAccepted === 'boolean' ? { agentToolAccepted: spawnEntry.agentToolAccepted } : {}), - ...(typeof spawnEntry?.hardFailure === 'boolean' - ? { hardFailure: spawnEntry.hardFailure } - : {}), + ...(typeof spawnHardFailure === 'boolean' ? { hardFailure: spawnHardFailure } : {}), ...(livenessKind ? { livenessKind } : {}), ...((spawnEntry?.livenessSource ?? params.livenessSource) ? { livenessSource: spawnEntry?.livenessSource ?? params.livenessSource } @@ -751,6 +823,9 @@ function parseStatusUpdatedAtMs(value: string | undefined): number | null { } function isFailedSpawnEntry(entry: MemberSpawnStatusEntry | undefined): boolean { + if (isBootstrapConfirmedProvisionedButNotAliveFailure(entry)) { + return hasUnsafeProvisionedButNotAliveRuntimeEvidence(entry); + } return entry?.launchState === 'failed_to_start' || entry?.status === 'error'; } diff --git a/src/renderer/utils/teamProvisioningPresentation.ts b/src/renderer/utils/teamProvisioningPresentation.ts index f246a707..94169385 100644 --- a/src/renderer/utils/teamProvisioningPresentation.ts +++ b/src/renderer/utils/teamProvisioningPresentation.ts @@ -5,6 +5,10 @@ import { getLaunchJoinState, } from '@renderer/components/team/provisioningSteps'; import { isLeadMember } from '@shared/utils/leadDetection'; +import { + hasUnsafeProvisionedButNotAliveRuntimeEvidence, + isBootstrapConfirmedProvisionedButNotAliveFailure, +} from '@shared/utils/teamLaunchFailureReason'; import type { MemberSpawnStatusEntry, @@ -85,9 +89,19 @@ function parseStatusUpdatedAtMs(value: string | undefined): number | null { } function isFailedSpawnEntry(entry: MemberSpawnStatusEntry | undefined): boolean { + if (isBootstrapConfirmedProvisionedButNotAliveFailure(entry)) { + return hasUnsafeProvisionedButNotAliveRuntimeEvidence(entry); + } return entry?.launchState === 'failed_to_start' || entry?.status === 'error'; } +function isConfirmedSpawnEntry(entry: MemberSpawnStatusEntry | undefined): boolean { + if (isBootstrapConfirmedProvisionedButNotAliveFailure(entry)) { + return !isFailedSpawnEntry(entry); + } + return entry?.launchState === 'confirmed_alive' || entry?.bootstrapConfirmed === true; +} + function isSkippedSpawnEntry(entry: MemberSpawnStatusEntry | undefined): boolean { return entry?.launchState === 'skipped_for_launch' || entry?.skippedForLaunch === true; } @@ -125,7 +139,7 @@ function isOpenCodeSecondaryRetryCandidate(params: { ) { return false; } - return entry.launchState === 'failed_to_start' || entry.status === 'error'; + return isFailedSpawnEntry(entry); } function shouldPreferSnapshotEntryOverLive(params: { @@ -275,7 +289,7 @@ function getPendingDiagnosticNameGroups(params: { }); if ( !entry || - entry.launchState === 'confirmed_alive' || + isConfirmedSpawnEntry(entry) || isFailedSpawnEntry(entry) || isSkippedSpawnEntry(entry) ) { @@ -328,7 +342,7 @@ function getPendingSpawnNames(params: { }); return ( entry != null && - entry.launchState !== 'confirmed_alive' && + !isConfirmedSpawnEntry(entry) && !isFailedSpawnEntry(entry) && !isSkippedSpawnEntry(entry) ); @@ -611,9 +625,7 @@ function getFailedSpawnDetails(params: { }), ] as const; }) - .filter( - ([, entry]) => entry && (entry.launchState === 'failed_to_start' || entry.status === 'error') - ) + .filter(([, entry]) => isFailedSpawnEntry(entry)) .map(([name, entry]) => ({ name, reason: diff --git a/src/shared/utils/teamLaunchFailureReason.ts b/src/shared/utils/teamLaunchFailureReason.ts new file mode 100644 index 00000000..fdac36f5 --- /dev/null +++ b/src/shared/utils/teamLaunchFailureReason.ts @@ -0,0 +1,142 @@ +import type { + MemberLaunchState, + MemberSpawnStatus, + TeamAgentRuntimeDiagnosticSeverity, + TeamAgentRuntimeLivenessKind, +} from '@shared/types'; + +export interface ProvisionedButNotAliveLaunchEntry { + launchState?: MemberLaunchState; + status?: MemberSpawnStatus; + hardFailure?: boolean; + hardFailureReason?: string; + error?: string; + runtimeDiagnostic?: string; + runtimeDiagnosticSeverity?: TeamAgentRuntimeDiagnosticSeverity; + bootstrapConfirmed?: boolean; + livenessKind?: TeamAgentRuntimeLivenessKind; +} + +export function stripProcessTableUnavailableDiagnosticSuffix(reason: string): string | null { + const match = /^(.*?);\s*process table (?:is )?unavailable$/i.exec(reason.trim()); + const baseReason = match?.[1]?.trim(); + return baseReason && baseReason.length > 0 ? baseReason : null; +} + +export function isProvisionedButNotAliveFailureReason(reason?: string): boolean { + return isCliProvisionedButNotAliveFailureReason(reason); +} + +export function isCliProvisionedButNotAliveFailureReason(reason?: string): boolean { + const text = reason?.trim(); + if (!text) { + return false; + } + const normalizedText = stripProcessTableUnavailableDiagnosticSuffix(text) ?? text; + return /^CLI process exited \(code (?:unknown|-?\d+|\?)\)\s+[-\u2013\u2014]\s+team provisioned but not alive$/i.test( + normalizedText + ); +} + +export function mentionsProcessTableUnavailable(value: string | undefined): boolean { + return /\bprocess table\b.*\bunavailable\b/i.test(value ?? ''); +} + +export function hasBootstrapConfirmationProofForLaunchFailure( + entry: ProvisionedButNotAliveLaunchEntry | undefined +): boolean { + return ( + entry?.bootstrapConfirmed === true || + entry?.launchState === 'confirmed_alive' || + entry?.livenessKind === 'confirmed_bootstrap' + ); +} + +export function isProvisionedButNotAliveLaunchFailure( + entry: ProvisionedButNotAliveLaunchEntry | undefined +): boolean { + if (!entry) { + return false; + } + const hardFailureReason = entry.hardFailureReason?.trim(); + const failureReasonMatches = hardFailureReason + ? isProvisionedButNotAliveFailureReason(hardFailureReason) + : isProvisionedButNotAliveFailureReason(entry.error ?? entry.runtimeDiagnostic); + if (!failureReasonMatches) { + return false; + } + return ( + entry.launchState === 'failed_to_start' || + entry.status === 'error' || + entry.hardFailure === true + ); +} + +export function isBootstrapConfirmedProvisionedButNotAliveFailure( + entry: ProvisionedButNotAliveLaunchEntry | undefined +): boolean { + return ( + isProvisionedButNotAliveLaunchFailure(entry) && + hasBootstrapConfirmationProofForLaunchFailure(entry) + ); +} + +export function hasUnsafeProvisionedButNotAliveRuntimeEvidence( + entry: ProvisionedButNotAliveLaunchEntry | undefined +): boolean { + if (!entry) { + return false; + } + if (entry.runtimeDiagnosticSeverity === 'error') { + return true; + } + if ( + entry.livenessKind === 'not_found' || + entry.livenessKind === 'shell_only' || + entry.livenessKind === 'permission_blocked' || + entry.livenessKind === 'runtime_process_candidate' + ) { + return true; + } + const hasProcessTableUnavailableMarker = + mentionsProcessTableUnavailable(entry.runtimeDiagnostic) || + mentionsProcessTableUnavailable(entry.hardFailureReason) || + mentionsProcessTableUnavailable(entry.error); + if (!entry.livenessKind) { + return !hasProcessTableUnavailableMarker; + } + if (entry.livenessKind !== 'registered_only' && entry.livenessKind !== 'stale_metadata') { + return false; + } + return !hasProcessTableUnavailableMarker; +} + +export function hasUnsafeProvisionedButNotAliveRuntimeEvidenceWithSpawnContext( + spawnEntry: ProvisionedButNotAliveLaunchEntry | undefined, + runtimeEntry: ProvisionedButNotAliveLaunchEntry | undefined +): boolean { + if (hasUnsafeProvisionedButNotAliveRuntimeEvidence(spawnEntry)) { + return true; + } + if (!runtimeEntry) { + return false; + } + + const runtimeDiagnostic = runtimeEntry.runtimeDiagnostic?.trim(); + if ( + !runtimeDiagnostic && + (runtimeEntry.livenessKind == null || + runtimeEntry.livenessKind === 'registered_only' || + runtimeEntry.livenessKind === 'stale_metadata') + ) { + return hasUnsafeProvisionedButNotAliveRuntimeEvidence({ + runtimeDiagnostic: spawnEntry?.runtimeDiagnostic, + hardFailureReason: spawnEntry?.hardFailureReason, + error: spawnEntry?.error, + runtimeDiagnosticSeverity: runtimeEntry.runtimeDiagnosticSeverity, + livenessKind: runtimeEntry.livenessKind, + }); + } + + return hasUnsafeProvisionedButNotAliveRuntimeEvidence(runtimeEntry); +} diff --git a/test/main/services/team/TeamConfigReader.test.ts b/test/main/services/team/TeamConfigReader.test.ts index 31fb3fca..86d598b0 100644 --- a/test/main/services/team/TeamConfigReader.test.ts +++ b/test/main/services/team/TeamConfigReader.test.ts @@ -151,6 +151,74 @@ describe('TeamConfigReader', () => { }); }); + it('projects bootstrap-confirmed provisioned-but-not-alive launch state as settled', async () => { + const teamName = 'signal-ops'; + const teamDir = path.join(tempDir, teamName); + await fs.mkdir(teamDir, { recursive: true }); + await fs.writeFile( + path.join(teamDir, 'config.json'), + JSON.stringify({ + name: 'Signal Ops', + members: [{ name: 'team-lead', agentType: 'team-lead' }], + }), + 'utf8' + ); + await fs.writeFile( + path.join(teamDir, 'launch-state.json'), + JSON.stringify({ + version: 2, + teamName, + updatedAt: '2026-05-25T20:14:02.147Z', + launchPhase: 'finished', + expectedMembers: ['tom'], + members: { + tom: { + name: 'tom', + providerId: 'anthropic', + launchState: 'failed_to_start', + agentToolAccepted: true, + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: 'CLI process exited (code 1) \u2014 team provisioned but not alive', + livenessKind: 'confirmed_bootstrap', + runtimeDiagnostic: + 'runtime pid could not be verified because process table is unavailable', + runtimeDiagnosticSeverity: 'warning', + firstSpawnAcceptedAt: '2026-05-25T20:13:46.326Z', + lastHeartbeatAt: '2026-05-25T20:13:56.110Z', + lastEvaluatedAt: '2026-05-25T20:14:02.147Z', + }, + }, + summary: { + confirmedCount: 0, + pendingCount: 0, + failedCount: 1, + runtimeAlivePendingCount: 0, + }, + teamLaunchState: 'partial_failure', + }), + 'utf8' + ); + + const reader = new TeamConfigReader(); + const teams = await reader.listTeams(); + + expect(teams).toHaveLength(1); + expect(teams[0]).toMatchObject({ + teamName, + displayName: 'Signal Ops', + teamLaunchState: 'clean_success', + confirmedMemberCount: 1, + confirmedCount: 1, + failedCount: 0, + }); + expect(teams[0]).not.toMatchObject({ + partialLaunchFailure: true, + missingMembers: ['tom'], + }); + }); + it('does not invent a partial-failure summary from artifact counts for mixed-aware teams when canonical launch truth is unavailable', async () => { const teamName = 'mixed-aware-team'; const teamDir = path.join(tempDir, teamName); @@ -578,16 +646,19 @@ describe('TeamConfigReader', () => { 'utf8' ); let ctimeMs = 1000; - vi.spyOn(nodeFs.promises, 'stat').mockImplementation(async () => ({ - size: BigInt(4096), - mode: BigInt(33188), - dev: BigInt(1), - ino: BigInt(2), - mtimeMs: 1000, - ctimeMs, - birthtimeMs: 1000, - isFile: () => true, - }) as never); + vi.spyOn(nodeFs.promises, 'stat').mockImplementation( + async () => + ({ + size: BigInt(4096), + mode: BigInt(33188), + dev: BigInt(1), + ino: BigInt(2), + mtimeMs: 1000, + ctimeMs, + birthtimeMs: 1000, + isFile: () => true, + }) as never + ); const readFileSpy = vi.spyOn(nodeFs.promises, 'readFile'); const reader = new TeamConfigReader(); @@ -682,15 +753,16 @@ describe('TeamConfigReader', () => { const readDeferred = createDeferred(); const realReadFile = nodeFs.promises.readFile.bind(nodeFs.promises); let intercepted = false; - vi.spyOn(nodeFs.promises, 'readFile').mockImplementation( - ((file: unknown, ...args: unknown[]) => { - if (!intercepted && String(file) === configPath) { - intercepted = true; - return readDeferred.promise as never; - } - return realReadFile(file as never, ...(args as never[])) as never; - }) as never - ); + vi.spyOn(nodeFs.promises, 'readFile').mockImplementation((( + file: unknown, + ...args: unknown[] + ) => { + if (!intercepted && String(file) === configPath) { + intercepted = true; + return readDeferred.promise as never; + } + return realReadFile(file as never, ...(args as never[])) as never; + }) as never); const reader = new TeamConfigReader(); const staleSnapshot = reader.getConfigSnapshot(teamName); @@ -730,15 +802,16 @@ describe('TeamConfigReader', () => { const readDeferred = createDeferred(); const realReadFile = nodeFs.promises.readFile.bind(nodeFs.promises); let intercepted = false; - vi.spyOn(nodeFs.promises, 'readFile').mockImplementation( - ((file: unknown, ...args: unknown[]) => { - if (!intercepted && String(file) === configPath) { - intercepted = true; - return readDeferred.promise as never; - } - return realReadFile(file as never, ...(args as never[])) as never; - }) as never - ); + vi.spyOn(nodeFs.promises, 'readFile').mockImplementation((( + file: unknown, + ...args: unknown[] + ) => { + if (!intercepted && String(file) === configPath) { + intercepted = true; + return readDeferred.promise as never; + } + return realReadFile(file as never, ...(args as never[])) as never; + }) as never); const reader = new TeamConfigReader(); const staleVerified = reader.getConfig(teamName); @@ -781,15 +854,16 @@ describe('TeamConfigReader', () => { const readDeferred = createDeferred(); const realReadFile = nodeFs.promises.readFile.bind(nodeFs.promises); let intercepted = false; - vi.spyOn(nodeFs.promises, 'readFile').mockImplementation( - ((file: unknown, ...args: unknown[]) => { - if (!intercepted && String(file) === configPath) { - intercepted = true; - return readDeferred.promise as never; - } - return realReadFile(file as never, ...(args as never[])) as never; - }) as never - ); + vi.spyOn(nodeFs.promises, 'readFile').mockImplementation((( + file: unknown, + ...args: unknown[] + ) => { + if (!intercepted && String(file) === configPath) { + intercepted = true; + return readDeferred.promise as never; + } + return realReadFile(file as never, ...(args as never[])) as never; + }) as never); const reader = new TeamConfigReader(); const staleSnapshot = reader.getConfigSnapshot(teamName); diff --git a/test/main/services/team/TeamLaunchSummaryProjection.test.ts b/test/main/services/team/TeamLaunchSummaryProjection.test.ts index b9319eb5..d84fbbaf 100644 --- a/test/main/services/team/TeamLaunchSummaryProjection.test.ts +++ b/test/main/services/team/TeamLaunchSummaryProjection.test.ts @@ -235,6 +235,428 @@ describe('TeamLaunchSummaryProjection', () => { }); }); + it('projects provisioned-but-not-alive failures with bootstrap proof as confirmed', () => { + const summary = choosePreferredLaunchStateSummary({ + launchSnapshot: { + version: 2, + teamName: 'signal-ops', + updatedAt: '2026-05-25T20:14:02.147Z', + launchPhase: 'finished', + expectedMembers: ['tom'], + members: { + tom: { + name: 'tom', + providerId: 'anthropic', + launchState: 'failed_to_start', + agentToolAccepted: true, + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: 'CLI process exited (code 1) \u2014 team provisioned but not alive', + livenessKind: 'confirmed_bootstrap', + runtimeDiagnostic: + 'runtime pid could not be verified because process table is unavailable', + runtimeDiagnosticSeverity: 'warning', + firstSpawnAcceptedAt: '2026-05-25T20:13:46.326Z', + lastHeartbeatAt: '2026-05-25T20:13:56.110Z', + lastEvaluatedAt: '2026-05-25T20:14:02.147Z', + }, + }, + summary: { + confirmedCount: 0, + pendingCount: 0, + failedCount: 1, + runtimeAlivePendingCount: 0, + }, + teamLaunchState: 'partial_failure', + } as never, + }); + + expect(summary).toMatchObject({ + teamLaunchState: 'clean_success', + confirmedMemberCount: 1, + confirmedCount: 1, + pendingCount: 0, + failedCount: 0, + }); + expect(summary).not.toMatchObject({ + partialLaunchFailure: true, + missingMembers: ['tom'], + }); + }); + + it('projects Windows process-table-unavailable provisioned-but-not-alive metadata as confirmed', () => { + const summary = choosePreferredLaunchStateSummary({ + launchSnapshot: { + version: 2, + teamName: 'signal-ops', + updatedAt: '2026-05-25T20:14:02.147Z', + launchPhase: 'finished', + expectedMembers: ['tom'], + members: { + tom: { + name: 'tom', + providerId: 'anthropic', + launchState: 'failed_to_start', + agentToolAccepted: true, + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: + 'CLI process exited (code 1) - team provisioned but not alive; process table unavailable', + livenessKind: 'registered_only', + runtimeDiagnostic: + 'runtime pid could not be verified because process table is unavailable', + runtimeDiagnosticSeverity: 'warning', + firstSpawnAcceptedAt: '2026-05-25T20:13:46.326Z', + lastHeartbeatAt: '2026-05-25T20:13:56.110Z', + lastEvaluatedAt: '2026-05-25T20:14:02.147Z', + }, + }, + summary: { + confirmedCount: 0, + pendingCount: 0, + failedCount: 1, + runtimeAlivePendingCount: 0, + }, + teamLaunchState: 'partial_failure', + } as never, + }); + + expect(summary).toMatchObject({ + teamLaunchState: 'clean_success', + confirmedMemberCount: 1, + confirmedCount: 1, + failedCount: 0, + }); + }); + + it('keeps provisioned-but-not-alive failures with runtime error evidence as failed', () => { + const summary = choosePreferredLaunchStateSummary({ + launchSnapshot: { + version: 2, + teamName: 'signal-ops', + updatedAt: '2026-05-25T20:14:02.147Z', + launchPhase: 'finished', + expectedMembers: ['tom'], + members: { + tom: { + name: 'tom', + providerId: 'anthropic', + launchState: 'failed_to_start', + agentToolAccepted: true, + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: 'CLI process exited (code 1) - team provisioned but not alive', + livenessKind: 'confirmed_bootstrap', + runtimeDiagnostic: 'Runtime process crashed', + runtimeDiagnosticSeverity: 'error', + lastEvaluatedAt: '2026-05-25T20:14:02.147Z', + }, + }, + summary: { + confirmedCount: 0, + pendingCount: 0, + failedCount: 1, + runtimeAlivePendingCount: 0, + }, + teamLaunchState: 'partial_failure', + } as never, + }); + + expect(summary).toMatchObject({ + partialLaunchFailure: true, + missingMembers: ['tom'], + teamLaunchState: 'partial_failure', + confirmedCount: 0, + failedCount: 1, + }); + }); + + it('reconciles unhealed launch-summary projections with bootstrap proof', () => { + const summary = choosePreferredLaunchStateSummary({ + bootstrapSnapshot: { + version: 2, + teamName: 'signal-ops', + updatedAt: '2026-05-25T20:13:56.110Z', + launchPhase: 'finished', + expectedMembers: ['tom'], + members: { + tom: { + name: 'tom', + providerId: 'anthropic', + launchState: 'confirmed_alive', + agentToolAccepted: true, + runtimeAlive: true, + bootstrapConfirmed: true, + hardFailure: false, + livenessKind: 'confirmed_bootstrap', + firstSpawnAcceptedAt: '2026-05-25T20:13:46.326Z', + lastHeartbeatAt: '2026-05-25T20:13:56.110Z', + lastEvaluatedAt: '2026-05-25T20:13:56.110Z', + }, + }, + summary: { + confirmedCount: 1, + pendingCount: 0, + failedCount: 0, + runtimeAlivePendingCount: 0, + }, + teamLaunchState: 'clean_success', + } as never, + launchSummaryProjection: { + version: 1, + teamName: 'signal-ops', + updatedAt: '2026-05-25T20:14:02.147Z', + launchUpdatedAt: '2026-05-25T20:14:02.147Z', + teamLaunchState: 'partial_failure', + partialLaunchFailure: true, + expectedMemberCount: 1, + confirmedMemberCount: 0, + missingMembers: ['tom'], + confirmedCount: 0, + pendingCount: 0, + failedCount: 1, + runtimeAlivePendingCount: 0, + }, + }); + + expect(summary).toMatchObject({ + teamLaunchState: 'clean_success', + confirmedMemberCount: 1, + confirmedCount: 1, + failedCount: 0, + pendingCount: 0, + }); + expect(summary).not.toMatchObject({ + partialLaunchFailure: true, + missingMembers: ['tom'], + }); + }); + + it('does not reconcile launch-summary projections from stale bootstrap proof', () => { + const summary = choosePreferredLaunchStateSummary({ + bootstrapSnapshot: { + version: 2, + teamName: 'signal-ops', + updatedAt: '2026-05-25T20:10:10.000Z', + launchPhase: 'finished', + expectedMembers: ['tom'], + members: { + tom: { + name: 'tom', + providerId: 'anthropic', + launchState: 'confirmed_alive', + agentToolAccepted: true, + runtimeAlive: true, + bootstrapConfirmed: true, + hardFailure: false, + livenessKind: 'confirmed_bootstrap', + firstSpawnAcceptedAt: '2026-05-25T20:10:00.000Z', + lastHeartbeatAt: '2026-05-25T20:10:05.000Z', + lastEvaluatedAt: '2026-05-25T20:10:10.000Z', + }, + }, + summary: { + confirmedCount: 1, + pendingCount: 0, + failedCount: 0, + runtimeAlivePendingCount: 0, + }, + teamLaunchState: 'clean_success', + } as never, + launchSummaryProjection: { + version: 1, + teamName: 'signal-ops', + updatedAt: '2026-05-25T20:14:02.147Z', + launchUpdatedAt: '2026-05-25T20:14:02.147Z', + teamLaunchState: 'partial_failure', + partialLaunchFailure: true, + expectedMemberCount: 1, + confirmedMemberCount: 0, + missingMembers: ['tom'], + confirmedCount: 0, + pendingCount: 0, + failedCount: 1, + runtimeAlivePendingCount: 0, + }, + }); + + expect(summary).toMatchObject({ + partialLaunchFailure: true, + missingMembers: ['tom'], + teamLaunchState: 'partial_failure', + confirmedCount: 0, + failedCount: 1, + }); + }); + + it('does not reconcile launch-summary projections from stopped bootstrap proof', () => { + const summary = choosePreferredLaunchStateSummary({ + bootstrapSnapshot: { + version: 2, + teamName: 'signal-ops', + updatedAt: '2026-05-25T20:13:56.110Z', + launchPhase: 'finished', + expectedMembers: ['tom'], + members: { + tom: { + name: 'tom', + providerId: 'anthropic', + launchState: 'confirmed_alive', + agentToolAccepted: true, + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: false, + livenessKind: 'not_found', + firstSpawnAcceptedAt: '2026-05-25T20:13:46.326Z', + lastEvaluatedAt: '2026-05-25T20:13:56.110Z', + }, + }, + summary: { + confirmedCount: 1, + pendingCount: 0, + failedCount: 0, + runtimeAlivePendingCount: 0, + }, + teamLaunchState: 'clean_success', + } as never, + launchSummaryProjection: { + version: 1, + teamName: 'signal-ops', + updatedAt: '2026-05-25T20:14:02.147Z', + launchUpdatedAt: '2026-05-25T20:14:02.147Z', + teamLaunchState: 'partial_failure', + partialLaunchFailure: true, + expectedMemberCount: 1, + confirmedMemberCount: 0, + missingMembers: ['tom'], + confirmedCount: 0, + pendingCount: 0, + failedCount: 1, + runtimeAlivePendingCount: 0, + }, + }); + + expect(summary).toMatchObject({ + partialLaunchFailure: true, + missingMembers: ['tom'], + teamLaunchState: 'partial_failure', + confirmedCount: 0, + failedCount: 1, + }); + }); + + it('keeps provisioned-but-not-alive failures without bootstrap proof as failed', () => { + const summary = choosePreferredLaunchStateSummary({ + launchSnapshot: { + version: 2, + teamName: 'signal-ops', + updatedAt: '2026-05-25T20:14:02.147Z', + launchPhase: 'finished', + expectedMembers: ['tom'], + members: { + tom: { + name: 'tom', + providerId: 'anthropic', + launchState: 'failed_to_start', + agentToolAccepted: true, + runtimeAlive: false, + bootstrapConfirmed: false, + hardFailure: true, + hardFailureReason: 'CLI process exited (code 1) \u2014 team provisioned but not alive', + lastEvaluatedAt: '2026-05-25T20:14:02.147Z', + }, + }, + summary: { + confirmedCount: 0, + pendingCount: 0, + failedCount: 1, + runtimeAlivePendingCount: 0, + }, + teamLaunchState: 'partial_failure', + } as never, + }); + + expect(summary).toMatchObject({ + partialLaunchFailure: true, + missingMembers: ['tom'], + teamLaunchState: 'partial_failure', + confirmedCount: 0, + failedCount: 1, + }); + }); + + it('does not project provisioned-but-not-alive from stale bootstrap proof before spawn acceptance', () => { + const summary = choosePreferredLaunchStateSummary({ + bootstrapSnapshot: { + version: 2, + teamName: 'signal-ops', + updatedAt: '2026-05-25T20:10:10.000Z', + launchPhase: 'finished', + expectedMembers: ['tom'], + members: { + tom: { + name: 'tom', + providerId: 'anthropic', + launchState: 'confirmed_alive', + agentToolAccepted: true, + runtimeAlive: true, + bootstrapConfirmed: true, + hardFailure: false, + firstSpawnAcceptedAt: '2026-05-25T20:10:00.000Z', + lastHeartbeatAt: '2026-05-25T20:10:05.000Z', + lastEvaluatedAt: '2026-05-25T20:10:10.000Z', + }, + }, + summary: { + confirmedCount: 1, + pendingCount: 0, + failedCount: 0, + runtimeAlivePendingCount: 0, + }, + teamLaunchState: 'clean_success', + } as never, + launchSnapshot: { + version: 2, + teamName: 'signal-ops', + updatedAt: '2026-05-25T20:14:02.147Z', + launchPhase: 'finished', + expectedMembers: ['tom'], + members: { + tom: { + name: 'tom', + providerId: 'anthropic', + launchState: 'failed_to_start', + agentToolAccepted: true, + runtimeAlive: false, + bootstrapConfirmed: false, + hardFailure: true, + hardFailureReason: 'CLI process exited (code 1) \u2014 team provisioned but not alive', + firstSpawnAcceptedAt: '2026-05-25T20:13:46.326Z', + lastEvaluatedAt: '2026-05-25T20:14:02.147Z', + }, + }, + summary: { + confirmedCount: 0, + pendingCount: 0, + failedCount: 1, + runtimeAlivePendingCount: 0, + }, + teamLaunchState: 'partial_failure', + } as never, + }); + + expect(summary).toMatchObject({ + partialLaunchFailure: true, + missingMembers: ['tom'], + teamLaunchState: 'partial_failure', + confirmedCount: 0, + failedCount: 1, + }); + }); + it('prefers a mixed-aware persisted summary projection over a newer but poorer bootstrap snapshot', () => { const bootstrapSnapshot = { version: 2, diff --git a/test/main/services/team/TeamProvisioningLaunchDiagnostics.test.ts b/test/main/services/team/TeamProvisioningLaunchDiagnostics.test.ts index d05f9957..bfb19a79 100644 --- a/test/main/services/team/TeamProvisioningLaunchDiagnostics.test.ts +++ b/test/main/services/team/TeamProvisioningLaunchDiagnostics.test.ts @@ -21,7 +21,7 @@ function spawnEntry(overrides: Partial): MemberSpawnStat }; } -function buildRun(entries: Array<[string, Partial]>, isLaunch = true) { +function buildRun(entries: [string, Partial][], isLaunch = true) { return { isLaunch, memberSpawnStatuses: new Map( @@ -215,6 +215,139 @@ describe('TeamProvisioningLaunchDiagnostics', () => { ]); }); + it('classifies bootstrap-confirmed provisioned-but-not-alive entries as confirmed', () => { + const diagnostics = buildLaunchDiagnosticsFromRun( + buildRun([ + [ + 'tom', + { + status: 'error', + launchState: 'failed_to_start', + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: 'CLI process exited (code 1) \u2014 team provisioned but not alive', + livenessKind: 'confirmed_bootstrap', + runtimeDiagnostic: + 'runtime pid could not be verified because process table is unavailable', + runtimeDiagnosticSeverity: 'warning', + }, + ], + ]), + { nowIso } + ); + + expect(diagnostics).toEqual([ + { + id: 'tom:bootstrap_confirmed', + memberName: 'tom', + severity: 'info', + code: 'bootstrap_confirmed', + label: 'tom - bootstrap confirmed', + observedAt: NOW, + }, + ]); + }); + + it('classifies process-table-unavailable registered metadata as confirmed', () => { + const diagnostics = buildLaunchDiagnosticsFromRun( + buildRun([ + [ + 'tom', + { + status: 'error', + launchState: 'failed_to_start', + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: + 'CLI process exited (code 1) - team provisioned but not alive; process table unavailable', + livenessKind: 'registered_only', + runtimeDiagnostic: + 'runtime pid could not be verified because process table is unavailable', + runtimeDiagnosticSeverity: 'warning', + }, + ], + ]), + { nowIso } + ); + + expect(diagnostics).toEqual([ + { + id: 'tom:bootstrap_confirmed', + memberName: 'tom', + severity: 'info', + code: 'bootstrap_confirmed', + label: 'tom - bootstrap confirmed', + observedAt: NOW, + }, + ]); + }); + + it('keeps error diagnostics for bootstrap-confirmed provisioned-but-not-alive entries', () => { + const diagnostics = buildLaunchDiagnosticsFromRun( + buildRun([ + [ + 'tom', + { + status: 'error', + launchState: 'failed_to_start', + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: 'CLI process exited (code 1) - team provisioned but not alive', + livenessKind: 'confirmed_bootstrap', + runtimeDiagnostic: 'Runtime process crashed', + runtimeDiagnosticSeverity: 'error', + }, + ], + ]), + { nowIso } + ); + + expect(diagnostics).toEqual([ + { + id: 'tom:bootstrap_stalled', + memberName: 'tom', + severity: 'error', + code: 'bootstrap_stalled', + label: 'tom - launch diagnostic error', + detail: 'Runtime process crashed', + observedAt: NOW, + }, + ]); + }); + + it('keeps stopped liveness diagnostics for bootstrap-confirmed provisioned-but-not-alive entries', () => { + const diagnostics = buildLaunchDiagnosticsFromRun( + buildRun([ + [ + 'tom', + { + status: 'error', + launchState: 'failed_to_start', + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: 'CLI process exited (code 1) - team provisioned but not alive', + livenessKind: 'not_found', + runtimeDiagnostic: 'Runtime is no longer registered', + runtimeDiagnosticSeverity: 'warning', + }, + ], + ]), + { nowIso } + ); + + expect(diagnostics).toEqual([ + { + id: 'tom:bootstrap_stalled', + memberName: 'tom', + severity: 'error', + code: 'bootstrap_stalled', + label: 'tom - launch diagnostic error', + detail: 'Runtime is no longer registered', + observedAt: NOW, + }, + ]); + }); + it('uses failed launch error when hard failure reason is absent', () => { expect( buildLaunchDiagnosticsFromRun( diff --git a/test/main/services/team/TeamProvisioningLaunchFailurePolicy.test.ts b/test/main/services/team/TeamProvisioningLaunchFailurePolicy.test.ts index 9d0d6f38..99a94de7 100644 --- a/test/main/services/team/TeamProvisioningLaunchFailurePolicy.test.ts +++ b/test/main/services/team/TeamProvisioningLaunchFailurePolicy.test.ts @@ -3,6 +3,7 @@ import { isAutoClearableLaunchFailureReason, isBootstrapCheckInTimeoutFailureReason, isBootstrapInstructionPromptFailureReason, + isCliProvisionedButNotAliveFailureReason, isBootstrapMcpResourceReadFailureReason, isConfigRegistrationFailureReason, isLaunchCleanupBootstrapIncompleteFailureReason, @@ -10,9 +11,11 @@ import { isNeverSpawnedDuringLaunchReason, isOpenCodeBridgeLaunchFailureReason, isProcessTableUnavailableFailureReason, + isProvisionedButNotAliveFailureReason, isRegisteredRuntimeMetadataFailureReason, stripProcessTableUnavailableDiagnosticSuffix, } from '@main/services/team/provisioning/TeamProvisioningLaunchFailurePolicy'; +import { isBootstrapConfirmedProvisionedButNotAliveFailure } from '@shared/utils/teamLaunchFailureReason'; import { describe, expect, it } from 'vitest'; describe('TeamProvisioningLaunchFailurePolicy', () => { @@ -28,12 +31,27 @@ describe('TeamProvisioningLaunchFailurePolicy', () => { 'Teammate was not registered in config.json during launch. Persistent spawn failed.' ) ).toBe(true); - expect(isOpenCodeBridgeLaunchFailureReason('OpenCode bridge reported member launch failure')).toBe( - true - ); + expect( + isOpenCodeBridgeLaunchFailureReason('OpenCode bridge reported member launch failure') + ).toBe(true); expect( isRegisteredRuntimeMetadataFailureReason('registered runtime metadata without live process') ).toBe(true); + expect( + isProvisionedButNotAliveFailureReason( + 'CLI process exited (code 1) \u2014 team provisioned but not alive' + ) + ).toBe(true); + expect( + isProvisionedButNotAliveFailureReason( + 'CLI process exited (code unknown) - team provisioned but not alive; process table unavailable' + ) + ).toBe(true); + expect( + isCliProvisionedButNotAliveFailureReason( + 'CLI process exited (code ?) - team provisioned but not alive' + ) + ).toBe(true); }); it('recognizes bootstrap-specific failure reasons without accepting unrelated text', () => { @@ -42,9 +60,9 @@ describe('TeamProvisioningLaunchFailurePolicy', () => { 'resources/read failed for member_briefing: MCP error method not found' ) ).toBe(true); - expect(isBootstrapMcpResourceReadFailureReason('resources/read failed for other resource')).toBe( - false - ); + expect( + isBootstrapMcpResourceReadFailureReason('resources/read failed for other resource') + ).toBe(false); expect( isBootstrapCheckInTimeoutFailureReason( 'Teammate was registered but did not bootstrap-confirm before timeout.' @@ -69,9 +87,9 @@ describe('TeamProvisioningLaunchFailurePolicy', () => { 'runtime pid could not be verified because process table is unavailable' ) ).toBe(true); - expect(isProcessTableUnavailableFailureReason('runtime failed; process table unavailable')).toBe( - false - ); + expect( + isProcessTableUnavailableFailureReason('runtime failed; process table unavailable') + ).toBe(false); expect( stripProcessTableUnavailableDiagnosticSuffix( 'Teammate did not join within the launch grace window.; process table unavailable' @@ -80,9 +98,9 @@ describe('TeamProvisioningLaunchFailurePolicy', () => { }); it('keeps auto-clear policy narrow but accepts known recoverable suffixes', () => { - expect( - isAutoClearableLaunchFailureReason('Teammate was never spawned during launch.') - ).toBe(true); + expect(isAutoClearableLaunchFailureReason('Teammate was never spawned during launch.')).toBe( + true + ); expect(isAutoClearableLaunchFailureReason('process table is unavailable')).toBe(true); expect( isAutoClearableLaunchFailureReason( @@ -91,11 +109,57 @@ describe('TeamProvisioningLaunchFailurePolicy', () => { ).toBe(true); expect( isAutoClearableLaunchFailureReason( - 'CLI process exited (code 1) — team provisioned but not alive' + 'CLI process exited (code 1) \u2014 team provisioned but not alive' ) - ).toBe(true); + ).toBe(false); expect(isAutoClearableLaunchFailureReason('model not found')).toBe(false); - expect(isAutoClearableLaunchFailureReason(undefined)).toBe(false); + expect(isAutoClearableLaunchFailureReason()).toBe(false); + }); + + it('requires bootstrap proof before treating provisioned-but-not-alive as healed', () => { + const reason = 'CLI process exited (code 1) \u2014 team provisioned but not alive'; + + expect( + isBootstrapConfirmedProvisionedButNotAliveFailure({ + status: 'error', + launchState: 'failed_to_start', + hardFailure: true, + error: reason, + bootstrapConfirmed: true, + }) + ).toBe(true); + + expect( + isBootstrapConfirmedProvisionedButNotAliveFailure({ + status: 'error', + launchState: 'failed_to_start', + hardFailure: true, + hardFailureReason: 'model not found', + error: reason, + bootstrapConfirmed: true, + }) + ).toBe(false); + + expect( + isBootstrapConfirmedProvisionedButNotAliveFailure({ + status: 'error', + launchState: 'failed_to_start', + hardFailure: true, + hardFailureReason: reason, + bootstrapConfirmed: false, + livenessKind: 'registered_only', + }) + ).toBe(false); + + expect( + isBootstrapConfirmedProvisionedButNotAliveFailure({ + status: 'error', + launchState: 'failed_to_start', + hardFailure: true, + runtimeDiagnostic: reason, + bootstrapConfirmed: true, + }) + ).toBe(true); }); it('derives member launch state by the existing precedence order', () => { @@ -110,9 +174,7 @@ describe('TeamProvisioningLaunchFailurePolicy', () => { 'runtime_pending_permission' ); expect(deriveMemberLaunchState({ runtimeAlive: true })).toBe('runtime_pending_bootstrap'); - expect(deriveMemberLaunchState({ agentToolAccepted: true })).toBe( - 'runtime_pending_bootstrap' - ); + expect(deriveMemberLaunchState({ agentToolAccepted: true })).toBe('runtime_pending_bootstrap'); expect(deriveMemberLaunchState({})).toBe('starting'); }); }); diff --git a/test/main/services/team/TeamProvisioningPromptBuilders.test.ts b/test/main/services/team/TeamProvisioningPromptBuilders.test.ts new file mode 100644 index 00000000..f1d5ef9a --- /dev/null +++ b/test/main/services/team/TeamProvisioningPromptBuilders.test.ts @@ -0,0 +1,59 @@ +import { buildGeminiPostLaunchHydrationPrompt } from '@main/services/team/provisioning/TeamProvisioningPromptBuilders'; +import { describe, expect, it } from 'vitest'; + +import type { MemberSpawnStatusEntry, TeamCreateRequest } from '@shared/types'; + +function buildPromptWithStatus(status: MemberSpawnStatusEntry): string { + return buildGeminiPostLaunchHydrationPrompt( + { + teamName: 'signal-ops', + request: { prompt: 'Check readiness.' }, + memberSpawnStatuses: new Map([['tom', status]]), + }, + 'lead', + [{ name: 'tom', providerId: 'anthropic', model: 'sonnet' }] as TeamCreateRequest['members'], + [] + ); +} + +describe('TeamProvisioningPromptBuilders', () => { + it('keeps errored provisioned-but-not-alive members failed in Gemini hydration prompts', () => { + const prompt = buildPromptWithStatus({ + status: 'error', + launchState: 'failed_to_start', + agentToolAccepted: true, + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: 'CLI process exited (code 1) - team provisioned but not alive', + livenessKind: 'confirmed_bootstrap', + runtimeDiagnostic: 'Runtime process crashed', + runtimeDiagnosticSeverity: 'error', + updatedAt: '2026-05-25T20:14:02.147Z', + }); + + expect(prompt).toContain( + '- @tom: failed to start - CLI process exited (code 1) - team provisioned but not alive' + ); + expect(prompt).not.toContain('- @tom: bootstrap confirmed'); + }); + + it('keeps benign provisioned-but-not-alive members confirmed in Gemini hydration prompts', () => { + const prompt = buildPromptWithStatus({ + status: 'error', + launchState: 'failed_to_start', + agentToolAccepted: true, + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: 'CLI process exited (code 1) - team provisioned but not alive', + livenessKind: 'confirmed_bootstrap', + runtimeDiagnostic: 'runtime pid could not be verified because process table is unavailable', + runtimeDiagnosticSeverity: 'warning', + updatedAt: '2026-05-25T20:14:02.147Z', + }); + + expect(prompt).toContain('- @tom: bootstrap confirmed'); + expect(prompt).not.toContain('- @tom: failed to start'); + }); +}); diff --git a/test/main/services/team/TeamProvisioningService.test.ts b/test/main/services/team/TeamProvisioningService.test.ts index 6e8c9204..c9f6fc91 100644 --- a/test/main/services/team/TeamProvisioningService.test.ts +++ b/test/main/services/team/TeamProvisioningService.test.ts @@ -495,8 +495,7 @@ async function writeCommittedOpenCodeSessionStore(input: { { clock: () => new Date('2026-04-22T12:00:00.000Z'), batchIdFactory: () => `batch-${input.runId}${input.batchKey ? `-${input.batchKey}` : ''}`, - receiptIdFactory: () => - `receipt-${input.runId}${input.batchKey ? `-${input.batchKey}` : ''}`, + receiptIdFactory: () => `receipt-${input.runId}${input.batchKey ? `-${input.batchKey}` : ''}`, } ); await writer.writeBatch({ @@ -618,11 +617,12 @@ type TeamProvisioningServicePrivateHarness = { applyBootstrapTranscriptEvidenceOverlay: ( snapshot: ReturnType | null ) => Promise | null>; + applyProcessBootstrapTransportOverlay: ( + input: Record + ) => Record; }; -function privateHarness( - svc: TeamProvisioningService -): TeamProvisioningServicePrivateHarness { +function privateHarness(svc: TeamProvisioningService): TeamProvisioningServicePrivateHarness { return svc as unknown as TeamProvisioningServicePrivateHarness; } @@ -858,20 +858,12 @@ describe('TeamProvisioningService', () => { lastLeadTextEmitMs: 0, }; - internals.pushLiveLeadTextMessage( - run, - 'Соз', - undefined, - '2026-04-17T12:00:00.000Z', - { coalesceStreamChunk: true } - ); - internals.pushLiveLeadTextMessage( - run, - 'дал', - undefined, - '2026-04-17T12:00:00.010Z', - { coalesceStreamChunk: true } - ); + internals.pushLiveLeadTextMessage(run, 'Соз', undefined, '2026-04-17T12:00:00.000Z', { + coalesceStreamChunk: true, + }); + internals.pushLiveLeadTextMessage(run, 'дал', undefined, '2026-04-17T12:00:00.010Z', { + coalesceStreamChunk: true, + }); internals.pushLiveLeadTextMessage( run, ' стартовую задачу', @@ -2641,17 +2633,9 @@ describe('TeamProvisioningService', () => { internals.setLeadActivity(run, 'idle'); expect(resumeSpy).toHaveBeenCalledTimes(1); - expect(resumeSpy).toHaveBeenCalledWith( - teamName, - 'team-lead', - '2026-05-02T10:05:00.000Z' - ); + expect(resumeSpy).toHaveBeenCalledWith(teamName, 'team-lead', '2026-05-02T10:05:00.000Z'); expect(pauseSpy).toHaveBeenCalledTimes(1); - expect(pauseSpy).toHaveBeenCalledWith( - teamName, - 'team-lead', - '2026-05-02T10:05:00.000Z' - ); + expect(pauseSpy).toHaveBeenCalledWith(teamName, 'team-lead', '2026-05-02T10:05:00.000Z'); const staleRun = toLeadActivityTestRun( { @@ -8647,17 +8631,15 @@ describe('TeamProvisioningService', () => { generation: 2, observedAt: '2026-04-25T10:00:00.000Z', }; - const transportSpy = vi - .spyOn(agentTeamsMcpHttpServer, 'getCurrentHandle') - .mockReturnValue({ - url: currentTransportEvidence.url, - port: currentTransportEvidence.port, - child: { pid: 43123 }, - generation: currentTransportEvidence.generation, - urlHash: currentTransportEvidence.urlHash, - transportEvidence: currentTransportEvidence, - diagnostics: [], - } as any); + const transportSpy = vi.spyOn(agentTeamsMcpHttpServer, 'getCurrentHandle').mockReturnValue({ + url: currentTransportEvidence.url, + port: currentTransportEvidence.port, + child: { pid: 43123 }, + generation: currentTransportEvidence.generation, + urlHash: currentTransportEvidence.urlHash, + transportEvidence: currentTransportEvidence, + diagnostics: [], + } as any); try { await expect( @@ -8730,59 +8712,56 @@ describe('TeamProvisioningService', () => { generation: 3, observedAt: '2026-04-25T10:00:00.000Z', }; - const transportSpy = vi - .spyOn(agentTeamsMcpHttpServer, 'getCurrentHandle') - .mockReturnValue({ - url: currentTransportEvidence.url, - port: currentTransportEvidence.port, - child: { pid: 43124 }, - generation: currentTransportEvidence.generation, - urlHash: currentTransportEvidence.urlHash, - transportEvidence: currentTransportEvidence, - diagnostics: [], - } as any); + const transportSpy = vi.spyOn(agentTeamsMcpHttpServer, 'getCurrentHandle').mockReturnValue({ + url: currentTransportEvidence.url, + port: currentTransportEvidence.port, + child: { pid: 43124 }, + generation: currentTransportEvidence.generation, + urlHash: currentTransportEvidence.urlHash, + transportEvidence: currentTransportEvidence, + diagnostics: [], + } as any); const directBridgeExecute = vi.fn(async () => { throw new Error('direct OpenCode bridge executor should not be used for acceptance send'); }); - const stateChangingExecute = vi.fn(async (input: { - command: string; - body: Record; - }) => ({ - ok: true as const, - schemaVersion: OPEN_CODE_BRIDGE_SCHEMA_VERSION, - requestId: 'send-refresh-command', - command: input.command as any, - completedAt: '2026-04-25T10:00:01.000Z', - durationMs: 10, - runtime: { - providerId: 'opencode' as const, - binaryPath: '/opt/homebrew/bin/opencode', - binaryFingerprint: 'test-opencode-binary', - version: '1.0.0', - capabilitySnapshotId: 'test-capability-snapshot', - }, - diagnostics: [], - data: { - accepted: true, - memberName: 'bob', - sessionId: 'oc-session-bob-production-refresh', - runtimePid: 456, - runtimePromptMessageId: 'msg_prompt_production_refresh', - prePromptCursor: 'cursor-production-refresh', - responseObservation: { - state: 'pending', - deliveredUserMessageId: 'oc-user-production-refresh', - assistantMessageId: null, - toolCallNames: [], - visibleMessageToolCallId: null, - visibleReplyMessageId: null, - visibleReplyCorrelation: null, - latestAssistantPreview: null, - reason: 'assistant_response_pending', + const stateChangingExecute = vi.fn( + async (input: { command: string; body: Record }) => ({ + ok: true as const, + schemaVersion: OPEN_CODE_BRIDGE_SCHEMA_VERSION, + requestId: 'send-refresh-command', + command: input.command as any, + completedAt: '2026-04-25T10:00:01.000Z', + durationMs: 10, + runtime: { + providerId: 'opencode' as const, + binaryPath: '/opt/homebrew/bin/opencode', + binaryFingerprint: 'test-opencode-binary', + version: '1.0.0', + capabilitySnapshotId: 'test-capability-snapshot', }, diagnostics: [], - }, - })); + data: { + accepted: true, + memberName: 'bob', + sessionId: 'oc-session-bob-production-refresh', + runtimePid: 456, + runtimePromptMessageId: 'msg_prompt_production_refresh', + prePromptCursor: 'cursor-production-refresh', + responseObservation: { + state: 'pending', + deliveredUserMessageId: 'oc-user-production-refresh', + assistantMessageId: null, + toolCallNames: [], + visibleMessageToolCallId: null, + visibleReplyMessageId: null, + visibleReplyCorrelation: null, + latestAssistantPreview: null, + reason: 'assistant_response_pending', + }, + diagnostics: [], + }, + }) + ); const productionBridge = new OpenCodeReadinessBridge( { execute: directBridgeExecute }, { stateChangingCommands: { execute: stateChangingExecute as any } } @@ -8904,17 +8883,15 @@ describe('TeamProvisioningService', () => { generation: 7, observedAt: '2026-04-25T10:00:00.000Z', }; - const transportSpy = vi - .spyOn(agentTeamsMcpHttpServer, 'getCurrentHandle') - .mockReturnValue({ - url: currentTransportEvidence.url, - port: currentTransportEvidence.port, - child: { pid: 43128 }, - generation: currentTransportEvidence.generation, - urlHash: currentTransportEvidence.urlHash, - transportEvidence: currentTransportEvidence, - diagnostics: [], - } as any); + const transportSpy = vi.spyOn(agentTeamsMcpHttpServer, 'getCurrentHandle').mockReturnValue({ + url: currentTransportEvidence.url, + port: currentTransportEvidence.port, + child: { pid: 43128 }, + generation: currentTransportEvidence.generation, + urlHash: currentTransportEvidence.urlHash, + transportEvidence: currentTransportEvidence, + diagnostics: [], + } as any); try { await expect( @@ -9264,7 +9241,9 @@ describe('TeamProvisioningService', () => { { label: 'resolved behavior changes', staleReason: 'resolved_behavior_changed:old->new', - staleDiagnostics: ['OpenCode session reconcile skipped because the stored session is stale'], + staleDiagnostics: [ + 'OpenCode session reconcile skipped because the stored session is stale', + ], }, { label: 'action-required reasons', @@ -16618,9 +16597,7 @@ describe('TeamProvisioningService', () => { ] as never); mcpConfigBuilder.writeConfigFile.mockImplementation(async (_projectPath, policy) => { const mode = getMockMcpPolicyMode(policy); - return mode === 'appOnly' - ? '/mock/member-mcp-app-only.json' - : '/mock/lead-mcp-config.json'; + return mode === 'appOnly' ? '/mock/member-mcp-app-only.json' : '/mock/lead-mcp-config.json'; }); const { runId } = await svc.launchTeam( @@ -19465,9 +19442,7 @@ describe('TeamProvisioningService', () => { }; expect(persisted.teamLaunchState).toBe('partial_failure'); expect(persisted.members?.alice?.launchState).toBe('failed_to_start'); - expect(persisted.members?.alice?.hardFailureReason).toContain( - 'team provisioned but not alive' - ); + expect(persisted.members?.alice?.hardFailureReason).toContain('team provisioned but not alive'); const reconciled = await (svc as any).reconcilePersistedLaunchState(teamName); expect(reconciled.snapshot?.teamLaunchState).toBe('partial_failure'); @@ -20429,6 +20404,188 @@ describe('TeamProvisioningService', () => { expect(result.statuses.jack?.runtimeDiagnosticSeverity).toBeUndefined(); }); + it('heals provisioned-but-not-alive launch failures when bootstrap-state confirms the member', async () => { + allowConsoleLogs(); + const teamName = 'zz-unit-provisioned-not-alive-bootstrap-state-heals'; + const leadSessionId = 'lead-session'; + const projectPath = '/Users/test/proj'; + const acceptedAt = new Date(Date.now() - 90_000).toISOString(); + const bootstrapAt = new Date(Date.now() - 60_000).toISOString(); + const cleanupAt = new Date(Date.now() - 30_000).toISOString(); + const runtimePid = 27_036; + const exitReason = 'CLI process exited (code 1) \u2014 team provisioned but not alive'; + const processTableReason = + 'runtime pid could not be verified because process table is unavailable'; + + writeLaunchConfig(teamName, projectPath, leadSessionId, ['tom']); + writeLaunchState( + teamName, + leadSessionId, + { + tom: { + providerId: 'anthropic', + model: 'sonnet', + laneId: 'primary', + laneKind: 'primary', + laneOwnerProviderId: 'anthropic', + launchState: 'failed_to_start', + agentToolAccepted: true, + runtimeAlive: false, + runtimePid, + bootstrapConfirmed: false, + hardFailure: true, + hardFailureReason: exitReason, + livenessKind: 'registered_only', + runtimeDiagnostic: processTableReason, + runtimeDiagnosticSeverity: 'warning', + firstSpawnAcceptedAt: acceptedAt, + lastEvaluatedAt: cleanupAt, + }, + }, + { launchPhase: 'finished', updatedAt: cleanupAt } + ); + writeBootstrapState( + teamName, + [ + { + name: 'tom', + status: 'bootstrap_confirmed', + lastAttemptAt: Date.parse(acceptedAt), + lastObservedAt: Date.parse(bootstrapAt), + }, + ], + bootstrapAt + ); + + const svc = new TeamProvisioningService(); + privateHarness(svc).getLiveTeamAgentRuntimeMetadata = vi.fn( + async () => + new Map([ + [ + 'tom', + { + alive: false, + backendType: 'process', + providerId: 'anthropic', + livenessKind: 'registered_only', + pidSource: 'persisted_metadata', + runtimeDiagnostic: processTableReason, + runtimeDiagnosticSeverity: 'warning', + metricsPid: runtimePid, + model: 'sonnet', + }, + ], + ]) + ); + + const result = await svc.getMemberSpawnStatuses(teamName); + + expect(result.teamLaunchState).toBe('clean_success'); + expect(result.statuses.tom).toMatchObject({ + status: 'online', + launchState: 'confirmed_alive', + bootstrapConfirmed: true, + runtimeAlive: true, + livenessKind: 'confirmed_bootstrap', + hardFailure: false, + error: undefined, + }); + expect(result.statuses.tom?.hardFailureReason).toBeUndefined(); + expect(result.statuses.tom?.runtimeDiagnostic).toBeUndefined(); + expect(result.statuses.tom?.runtimeDiagnosticSeverity).toBeUndefined(); + }); + + it('does not heal provisioned-but-not-alive live status when refreshed runtime metadata is unsafe', async () => { + allowConsoleLogs(); + const teamName = 'zz-unit-provisioned-not-alive-live-runtime-error-stays-failed'; + const leadSessionId = 'lead-session'; + const projectPath = '/Users/test/proj'; + const acceptedAt = new Date(Date.now() - 90_000).toISOString(); + const bootstrapAt = new Date(Date.now() - 60_000).toISOString(); + const cleanupAt = new Date(Date.now() - 30_000).toISOString(); + const runtimePid = 27_036; + const exitReason = + 'CLI process exited (code 1) - team provisioned but not alive; process table unavailable'; + const processTableReason = + 'runtime pid could not be verified because process table is unavailable'; + + writeLaunchConfig(teamName, projectPath, leadSessionId, ['tom']); + writeLaunchState( + teamName, + leadSessionId, + { + tom: { + providerId: 'anthropic', + model: 'sonnet', + laneId: 'primary', + laneKind: 'primary', + laneOwnerProviderId: 'anthropic', + launchState: 'failed_to_start', + agentToolAccepted: true, + runtimeAlive: false, + runtimePid, + bootstrapConfirmed: false, + hardFailure: true, + hardFailureReason: exitReason, + livenessKind: 'registered_only', + runtimeDiagnostic: processTableReason, + runtimeDiagnosticSeverity: 'warning', + firstSpawnAcceptedAt: acceptedAt, + lastEvaluatedAt: cleanupAt, + }, + }, + { launchPhase: 'finished', updatedAt: cleanupAt } + ); + writeBootstrapState( + teamName, + [ + { + name: 'tom', + status: 'bootstrap_confirmed', + lastAttemptAt: Date.parse(acceptedAt), + lastObservedAt: Date.parse(bootstrapAt), + }, + ], + bootstrapAt + ); + + const svc = new TeamProvisioningService(); + privateHarness(svc).getLiveTeamAgentRuntimeMetadata = vi.fn( + async () => + new Map([ + [ + 'tom', + { + alive: false, + backendType: 'process', + providerId: 'anthropic', + livenessKind: 'not_found', + pidSource: 'process_table', + runtimeDiagnostic: 'Runtime process crashed', + runtimeDiagnosticSeverity: 'error', + metricsPid: runtimePid, + model: 'sonnet', + }, + ], + ]) + ); + + const result = await svc.getMemberSpawnStatuses(teamName); + + expect(result.teamLaunchState).toBe('partial_failure'); + expect(result.statuses.tom).toMatchObject({ + status: 'error', + launchState: 'failed_to_start', + bootstrapConfirmed: true, + runtimeAlive: false, + livenessKind: 'not_found', + hardFailure: true, + hardFailureReason: exitReason, + runtimeDiagnostic: 'Runtime process crashed', + runtimeDiagnosticSeverity: 'error', + }); + }); + it('heals process-table unavailable failure when Anthropic bootstrap confirmation slightly predates delayed app acceptance', async () => { allowConsoleLogs(); const teamName = 'zz-unit-process-table-unavailable-bootstrap-skew-heals'; @@ -21183,6 +21340,36 @@ describe('TeamProvisioningService', () => { ); }); + it('does not downgrade provisioned-but-not-alive failures from process transport progress alone', () => { + const svc = new TeamProvisioningService(); + const reason = 'CLI process exited (code 1) \u2014 team provisioned but not alive'; + const result = privateHarness(svc).applyProcessBootstrapTransportOverlay({ + member: { + name: 'jack', + launchState: 'failed_to_start', + agentToolAccepted: true, + runtimeAlive: false, + bootstrapConfirmed: false, + hardFailure: true, + hardFailureReason: reason, + lastEvaluatedAt: '2026-05-25T20:14:05.411Z', + }, + summary: { + hasProgress: true, + submitted: true, + lastStage: 'bootstrap submitted', + }, + launchPhase: 'active', + }); + + expect(result).toMatchObject({ + launchState: 'failed_to_start', + bootstrapConfirmed: false, + hardFailure: true, + hardFailureReason: reason, + }); + }); + it('uses the last process transport stage when active launch grace expires', async () => { allowConsoleLogs(); const teamName = 'zz-unit-process-bootstrap-transport-timeout'; @@ -22426,7 +22613,7 @@ describe('TeamProvisioningService', () => { }); }); - it('marks a live teammate bootstrap as confirmed from transcript even when runtime discovery is stale', async () => { + it('marks a live teammate bootstrap as confirmed from transcript without claiming runtime is alive', async () => { allowConsoleLogs(); const teamName = 'zz-live-bootstrap-transcript-success-without-runtime'; const leadSessionId = 'lead-session'; @@ -23187,7 +23374,11 @@ describe('TeamProvisioningService', () => { fs.mkdirSync(worktreeGitDir, { recursive: true }); fs.writeFileSync(path.join(worktreeDir, '.git'), `gitdir: ${worktreeGitDir}\n`, 'utf8'); fs.writeFileSync(path.join(worktreeGitDir, 'commondir'), '../..\n', 'utf8'); - fs.writeFileSync(path.join(worktreeGitDir, 'gitdir'), `${path.join(worktreeDir, '.git')}\n`, 'utf8'); + fs.writeFileSync( + path.join(worktreeGitDir, 'gitdir'), + `${path.join(worktreeDir, '.git')}\n`, + 'utf8' + ); const workspaces = await harness.collectWorkspaceTrustWorkspaces({ cwd: repoDir, @@ -23202,9 +23393,7 @@ describe('TeamProvisioningService', () => { gitRootConfigKey: repoDir, memberId: 'alice', }); - expect(memberWorktrees.every((workspace) => workspace.gitRootConfigKey === repoDir)).toBe( - true - ); + expect(memberWorktrees.every((workspace) => workspace.gitRootConfigKey === repoDir)).toBe(true); }); it('degrades workspace trust planning failures without blocking launch preparation', async () => { @@ -23835,7 +24024,8 @@ describe('TeamProvisioningService', () => { { alive: false, livenessKind: 'registered_only', - runtimeDiagnostic: 'runtime pid could not be verified because process table is unavailable', + runtimeDiagnostic: + 'runtime pid could not be verified because process table is unavailable', runtimeDiagnosticSeverity: 'warning', }, ], @@ -23869,6 +24059,164 @@ describe('TeamProvisioningService', () => { }); }); + it('clears provisioned-but-not-alive failure from confirmed bootstrap even with weak metadata', async () => { + const svc = new TeamProvisioningService(); + const harness = privateHarness(svc); + const processTableReason = + 'runtime pid could not be verified because process table is unavailable'; + const exitReason = 'CLI process exited (code 1) \u2014 team provisioned but not alive'; + harness.getLiveTeamAgentRuntimeMetadata = vi.fn( + async () => + new Map([ + [ + 'tom', + { + alive: false, + model: 'sonnet', + livenessKind: 'registered_only', + pidSource: 'persisted_metadata', + runtimeDiagnostic: processTableReason, + runtimeDiagnosticSeverity: 'warning', + }, + ], + ]) + ); + + const result = await harness.attachLiveRuntimeMetadataToStatuses('signal-ops', { + tom: createMemberSpawnStatusEntry({ + status: 'error', + launchState: 'failed_to_start', + error: exitReason, + hardFailure: true, + hardFailureReason: exitReason, + agentToolAccepted: true, + runtimeAlive: false, + bootstrapConfirmed: true, + livenessKind: 'confirmed_bootstrap', + runtimeDiagnostic: processTableReason, + runtimeDiagnosticSeverity: 'warning', + firstSpawnAcceptedAt: '2026-05-25T20:13:46.326Z', + lastHeartbeatAt: '2026-05-25T20:13:56.110Z', + }), + }); + + expect(result.tom).toMatchObject({ + status: 'online', + launchState: 'confirmed_alive', + runtimeAlive: true, + bootstrapConfirmed: true, + hardFailure: false, + hardFailureReason: undefined, + error: undefined, + runtimeModel: 'sonnet', + livenessKind: 'confirmed_bootstrap', + runtimeDiagnostic: undefined, + runtimeDiagnosticSeverity: undefined, + livenessSource: undefined, + }); + }); + + it('does not let weak metadata undo confirmed bootstrap failure healing', async () => { + const svc = new TeamProvisioningService(); + const harness = privateHarness(svc); + const processTableReason = + 'runtime pid could not be verified because process table is unavailable'; + const exitReason = 'CLI process exited (code 1) \u2014 team provisioned but not alive'; + harness.getLiveTeamAgentRuntimeMetadata = vi.fn( + async () => + new Map([ + [ + 'tom', + { + alive: false, + model: 'sonnet', + livenessKind: 'registered_only', + pidSource: 'persisted_metadata', + runtimeDiagnostic: processTableReason, + runtimeDiagnosticSeverity: 'warning', + }, + ], + ]) + ); + + const result = await harness.attachLiveRuntimeMetadataToStatuses('signal-ops', { + tom: createMemberSpawnStatusEntry({ + status: 'error', + launchState: 'confirmed_alive', + error: exitReason, + hardFailure: true, + hardFailureReason: exitReason, + agentToolAccepted: true, + runtimeAlive: false, + bootstrapConfirmed: false, + runtimeDiagnostic: processTableReason, + runtimeDiagnosticSeverity: 'warning', + firstSpawnAcceptedAt: '2026-05-25T20:13:46.326Z', + lastHeartbeatAt: '2026-05-25T20:13:56.110Z', + }), + }); + + expect(result.tom).toMatchObject({ + status: 'online', + launchState: 'confirmed_alive', + runtimeAlive: true, + bootstrapConfirmed: true, + hardFailure: false, + hardFailureReason: undefined, + error: undefined, + runtimeModel: 'sonnet', + }); + }); + + it('does not keep healed confirmed-bootstrap status alive when refreshed runtime metadata is an error', async () => { + const svc = new TeamProvisioningService(); + const harness = privateHarness(svc); + harness.getLiveTeamAgentRuntimeMetadata = vi.fn( + () => + Promise.resolve( + new Map([ + [ + 'tom', + { + alive: false, + model: 'sonnet', + livenessKind: 'not_found', + pidSource: 'process_table', + runtimeDiagnostic: 'Runtime process crashed', + runtimeDiagnosticSeverity: 'error', + }, + ], + ]) + ) + ); + + const result = await harness.attachLiveRuntimeMetadataToStatuses('signal-ops', { + tom: createMemberSpawnStatusEntry({ + status: 'online', + launchState: 'confirmed_alive', + agentToolAccepted: true, + runtimeAlive: true, + bootstrapConfirmed: true, + hardFailure: false, + livenessKind: 'confirmed_bootstrap', + runtimeModel: 'sonnet', + }), + }); + + expect(result.tom).toMatchObject({ + status: 'online', + launchState: 'confirmed_alive', + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: false, + livenessKind: 'not_found', + runtimeDiagnostic: 'Runtime process crashed', + runtimeDiagnosticSeverity: 'error', + runtimeModel: 'sonnet', + livenessSource: undefined, + }); + }); + it('does not clear OpenCode bridge launch failure from process-only liveness', async () => { const svc = new TeamProvisioningService(); (svc as any).getLiveTeamAgentRuntimeMetadata = vi.fn( @@ -26824,7 +27172,7 @@ describe('TeamProvisioningService', () => { status: 'online', launchState: 'confirmed_alive', bootstrapConfirmed: true, - runtimeAlive: false, + runtimeAlive: true, livenessKind: 'confirmed_bootstrap', hardFailure: false, error: undefined, @@ -26834,6 +27182,288 @@ describe('TeamProvisioningService', () => { expect(result.statuses.tom?.runtimeDiagnosticSeverity).toBeUndefined(); }); + it('keeps primary provisioned-but-not-alive reporting failed when runtime evidence is unsafe', async () => { + const teamName = 'primary-bootstrap-cli-provisioned-not-alive-runtime-error'; + const bootstrapRunId = 'run-primary-cli-exit-runtime-error'; + const reason = 'CLI process exited (code 1) - team provisioned but not alive'; + writeTeamMeta(teamName, { + providerId: 'anthropic', + model: 'sonnet', + }); + writeMembersMeta(teamName, [{ name: 'tom', providerId: 'anthropic', model: 'sonnet' }]); + writeLaunchConfig(teamName, '/Users/test/proj', 'lead-session', ['tom']); + writeMemberBootstrapRunId(teamName, 'tom', bootstrapRunId); + writeBootstrapState( + teamName, + [ + { + name: 'tom', + status: 'bootstrap_confirmed', + lastAttemptAt: Date.parse('2026-05-25T20:13:46.326Z'), + lastObservedAt: Date.parse('2026-05-25T20:13:56.110Z'), + }, + ], + '2026-05-25T20:14:03.317Z', + { runId: bootstrapRunId } + ); + fs.writeFileSync( + getTeamLaunchStatePath(teamName), + `${JSON.stringify( + createPersistedLaunchSnapshot({ + teamName, + leadSessionId: 'lead-session', + launchPhase: 'finished', + expectedMembers: ['tom'], + members: { + tom: { + name: 'tom', + providerId: 'anthropic', + model: 'sonnet', + laneId: 'primary', + laneKind: 'primary', + laneOwnerProviderId: 'anthropic', + launchState: 'failed_to_start', + agentToolAccepted: true, + runtimeAlive: false, + runtimePid: 27_036, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: reason, + livenessKind: 'confirmed_bootstrap', + pidSource: 'persisted_metadata', + runtimeDiagnostic: 'Runtime process crashed', + runtimeDiagnosticSeverity: 'error', + firstSpawnAcceptedAt: '2026-05-25T20:13:46.326Z', + lastHeartbeatAt: '2026-05-25T20:13:56.110Z', + runtimeLastSeenAt: '2026-05-25T20:13:46.326Z', + lastEvaluatedAt: '2026-05-25T20:14:05.411Z', + }, + }, + updatedAt: '2026-05-25T20:14:05.411Z', + }), + null, + 2 + )}\n`, + 'utf8' + ); + + const svc = new TeamProvisioningService(); + const result = await svc.getMemberSpawnStatuses(teamName); + + expect(result.statuses.tom).toMatchObject({ + status: 'error', + launchState: 'failed_to_start', + bootstrapConfirmed: true, + runtimeAlive: false, + livenessKind: 'stale_metadata', + hardFailure: true, + hardFailureReason: reason, + runtimeDiagnostic: 'Runtime process crashed', + runtimeDiagnosticSeverity: 'error', + }); + expect(result.teamLaunchState).toBe('partial_failure'); + }); + + it('keeps provisioned-but-not-alive failed when refreshed runtime evidence is unsafe', async () => { + const teamName = 'primary-bootstrap-cli-provisioned-not-alive-refreshed-runtime-error'; + const bootstrapRunId = 'run-primary-cli-exit-refreshed-runtime-error'; + const reason = + 'CLI process exited (code 1) - team provisioned but not alive; process table unavailable'; + writeTeamMeta(teamName, { + providerId: 'anthropic', + model: 'sonnet', + }); + writeMembersMeta(teamName, [{ name: 'tom', providerId: 'anthropic', model: 'sonnet' }]); + writeLaunchConfig(teamName, '/Users/test/proj', 'lead-session', ['tom']); + writeMemberBootstrapRunId(teamName, 'tom', bootstrapRunId); + writeBootstrapState( + teamName, + [ + { + name: 'tom', + status: 'bootstrap_confirmed', + lastAttemptAt: Date.parse('2026-05-25T20:13:46.326Z'), + lastObservedAt: Date.parse('2026-05-25T20:13:56.110Z'), + }, + ], + '2026-05-25T20:14:03.317Z', + { runId: bootstrapRunId } + ); + fs.writeFileSync( + getTeamLaunchStatePath(teamName), + `${JSON.stringify( + createPersistedLaunchSnapshot({ + teamName, + leadSessionId: 'lead-session', + launchPhase: 'finished', + expectedMembers: ['tom'], + members: { + tom: { + name: 'tom', + providerId: 'anthropic', + model: 'sonnet', + laneId: 'primary', + laneKind: 'primary', + laneOwnerProviderId: 'anthropic', + launchState: 'failed_to_start', + agentToolAccepted: true, + runtimeAlive: false, + runtimePid: 27_036, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: reason, + livenessKind: 'registered_only', + pidSource: 'persisted_metadata', + runtimeDiagnostic: + 'runtime pid could not be verified because process table is unavailable', + runtimeDiagnosticSeverity: 'warning', + firstSpawnAcceptedAt: '2026-05-25T20:13:46.326Z', + lastHeartbeatAt: '2026-05-25T20:13:56.110Z', + runtimeLastSeenAt: '2026-05-25T20:13:46.326Z', + lastEvaluatedAt: '2026-05-25T20:14:05.411Z', + }, + }, + updatedAt: '2026-05-25T20:14:05.411Z', + }), + null, + 2 + )}\n`, + 'utf8' + ); + + const svc = new TeamProvisioningService(); + (svc as any).getLiveTeamAgentRuntimeMetadata = vi.fn( + async () => + new Map([ + [ + 'tom', + { + alive: false, + livenessKind: 'not_found', + runtimeDiagnostic: 'Runtime process crashed', + runtimeDiagnosticSeverity: 'error', + pidSource: 'process_table', + }, + ], + ]) + ); + const result = await svc.getMemberSpawnStatuses(teamName); + + expect(result.statuses.tom).toMatchObject({ + status: 'error', + launchState: 'failed_to_start', + bootstrapConfirmed: true, + runtimeAlive: false, + livenessKind: 'not_found', + hardFailure: true, + hardFailureReason: reason, + runtimeDiagnostic: 'Runtime process crashed', + runtimeDiagnosticSeverity: 'error', + }); + expect(result.teamLaunchState).toBe('partial_failure'); + }); + + it('keeps provisioned-but-not-alive failed when refreshed runtime evidence is only a candidate', async () => { + const teamName = 'primary-bootstrap-cli-provisioned-not-alive-runtime-candidate'; + const bootstrapRunId = 'run-primary-cli-exit-runtime-candidate'; + const reason = + 'CLI process exited (code 1) - team provisioned but not alive; process table unavailable'; + writeTeamMeta(teamName, { + providerId: 'anthropic', + model: 'sonnet', + }); + writeMembersMeta(teamName, [{ name: 'tom', providerId: 'anthropic', model: 'sonnet' }]); + writeLaunchConfig(teamName, '/Users/test/proj', 'lead-session', ['tom']); + writeMemberBootstrapRunId(teamName, 'tom', bootstrapRunId); + writeBootstrapState( + teamName, + [ + { + name: 'tom', + status: 'bootstrap_confirmed', + lastAttemptAt: Date.parse('2026-05-25T20:13:46.326Z'), + lastObservedAt: Date.parse('2026-05-25T20:13:56.110Z'), + }, + ], + '2026-05-25T20:14:03.317Z', + { runId: bootstrapRunId } + ); + fs.writeFileSync( + getTeamLaunchStatePath(teamName), + `${JSON.stringify( + createPersistedLaunchSnapshot({ + teamName, + leadSessionId: 'lead-session', + launchPhase: 'finished', + expectedMembers: ['tom'], + members: { + tom: { + name: 'tom', + providerId: 'anthropic', + model: 'sonnet', + laneId: 'primary', + laneKind: 'primary', + laneOwnerProviderId: 'anthropic', + launchState: 'failed_to_start', + agentToolAccepted: true, + runtimeAlive: false, + runtimePid: 27_036, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: reason, + livenessKind: 'registered_only', + pidSource: 'persisted_metadata', + runtimeDiagnostic: + 'runtime pid could not be verified because process table is unavailable', + runtimeDiagnosticSeverity: 'warning', + firstSpawnAcceptedAt: '2026-05-25T20:13:46.326Z', + lastHeartbeatAt: '2026-05-25T20:13:56.110Z', + runtimeLastSeenAt: '2026-05-25T20:13:46.326Z', + lastEvaluatedAt: '2026-05-25T20:14:05.411Z', + }, + }, + updatedAt: '2026-05-25T20:14:05.411Z', + }), + null, + 2 + )}\n`, + 'utf8' + ); + + const svc = new TeamProvisioningService(); + (svc as any).getLiveTeamAgentRuntimeMetadata = vi.fn( + async () => + new Map([ + [ + 'tom', + { + alive: false, + livenessKind: 'runtime_process_candidate', + runtimeDiagnostic: + 'OpenCode runtime process detected, but teammate bootstrap is not confirmed', + runtimeDiagnosticSeverity: 'warning', + pidSource: 'opencode_bridge', + }, + ], + ]) + ); + const result = await svc.getMemberSpawnStatuses(teamName); + + expect(result.statuses.tom).toMatchObject({ + status: 'error', + launchState: 'failed_to_start', + bootstrapConfirmed: true, + runtimeAlive: false, + livenessKind: 'runtime_process_candidate', + hardFailure: true, + hardFailureReason: reason, + runtimeDiagnostic: + 'OpenCode runtime process detected, but teammate bootstrap is not confirmed', + runtimeDiagnosticSeverity: 'warning', + }); + expect(result.teamLaunchState).toBe('partial_failure'); + }); + it('cleans stale confirmed primary diagnostics from an already successful mixed launch', async () => { const teamName = 'mixed-confirmed-primary-stale-diagnostic-cleans'; writeTeamMeta(teamName, { @@ -27160,9 +27790,7 @@ describe('TeamProvisioningService', () => { model: 'claude-opus-4-7', members: [], }; - run.effectiveMembers = [ - { name: 'jack', providerId: 'anthropic', model: 'claude-opus-4-7' }, - ]; + run.effectiveMembers = [{ name: 'jack', providerId: 'anthropic', model: 'claude-opus-4-7' }]; fs.mkdirSync(path.join(tempTeamsBase, teamName), { recursive: true }); writeBootstrapState( teamName, @@ -27188,11 +27816,11 @@ describe('TeamProvisioningService', () => { ).persistLaunchStateSnapshot(run, 'finished'); expect(snapshot).toBeNull(); - await expect(fsPromises.readFile(getTeamLaunchStatePath(teamName), 'utf8')).rejects.toMatchObject( - { - code: 'ENOENT', - } - ); + await expect( + fsPromises.readFile(getTeamLaunchStatePath(teamName), 'utf8') + ).rejects.toMatchObject({ + code: 'ENOENT', + }); }); it('includes queued OpenCode secondary lanes in live spawn statuses during createTeam runs', async () => { diff --git a/test/renderer/components/team/members/MemberCard.test.ts b/test/renderer/components/team/members/MemberCard.test.ts index 10e43148..c0c0fd04 100644 --- a/test/renderer/components/team/members/MemberCard.test.ts +++ b/test/renderer/components/team/members/MemberCard.test.ts @@ -1217,6 +1217,55 @@ describe('MemberCard starting-state visuals', () => { }); }); + it('keeps stopped provisioned-but-not-alive launches failed and retryable', async () => { + vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true); + const host = document.createElement('div'); + document.body.appendChild(host); + const root = createRoot(host); + const reason = 'CLI process exited (code 1) - team provisioned but not alive'; + const spawnEntry: MemberSpawnStatusEntry = { + status: 'error', + launchState: 'failed_to_start', + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: reason, + agentToolAccepted: true, + livenessKind: 'not_found', + runtimeDiagnostic: 'Runtime is no longer registered', + runtimeDiagnosticSeverity: 'warning', + updatedAt: '2026-05-25T20:14:02.147Z', + }; + + await act(async () => { + root.render( + React.createElement(MemberCard, { + member, + memberColor: 'blue', + isTeamAlive: true, + isTeamProvisioning: false, + spawnStatus: 'error', + spawnLaunchState: 'failed_to_start', + spawnRuntimeAlive: false, + spawnError: reason, + spawnEntry, + onRestartMember: vi.fn(), + onSkipMemberForLaunch: vi.fn(), + }) + ); + await Promise.resolve(); + }); + + expect(host.querySelector('[data-testid="member-launch-failure-reason"]')).not.toBeNull(); + expect(host.querySelector('[aria-label="Retry teammate"]')).not.toBeNull(); + expect(host.querySelector('[aria-label="Skip for this launch"]')).not.toBeNull(); + + await act(async () => { + root.unmount(); + await Promise.resolve(); + }); + }); + it('shows a compact failed launch reason on the member row with clickable links', async () => { vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true); const host = document.createElement('div'); diff --git a/test/renderer/components/team/members/MemberDetailDialog.test.ts b/test/renderer/components/team/members/MemberDetailDialog.test.ts index 2cbeddaa..6d6ce4e7 100644 --- a/test/renderer/components/team/members/MemberDetailDialog.test.ts +++ b/test/renderer/components/team/members/MemberDetailDialog.test.ts @@ -531,6 +531,67 @@ describe('MemberDetailDialog activity count', () => { }); }); + it('shows Relaunch OpenCode copy for unsafe provisioned-but-not-alive OpenCode teammates without runtime evidence', async () => { + const member: ResolvedTeamMember = { + name: 'jack', + status: 'active', + currentTaskId: null, + taskCount: 0, + lastActiveAt: null, + messageCount: 0, + providerId: 'opencode', + }; + const onRestartMember = vi.fn(() => Promise.resolve(undefined)); + const host = document.createElement('div'); + document.body.appendChild(host); + const root = createRoot(host); + + await act(async () => { + root.render( + React.createElement(MemberDetailDialog, { + open: true, + member, + teamName: 'demo-team', + members: [member], + tasks: [], + isTeamAlive: true, + spawnEntry: { + status: 'error', + launchState: 'failed_to_start', + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: 'CLI process exited (code 1) - team provisioned but not alive', + agentToolAccepted: true, + livenessKind: 'not_found', + runtimeDiagnostic: 'Runtime is no longer registered', + runtimeDiagnosticSeverity: 'warning', + updatedAt: '2026-04-24T12:00:00.000Z', + }, + onClose: () => undefined, + onSendMessage: () => undefined, + onAssignTask: () => undefined, + onTaskClick: () => undefined, + onRestartMember, + }) + ); + await Promise.resolve(); + }); + + expect(host.textContent).toContain( + 'No OpenCode runtime session was recorded. Relaunch this teammate to start a fresh OpenCode session.' + ); + const relaunchButton = Array.from(host.querySelectorAll('button')).find((button) => + button.textContent?.includes('Relaunch OpenCode') + ); + expect(relaunchButton).not.toBeUndefined(); + + await act(async () => { + root.unmount(); + await Promise.resolve(); + }); + }); + it('shows Relaunch OpenCode copy for stalled OpenCode bootstrap', async () => { const member: ResolvedTeamMember = { name: 'tom', diff --git a/test/renderer/components/team/members/MemberList.test.ts b/test/renderer/components/team/members/MemberList.test.ts index 6f5528de..f2bcb0d9 100644 --- a/test/renderer/components/team/members/MemberList.test.ts +++ b/test/renderer/components/team/members/MemberList.test.ts @@ -3,7 +3,12 @@ import { createRoot } from 'react-dom/client'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import type { MemberSpawnStatusEntry, ResolvedTeamMember, TeamTaskWithKanban } from '@shared/types'; +import type { + MemberSpawnStatusEntry, + ResolvedTeamMember, + TeamAgentRuntimeEntry, + TeamTaskWithKanban, +} from '@shared/types'; vi.mock('@renderer/components/team/members/MemberCard', () => ({ MemberCard: ({ @@ -115,6 +120,19 @@ function offlineSpawnStatus(): MemberSpawnStatusEntry { }; } +function provisionedButNotAliveSpawnStatus(): MemberSpawnStatusEntry { + return { + status: 'error', + launchState: 'failed_to_start', + updatedAt: '2026-05-25T20:14:02.147Z', + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: 'CLI process exited (code 1) \u2014 team provisioned but not alive', + livenessKind: 'confirmed_bootstrap', + }; +} + function activeTask(id = 'task-active'): TeamTaskWithKanban { return { id, @@ -543,6 +561,142 @@ describe('MemberList spawn-status memoization', () => { }); }); + it('keeps tasks visible and suppresses launch actions for healed provisioned-but-not-alive status', async () => { + vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true); + const host = document.createElement('div'); + document.body.appendChild(host); + const root = createRoot(host); + const task = activeTask(); + const members: ResolvedTeamMember[] = [{ ...member, currentTaskId: task.id }]; + const restart = vi.fn(); + const skip = vi.fn(); + + await act(async () => { + root.render( + React.createElement(MemberList, { + members, + isTeamAlive: true, + taskMap: new Map([[task.id, task]]), + memberSpawnStatuses: new Map([['bob', provisionedButNotAliveSpawnStatus()]]), + memberRuntimeEntries: new Map([ + [ + 'bob', + { + memberName: 'bob', + alive: false, + restartable: true, + livenessKind: 'confirmed_bootstrap', + runtimeDiagnostic: + 'runtime pid could not be verified because process table is unavailable', + runtimeDiagnosticSeverity: 'warning', + updatedAt: '2026-05-25T20:14:05.411Z', + }, + ], + ]), + onRestartMember: restart, + onSkipMemberForLaunch: skip, + }) + ); + await Promise.resolve(); + }); + + expect(host.querySelector('[data-testid="current-bob"]')?.textContent).toBe(task.id); + expect(host.querySelector('[data-testid="retry-bob"]')).toBeNull(); + expect(host.querySelector('[data-testid="skip-bob"]')).toBeNull(); + expect(host.textContent).not.toContain('team provisioned but not alive'); + + await act(async () => { + root.unmount(); + await Promise.resolve(); + }); + }); + + it('keeps stopped provisioned-but-not-alive status failed and actionable', async () => { + vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true); + const host = document.createElement('div'); + document.body.appendChild(host); + const root = createRoot(host); + const task = activeTask(); + const members: ResolvedTeamMember[] = [{ ...member, currentTaskId: task.id }]; + const restart = vi.fn(); + const skip = vi.fn(); + const spawnEntry = { + ...provisionedButNotAliveSpawnStatus(), + livenessKind: 'not_found', + runtimeDiagnostic: 'Runtime is no longer registered', + runtimeDiagnosticSeverity: 'warning', + } satisfies MemberSpawnStatusEntry; + + await act(async () => { + root.render( + React.createElement(MemberList, { + members, + isTeamAlive: true, + taskMap: new Map([[task.id, task]]), + memberSpawnStatuses: new Map([['bob', spawnEntry]]), + onRestartMember: restart, + onSkipMemberForLaunch: skip, + }) + ); + await Promise.resolve(); + }); + + expect(host.querySelector('[data-testid="current-bob"]')).toBeNull(); + expect(host.querySelector('[data-testid="retry-bob"]')).not.toBeNull(); + expect(host.querySelector('[data-testid="skip-bob"]')).not.toBeNull(); + expect(host.textContent).toContain('team provisioned but not alive'); + + await act(async () => { + root.unmount(); + await Promise.resolve(); + }); + }); + + it('hides tasks for healed provisioned-but-not-alive status when runtime has an error', async () => { + vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true); + const host = document.createElement('div'); + document.body.appendChild(host); + const root = createRoot(host); + const task = activeTask(); + const members: ResolvedTeamMember[] = [{ ...member, currentTaskId: task.id }]; + + await act(async () => { + root.render( + React.createElement(MemberList, { + members, + isTeamAlive: true, + taskMap: new Map([[task.id, task]]), + memberSpawnStatuses: new Map([['bob', provisionedButNotAliveSpawnStatus()]]), + memberRuntimeEntries: new Map([ + [ + 'bob', + { + memberName: 'bob', + alive: false, + restartable: true, + livenessKind: 'confirmed_bootstrap', + runtimeDiagnostic: 'Runtime process crashed', + runtimeDiagnosticSeverity: 'error', + updatedAt: '2026-05-25T20:14:05.411Z', + }, + ], + ]), + }) + ); + await Promise.resolve(); + }); + + expect(host.querySelector('[data-testid="current-bob"]')).toBeNull(); + expect(host.querySelector('[data-testid="retry-bob"]')).toBeNull(); + expect(host.querySelector('[data-testid="skip-bob"]')).toBeNull(); + expect(host.textContent).toContain('team provisioned but not alive'); + + await act(async () => { + root.unmount(); + await Promise.resolve(); + }); + }); + it('passes skip callbacks to failed member cards and rerenders when the callback changes', async () => { vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true); const host = document.createElement('div'); diff --git a/test/renderer/components/team/provisioningSteps.test.ts b/test/renderer/components/team/provisioningSteps.test.ts index 78d8158c..ed26b347 100644 --- a/test/renderer/components/team/provisioningSteps.test.ts +++ b/test/renderer/components/team/provisioningSteps.test.ts @@ -144,6 +144,208 @@ describe('getLaunchJoinMilestonesFromMembers', () => { expect(milestones.pendingSpawnCount).toBe(3); }); + it('counts bootstrap-confirmed provisioned-but-not-alive entries as joined', () => { + const milestones = getLaunchJoinMilestonesFromMembers({ + members: [{ name: 'tom' }], + memberSpawnStatuses: { + tom: { + status: 'error', + launchState: 'failed_to_start', + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: 'CLI process exited (code 1) \u2014 team provisioned but not alive', + livenessKind: 'confirmed_bootstrap', + runtimeDiagnostic: + 'runtime pid could not be verified because process table is unavailable', + runtimeDiagnosticSeverity: 'warning', + updatedAt: '2026-05-25T20:14:02.147Z', + }, + }, + memberRuntimeEntries: { + tom: { + memberName: 'tom', + alive: false, + restartable: true, + livenessKind: 'registered_only', + runtimeDiagnostic: + 'runtime pid could not be verified because process table is unavailable', + runtimeDiagnosticSeverity: 'warning', + updatedAt: '2026-05-25T20:14:03.317Z', + }, + }, + }); + + expect(milestones.heartbeatConfirmedCount).toBe(1); + expect(milestones.failedSpawnCount).toBe(0); + expect(milestones.pendingSpawnCount).toBe(0); + }); + + it('uses spawn process-table proof when runtime registered metadata has no diagnostic text', () => { + const milestones = getLaunchJoinMilestonesFromMembers({ + members: [{ name: 'tom' }], + memberSpawnStatuses: { + tom: { + status: 'error', + launchState: 'failed_to_start', + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: + 'CLI process exited (code 1) - team provisioned but not alive; process table unavailable', + livenessKind: 'confirmed_bootstrap', + runtimeDiagnosticSeverity: 'warning', + updatedAt: '2026-05-25T20:14:02.147Z', + }, + }, + memberRuntimeEntries: { + tom: { + memberName: 'tom', + alive: false, + restartable: true, + livenessKind: 'registered_only', + runtimeDiagnosticSeverity: 'warning', + updatedAt: '2026-05-25T20:14:03.317Z', + }, + }, + }); + + expect(milestones.heartbeatConfirmedCount).toBe(1); + expect(milestones.failedSpawnCount).toBe(0); + expect(milestones.pendingSpawnCount).toBe(0); + }); + + it('uses spawn process-table proof when runtime metadata has no liveness or diagnostic text', () => { + const milestones = getLaunchJoinMilestonesFromMembers({ + members: [{ name: 'tom' }], + memberSpawnStatuses: { + tom: { + status: 'error', + launchState: 'failed_to_start', + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: + 'CLI process exited (code 1) - team provisioned but not alive; process table unavailable', + livenessKind: 'confirmed_bootstrap', + runtimeDiagnosticSeverity: 'warning', + updatedAt: '2026-05-25T20:14:02.147Z', + }, + }, + memberRuntimeEntries: { + tom: { + memberName: 'tom', + alive: false, + restartable: true, + runtimeDiagnosticSeverity: 'warning', + updatedAt: '2026-05-25T20:14:03.317Z', + }, + }, + }); + + expect(milestones.heartbeatConfirmedCount).toBe(1); + expect(milestones.failedSpawnCount).toBe(0); + expect(milestones.pendingSpawnCount).toBe(0); + }); + + it('counts unsafe bootstrap-confirmed provisioned-but-not-alive entries as failed', () => { + const milestones = getLaunchJoinMilestonesFromMembers({ + members: [{ name: 'tom' }], + memberSpawnStatuses: { + tom: { + status: 'error', + launchState: 'failed_to_start', + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: 'CLI process exited (code 1) - team provisioned but not alive', + livenessKind: 'not_found', + runtimeDiagnostic: 'Runtime is no longer registered', + runtimeDiagnosticSeverity: 'warning', + updatedAt: '2026-05-25T20:14:02.147Z', + }, + }, + }); + + expect(milestones.heartbeatConfirmedCount).toBe(0); + expect(milestones.failedSpawnCount).toBe(1); + expect(milestones.pendingSpawnCount).toBe(0); + }); + + it('keeps ambiguous runtime-offline entries pending even when provisioned-but-not-alive spawn evidence is safe', () => { + const milestones = getLaunchJoinMilestonesFromMembers({ + members: [{ name: 'tom' }], + memberSpawnStatuses: { + tom: { + status: 'error', + launchState: 'failed_to_start', + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: + 'CLI process exited (code 1) - team provisioned but not alive; process table unavailable', + livenessKind: 'registered_only', + runtimeDiagnostic: + 'runtime pid could not be verified because process table is unavailable', + runtimeDiagnosticSeverity: 'warning', + updatedAt: '2026-05-25T20:14:02.147Z', + }, + }, + memberRuntimeEntries: { + tom: { + memberName: 'tom', + alive: false, + restartable: true, + runtimeDiagnostic: 'Runtime heartbeat is not alive', + runtimeDiagnosticSeverity: 'warning', + updatedAt: '2026-05-25T20:14:03.317Z', + }, + }, + }); + + expect(milestones.heartbeatConfirmedCount).toBe(0); + expect(milestones.failedSpawnCount).toBe(0); + expect(milestones.pendingSpawnCount).toBe(1); + }); + + it('does not count safe provisioned-but-not-alive spawn evidence as joined when live runtime evidence is an error', () => { + const milestones = getLaunchJoinMilestonesFromMembers({ + members: [{ name: 'tom' }], + memberSpawnStatuses: { + tom: { + status: 'error', + launchState: 'failed_to_start', + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: + 'CLI process exited (code 1) - team provisioned but not alive; process table unavailable', + livenessKind: 'confirmed_bootstrap', + runtimeDiagnostic: + 'runtime pid could not be verified because process table is unavailable', + runtimeDiagnosticSeverity: 'warning', + updatedAt: '2026-05-25T20:14:02.147Z', + }, + }, + memberRuntimeEntries: { + tom: { + memberName: 'tom', + alive: false, + restartable: true, + livenessKind: 'registered_only', + runtimeDiagnostic: + 'runtime pid could not be verified because process table is unavailable', + runtimeDiagnosticSeverity: 'error', + updatedAt: '2026-05-25T20:14:03.317Z', + }, + }, + }); + + expect(milestones.heartbeatConfirmedCount).toBe(0); + expect(milestones.failedSpawnCount).toBe(0); + expect(milestones.pendingSpawnCount).toBe(1); + }); + it('does not let a stale clean snapshot hide live registered-only members', () => { const milestones = getLaunchJoinMilestonesFromMembers({ members, @@ -243,4 +445,131 @@ describe('getLaunchJoinMilestonesFromMembers', () => { expect(milestones.pendingSpawnCount).toBe(1); expect(milestones.expectedTeammateCount).toBe(4); }); + + it('does not count confirmed spawn as joined when spawn metadata carries runtime error evidence', () => { + const milestones = getLaunchJoinMilestonesFromMembers({ + members, + memberSpawnStatuses: { + alice: { + status: 'online', + launchState: 'confirmed_alive', + runtimeAlive: true, + bootstrapConfirmed: true, + updatedAt: '2026-04-24T12:00:00.000Z', + }, + bob: { + status: 'online', + launchState: 'confirmed_alive', + runtimeAlive: false, + bootstrapConfirmed: true, + livenessKind: 'not_found', + runtimeDiagnostic: 'Runtime process crashed', + runtimeDiagnosticSeverity: 'error', + updatedAt: '2026-04-24T12:00:01.000Z', + }, + tom: { + status: 'online', + launchState: 'confirmed_alive', + runtimeAlive: true, + bootstrapConfirmed: true, + updatedAt: '2026-04-24T12:00:00.000Z', + }, + jane: { + status: 'online', + launchState: 'confirmed_alive', + runtimeAlive: true, + bootstrapConfirmed: true, + updatedAt: '2026-04-24T12:00:00.000Z', + }, + }, + }); + + expect(milestones.heartbeatConfirmedCount).toBe(3); + expect(milestones.pendingSpawnCount).toBe(1); + expect(milestones.expectedTeammateCount).toBe(4); + }); + + it('does not count confirmed spawn as joined when stopped spawn metadata has no liveness kind', () => { + const milestones = getLaunchJoinMilestonesFromMembers({ + members, + memberSpawnStatuses: { + alice: { + status: 'online', + launchState: 'confirmed_alive', + runtimeAlive: true, + bootstrapConfirmed: true, + updatedAt: '2026-04-24T12:00:00.000Z', + }, + bob: { + status: 'online', + launchState: 'confirmed_alive', + runtimeAlive: false, + bootstrapConfirmed: true, + updatedAt: '2026-04-24T12:00:01.000Z', + }, + tom: { + status: 'online', + launchState: 'confirmed_alive', + runtimeAlive: true, + bootstrapConfirmed: true, + updatedAt: '2026-04-24T12:00:00.000Z', + }, + jane: { + status: 'online', + launchState: 'confirmed_alive', + runtimeAlive: true, + bootstrapConfirmed: true, + updatedAt: '2026-04-24T12:00:00.000Z', + }, + }, + }); + + expect(milestones.heartbeatConfirmedCount).toBe(3); + expect(milestones.pendingSpawnCount).toBe(1); + expect(milestones.expectedTeammateCount).toBe(4); + }); + + it('counts process-table-unavailable provisioned-but-not-alive spawn without liveness kind as joined', () => { + const milestones = getLaunchJoinMilestonesFromMembers({ + members, + memberSpawnStatuses: { + alice: { + status: 'online', + launchState: 'confirmed_alive', + runtimeAlive: true, + bootstrapConfirmed: true, + updatedAt: '2026-04-24T12:00:00.000Z', + }, + bob: { + status: 'error', + launchState: 'failed_to_start', + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: + 'CLI process exited (code 1) - team provisioned but not alive; process table unavailable', + updatedAt: '2026-04-24T12:00:01.000Z', + }, + tom: { + status: 'online', + launchState: 'confirmed_alive', + runtimeAlive: true, + bootstrapConfirmed: true, + updatedAt: '2026-04-24T12:00:00.000Z', + }, + jane: { + status: 'online', + launchState: 'confirmed_alive', + runtimeAlive: true, + bootstrapConfirmed: true, + updatedAt: '2026-04-24T12:00:00.000Z', + }, + }, + }); + + expect(milestones.heartbeatConfirmedCount).toBe(4); + expect(milestones.pendingSpawnCount).toBe(0); + expect(milestones.failedSpawnCount).toBe(0); + expect(milestones.expectedTeammateCount).toBe(4); + }); }); diff --git a/test/renderer/components/team/teamRuntimeDisplayRows.test.ts b/test/renderer/components/team/teamRuntimeDisplayRows.test.ts index 8ac8c479..a4c564bf 100644 --- a/test/renderer/components/team/teamRuntimeDisplayRows.test.ts +++ b/test/renderer/components/team/teamRuntimeDisplayRows.test.ts @@ -30,7 +30,9 @@ function createRuntimeSnapshot( }; } -function createSpawnStatus(overrides: Partial = {}): MemberSpawnStatusEntry { +function createSpawnStatus( + overrides: Partial = {} +): MemberSpawnStatusEntry { return { status: 'spawning', launchState: 'starting', @@ -251,6 +253,274 @@ describe('buildTeamRuntimeDisplayRows', () => { }); }); + it('does not degrade bootstrap-confirmed provisioned-but-not-alive rows', () => { + const rows = buildTeamRuntimeDisplayRows({ + members: [{ name: 'alice' }], + runtimeSnapshot: createRuntimeSnapshot({ + alice: createRuntimeEntry({ + alive: false, + livenessKind: 'confirmed_bootstrap', + runtimeDiagnostic: + 'runtime pid could not be verified because process table is unavailable', + runtimeDiagnosticSeverity: 'warning', + }), + }), + spawnStatuses: { + alice: createSpawnStatus({ + status: 'error', + launchState: 'failed_to_start', + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: 'CLI process exited (code 1) \u2014 team provisioned but not alive', + livenessKind: 'confirmed_bootstrap', + }), + }, + }); + + expect(rows[0]).toMatchObject({ + memberName: 'alice', + state: 'running', + source: 'mixed', + stateReason: 'Bootstrap confirmed', + diagnosticSeverity: 'warning', + actionsAllowed: false, + }); + }); + + it('does not degrade Windows process-table-unavailable registered metadata rows', () => { + const rows = buildTeamRuntimeDisplayRows({ + members: [{ name: 'alice' }], + runtimeSnapshot: createRuntimeSnapshot({ + alice: createRuntimeEntry({ + alive: false, + livenessKind: 'registered_only', + runtimeDiagnostic: + 'runtime pid could not be verified because process table is unavailable', + runtimeDiagnosticSeverity: 'warning', + }), + }), + spawnStatuses: { + alice: createSpawnStatus({ + status: 'error', + launchState: 'failed_to_start', + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: + 'CLI process exited (code 1) - team provisioned but not alive; process table unavailable', + livenessKind: 'registered_only', + runtimeDiagnostic: + 'runtime pid could not be verified because process table is unavailable', + runtimeDiagnosticSeverity: 'warning', + }), + }, + }); + + expect(rows[0]).toMatchObject({ + memberName: 'alice', + state: 'running', + source: 'mixed', + stateReason: 'Bootstrap confirmed', + diagnosticSeverity: 'warning', + actionsAllowed: false, + }); + }); + + it('uses spawn process-table proof when runtime registered metadata has no diagnostic text', () => { + const rows = buildTeamRuntimeDisplayRows({ + members: [{ name: 'alice' }], + runtimeSnapshot: createRuntimeSnapshot({ + alice: createRuntimeEntry({ + alive: false, + livenessKind: 'registered_only', + runtimeDiagnosticSeverity: 'warning', + }), + }), + spawnStatuses: { + alice: createSpawnStatus({ + status: 'error', + launchState: 'failed_to_start', + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: + 'CLI process exited (code 1) - team provisioned but not alive; process table unavailable', + livenessKind: 'confirmed_bootstrap', + }), + }, + }); + + expect(rows[0]).toMatchObject({ + memberName: 'alice', + state: 'running', + source: 'mixed', + stateReason: 'Bootstrap confirmed', + diagnosticSeverity: 'warning', + actionsAllowed: false, + }); + }); + + it('does not let stale provisioned-but-not-alive spawn evidence hide runtime errors', () => { + const rows = buildTeamRuntimeDisplayRows({ + members: [{ name: 'alice' }], + runtimeSnapshot: createRuntimeSnapshot({ + alice: createRuntimeEntry({ + alive: false, + livenessKind: 'confirmed_bootstrap', + runtimeDiagnostic: 'Runtime process crashed', + runtimeDiagnosticSeverity: 'error', + }), + }), + spawnStatuses: { + alice: createSpawnStatus({ + status: 'error', + launchState: 'failed_to_start', + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: 'CLI process exited (code 1) - team provisioned but not alive', + livenessKind: 'confirmed_bootstrap', + }), + }, + }); + + expect(rows[0]).toMatchObject({ + memberName: 'alice', + state: 'degraded', + source: 'mixed', + stateReason: 'Runtime process crashed', + diagnosticSeverity: 'error', + actionsAllowed: false, + }); + }); + + it('does not let provisioned-but-not-alive spawn evidence hide stopped runtime evidence', () => { + const rows = buildTeamRuntimeDisplayRows({ + members: [{ name: 'alice' }], + runtimeSnapshot: createRuntimeSnapshot({ + alice: createRuntimeEntry({ + alive: false, + livenessKind: 'not_found', + runtimeDiagnostic: 'Runtime metadata was not found', + runtimeDiagnosticSeverity: 'warning', + }), + }), + spawnStatuses: { + alice: createSpawnStatus({ + status: 'error', + launchState: 'failed_to_start', + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: 'CLI process exited (code 1) - team provisioned but not alive', + livenessKind: 'confirmed_bootstrap', + }), + }, + }); + + expect(rows[0]).toMatchObject({ + memberName: 'alice', + state: 'stopped', + source: 'mixed', + stateReason: 'Runtime metadata was not found', + diagnosticSeverity: 'warning', + actionsAllowed: false, + }); + }); + + it('does not let stopped provisioned-but-not-alive spawn evidence hide live runtime context', () => { + const rows = buildTeamRuntimeDisplayRows({ + members: [{ name: 'alice' }], + runtimeSnapshot: createRuntimeSnapshot({ + alice: createRuntimeEntry({ + alive: true, + livenessKind: 'runtime_process', + runtimeDiagnostic: 'Runtime process is alive', + runtimeDiagnosticSeverity: 'info', + }), + }), + spawnStatuses: { + alice: createSpawnStatus({ + status: 'error', + launchState: 'failed_to_start', + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: 'CLI process exited (code 1) - team provisioned but not alive', + livenessKind: 'not_found', + runtimeDiagnostic: 'Runtime is no longer registered', + runtimeDiagnosticSeverity: 'warning', + }), + }, + }); + + expect(rows[0]).toMatchObject({ + memberName: 'alice', + state: 'degraded', + source: 'mixed', + stateReason: 'Runtime is no longer registered. Process is still alive.', + diagnosticSeverity: 'warning', + actionsAllowed: false, + }); + }); + + it('keeps spawn-only runtime errors visible for provisioned-but-not-alive entries', () => { + const rows = buildTeamRuntimeDisplayRows({ + members: [{ name: 'alice' }], + spawnStatuses: { + alice: createSpawnStatus({ + status: 'error', + launchState: 'failed_to_start', + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: 'CLI process exited (code 1) - team provisioned but not alive', + livenessKind: 'confirmed_bootstrap', + runtimeDiagnostic: 'Runtime process crashed', + runtimeDiagnosticSeverity: 'error', + }), + }, + }); + + expect(rows[0]).toMatchObject({ + memberName: 'alice', + state: 'degraded', + source: 'spawn-status', + stateReason: 'Runtime process crashed', + diagnosticSeverity: 'error', + actionsAllowed: false, + }); + }); + + it('keeps spawn-only stopped liveness visible for provisioned-but-not-alive entries', () => { + const rows = buildTeamRuntimeDisplayRows({ + members: [{ name: 'alice' }], + spawnStatuses: { + alice: createSpawnStatus({ + status: 'error', + launchState: 'failed_to_start', + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: 'CLI process exited (code 1) - team provisioned but not alive', + livenessKind: 'not_found', + runtimeDiagnostic: 'Runtime is no longer registered', + runtimeDiagnosticSeverity: 'warning', + }), + }, + }); + + expect(rows[0]).toMatchObject({ + memberName: 'alice', + state: 'degraded', + source: 'spawn-status', + stateReason: 'Runtime is no longer registered', + diagnosticSeverity: 'warning', + actionsAllowed: false, + }); + }); + it('degrades spawn-only rows when online process evidence has stalled bootstrap', () => { const rows = buildTeamRuntimeDisplayRows({ members: [{ name: 'alice' }], diff --git a/test/renderer/features/agent-graph/TeamGraphAdapter.test.ts b/test/renderer/features/agent-graph/TeamGraphAdapter.test.ts index 1d3d1977..99fd1cf8 100644 --- a/test/renderer/features/agent-graph/TeamGraphAdapter.test.ts +++ b/test/renderer/features/agent-graph/TeamGraphAdapter.test.ts @@ -1777,6 +1777,154 @@ describe('TeamGraphAdapter particles', () => { }); }); + it('keeps bootstrap-confirmed spawn diagnostic errors in graph error state', () => { + const adapter = TeamGraphAdapter.create(); + const graph = adapter.adapt(createBaseTeamData(), 'my-team', { + alice: { + status: 'error', + launchState: 'failed_to_start', + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: 'CLI process exited (code 1) - team provisioned but not alive', + livenessKind: 'confirmed_bootstrap', + runtimeDiagnostic: 'Runtime process crashed', + runtimeDiagnosticSeverity: 'error', + updatedAt: '2026-05-25T20:14:02.147Z', + }, + }); + + expect(findNode(graph, 'member:my-team:alice')).toMatchObject({ + state: 'error', + spawnStatus: 'error', + launchVisualState: 'error', + launchStatusLabel: 'failed', + exceptionTone: 'error', + exceptionLabel: 'spawn failed', + }); + }); + + it('keeps bootstrap-confirmed stopped runtime evidence in graph error state', () => { + const adapter = TeamGraphAdapter.create(); + const graph = adapter.adapt(createBaseTeamData(), 'my-team', { + alice: { + status: 'error', + launchState: 'failed_to_start', + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: 'CLI process exited (code 1) - team provisioned but not alive', + livenessKind: 'not_found', + runtimeDiagnostic: 'Runtime is no longer registered', + runtimeDiagnosticSeverity: 'warning', + updatedAt: '2026-05-25T20:14:02.147Z', + }, + }); + + expect(findNode(graph, 'member:my-team:alice')).toMatchObject({ + state: 'error', + spawnStatus: 'error', + launchVisualState: 'error', + launchStatusLabel: 'failed', + exceptionTone: 'error', + exceptionLabel: 'spawn failed', + }); + }); + + it('uses spawn process-table proof when graph runtime metadata has no diagnostic text', () => { + const adapter = TeamGraphAdapter.create(); + const graph = adapter.adapt( + createBaseTeamData({ + runtimeEntriesByMember: { + alice: createLiveRuntimeEntry('alice', { + alive: false, + livenessKind: 'registered_only', + runtimeDiagnosticSeverity: 'warning', + }), + }, + }), + 'my-team', + { + alice: { + status: 'error', + launchState: 'failed_to_start', + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: + 'CLI process exited (code 1) - team provisioned but not alive; process table unavailable', + livenessKind: 'confirmed_bootstrap', + runtimeDiagnosticSeverity: 'warning', + updatedAt: '2026-05-25T20:14:02.147Z', + }, + } + ); + + expect(findNode(graph, 'member:my-team:alice')).toMatchObject({ + state: 'active', + spawnStatus: 'error', + launchVisualState: undefined, + launchStatusLabel: undefined, + exceptionTone: undefined, + exceptionLabel: undefined, + }); + }); + + it.each([ + { + name: 'runtime diagnostic error', + runtime: { + alive: false, + livenessKind: 'runtime_process', + runtimeDiagnostic: 'Runtime process crashed', + runtimeDiagnosticSeverity: 'error', + }, + }, + { + name: 'stopped runtime liveness', + runtime: { + alive: false, + livenessKind: 'not_found', + runtimeDiagnostic: 'Runtime is no longer registered', + runtimeDiagnosticSeverity: 'warning', + }, + }, + ] as const)( + 'keeps graph errors when live runtime has unsafe evidence for a safe bootstrap-confirmed spawn: $name', + ({ runtime }) => { + const adapter = TeamGraphAdapter.create(); + const graph = adapter.adapt( + createBaseTeamData({ + runtimeEntriesByMember: { + alice: createLiveRuntimeEntry('alice', runtime), + }, + }), + 'my-team', + { + alice: { + status: 'error', + launchState: 'failed_to_start', + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: 'CLI process exited (code 1) - team provisioned but not alive', + livenessKind: 'confirmed_bootstrap', + runtimeDiagnostic: 'runtime pid could not be verified because process table is unavailable', + runtimeDiagnosticSeverity: 'warning', + updatedAt: '2026-05-25T20:14:02.147Z', + }, + } + ); + + expect(findNode(graph, 'member:my-team:alice')).toMatchObject({ + state: 'error', + spawnStatus: 'error', + exceptionTone: 'error', + exceptionLabel: 'spawn failed', + }); + } + ); + it('treats permission-blocked spawn state as awaiting approval even without pending approval feed', () => { const adapter = TeamGraphAdapter.create(); const teamData = createBaseTeamData(); diff --git a/test/renderer/store/teamSlice.test.ts b/test/renderer/store/teamSlice.test.ts index 7351801c..ffd422fe 100644 --- a/test/renderer/store/teamSlice.test.ts +++ b/test/renderer/store/teamSlice.test.ts @@ -419,6 +419,8 @@ describe('teamSlice actions', () => { expect(fetchTeams).toHaveBeenCalledTimes(1); expect(refreshTeamData).toHaveBeenCalledTimes(1); expect(refreshTeamData).toHaveBeenCalledWith('my-team', { withDedup: true }); + expect(hoisted.getMemberSpawnStatuses).toHaveBeenCalledWith('my-team'); + expect(hoisted.getTeamAgentRuntime).toHaveBeenCalledWith('my-team'); const snapshot = getTeamRefreshFanoutSnapshotForTests( 'my-team' @@ -431,6 +433,16 @@ describe('teamSlice actions', () => { 'provisioning-progress:provisioning:terminal-ready:refreshTeamData:scheduled' ] ).toBe(1); + expect( + snapshot?.counts[ + 'provisioning-progress:provisioning:terminal-ready:fetchMemberSpawnStatuses:scheduled' + ] + ).toBe(1); + expect( + snapshot?.counts[ + 'provisioning-progress:provisioning:terminal-ready:fetchTeamAgentRuntime:scheduled' + ] + ).toBe(1); }); it('maps inbox verify failure to user-friendly text', async () => { @@ -6396,6 +6408,84 @@ describe('teamSlice actions', () => { ); }); + it('refreshes retained terminal spawn errors after disconnected progress', async () => { + const store = createSliceStore(); + const startedAt = '2026-03-12T10:00:00.000Z'; + const staleReason = 'CLI process exited (code 1) \u2014 team provisioned but not alive'; + store.setState({ + selectedTeamName: 'my-team', + selectedTeamData: createTeamSnapshot(), + paneLayout: { + focusedPaneId: 'pane-default', + panes: [ + { + id: 'pane-default', + widthFraction: 1, + tabs: [{ id: 'team-my-team', type: 'team', teamName: 'my-team', label: 'My Team' }], + activeTabId: 'team-my-team', + }, + ], + }, + currentProvisioningRunIdByTeam: { + 'my-team': 'run-current', + }, + currentRuntimeRunIdByTeam: { + 'my-team': 'run-current', + }, + memberSpawnStatusesByTeam: { + 'my-team': { + tom: createMemberSpawnStatus({ + status: 'error', + launchState: 'failed_to_start', + error: staleReason, + hardFailure: true, + hardFailureReason: staleReason, + bootstrapConfirmed: true, + runtimeAlive: false, + }), + }, + }, + }); + hoisted.getMemberSpawnStatuses.mockResolvedValue( + createMemberSpawnSnapshot({ + runId: 'run-current', + expectedMembers: ['tom'], + statuses: { + tom: createMemberSpawnStatus({ + status: 'online', + launchState: 'confirmed_alive', + runtimeAlive: false, + livenessKind: 'confirmed_bootstrap', + hardFailure: false, + hardFailureReason: undefined, + error: undefined, + }), + }, + }) + ); + + store.getState().onProvisioningProgress({ + runId: 'run-current', + teamName: 'my-team', + state: 'disconnected', + message: 'Disconnected', + startedAt, + updatedAt: '2026-03-12T10:00:01.000Z', + }); + + await vi.waitFor(() => { + expect(store.getState().memberSpawnStatusesByTeam['my-team']?.tom).toMatchObject({ + status: 'online', + launchState: 'confirmed_alive', + bootstrapConfirmed: true, + hardFailure: false, + }); + }); + expect( + store.getState().memberSpawnStatusesByTeam['my-team']?.tom?.hardFailureReason + ).toBeUndefined(); + }); + it('does not fall back to a team-wide latest run when no current run is pinned', () => { expect( getCurrentProvisioningProgressForTeam( diff --git a/test/renderer/utils/memberHelpers.test.ts b/test/renderer/utils/memberHelpers.test.ts index 9c559fa5..b717d5f0 100644 --- a/test/renderer/utils/memberHelpers.test.ts +++ b/test/renderer/utils/memberHelpers.test.ts @@ -11,7 +11,11 @@ import { shouldDisplayMemberCurrentTask, } from '@renderer/utils/memberHelpers'; -import type { ResolvedTeamMember } from '@shared/types'; +import type { + MemberSpawnStatusEntry, + ResolvedTeamMember, + TeamAgentRuntimeEntry, +} from '@shared/types'; const member: ResolvedTeamMember = { name: 'alice', @@ -27,6 +31,28 @@ const member: ResolvedTeamMember = { removedAt: undefined, }; +const provisionedButNotAliveSpawn: MemberSpawnStatusEntry = { + status: 'error', + launchState: 'failed_to_start', + updatedAt: '2026-05-25T20:14:02.147Z', + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: 'CLI process exited (code 1) \u2014 team provisioned but not alive', + livenessKind: 'confirmed_bootstrap', +}; + +const processTableUnavailableRuntime: TeamAgentRuntimeEntry = { + memberName: 'alice', + alive: false, + restartable: true, + providerId: 'anthropic', + livenessKind: 'confirmed_bootstrap', + runtimeDiagnostic: 'runtime pid could not be verified because process table is unavailable', + runtimeDiagnosticSeverity: 'warning', + updatedAt: '2026-05-25T20:14:05.411Z', +}; + describe('memberHelpers spawn-aware presence', () => { it('does not display current task labels for offline or terminal launch states', () => { expect( @@ -121,6 +147,92 @@ describe('memberHelpers spawn-aware presence', () => { ).toBe(true); }); + it('treats bootstrap-confirmed provisioned-but-not-alive entries as active for task display', () => { + expect( + shouldDisplayMemberCurrentTask({ + member: { ...member, currentTaskId: 'task-1' }, + isTeamAlive: true, + spawnStatus: provisionedButNotAliveSpawn.status, + spawnLaunchState: provisionedButNotAliveSpawn.launchState, + spawnRuntimeAlive: provisionedButNotAliveSpawn.runtimeAlive, + spawnEntry: provisionedButNotAliveSpawn, + runtimeEntry: processTableUnavailableRuntime, + }) + ).toBe(true); + }); + + it('treats spawn-only bootstrap-confirmed provisioned-but-not-alive entries as active for task display', () => { + expect( + shouldDisplayMemberCurrentTask({ + member: { ...member, currentTaskId: 'task-1' }, + isTeamAlive: true, + spawnStatus: provisionedButNotAliveSpawn.status, + spawnLaunchState: provisionedButNotAliveSpawn.launchState, + spawnRuntimeAlive: provisionedButNotAliveSpawn.runtimeAlive, + spawnEntry: provisionedButNotAliveSpawn, + }) + ).toBe(true); + }); + + it('does not show task activity for provisioned-but-not-alive entries with runtime errors', () => { + expect( + shouldDisplayMemberCurrentTask({ + member: { ...member, currentTaskId: 'task-1' }, + isTeamAlive: true, + spawnStatus: provisionedButNotAliveSpawn.status, + spawnLaunchState: provisionedButNotAliveSpawn.launchState, + spawnRuntimeAlive: provisionedButNotAliveSpawn.runtimeAlive, + spawnEntry: provisionedButNotAliveSpawn, + runtimeEntry: { + ...processTableUnavailableRuntime, + runtimeDiagnostic: 'Runtime process crashed', + runtimeDiagnosticSeverity: 'error', + }, + }) + ).toBe(false); + + expect( + shouldDisplayMemberCurrentTask({ + member: { ...member, currentTaskId: 'task-1' }, + isTeamAlive: true, + spawnStatus: provisionedButNotAliveSpawn.status, + spawnLaunchState: provisionedButNotAliveSpawn.launchState, + spawnRuntimeAlive: provisionedButNotAliveSpawn.runtimeAlive, + spawnEntry: { + ...provisionedButNotAliveSpawn, + runtimeDiagnostic: 'Runtime process crashed', + runtimeDiagnosticSeverity: 'error', + }, + runtimeEntry: processTableUnavailableRuntime, + }) + ).toBe(false); + }); + + it('does not show task activity for unsafe provisioned-but-not-alive runtime candidates', () => { + expect( + shouldDisplayMemberCurrentTask({ + member: { ...member, currentTaskId: 'task-1' }, + isTeamAlive: true, + spawnStatus: 'online', + spawnLaunchState: 'confirmed_alive', + spawnRuntimeAlive: true, + spawnEntry: { + ...provisionedButNotAliveSpawn, + status: 'online', + launchState: 'confirmed_alive', + runtimeAlive: true, + }, + runtimeEntry: { + ...processTableUnavailableRuntime, + alive: false, + livenessKind: 'runtime_process_candidate', + runtimeDiagnostic: + 'OpenCode runtime process detected, but teammate bootstrap is not confirmed', + }, + }) + ).toBe(false); + }); + it('shows process-online teammates as online with a green dot', () => { expect( getSpawnAwarePresenceLabel( @@ -657,6 +769,44 @@ describe('memberHelpers spawn-aware presence', () => { ).toBe(true); }); + it('marks unsafe provisioned-but-not-alive OpenCode entries as relaunchable', () => { + expect( + isOpenCodeRelaunchActionable({ + member: { ...member, providerId: 'opencode' }, + spawnEntry: { + status: 'error', + launchState: 'failed_to_start', + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: 'CLI process exited (code 1) - team provisioned but not alive', + livenessKind: 'not_found', + runtimeDiagnostic: 'Runtime is no longer registered', + runtimeDiagnosticSeverity: 'warning', + updatedAt: '2026-05-25T20:14:02.147Z', + }, + }) + ).toBe(true); + + expect( + isOpenCodeRelaunchActionable({ + member: { ...member, providerId: 'opencode' }, + spawnEntry: { + status: 'error', + launchState: 'failed_to_start', + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: 'CLI process exited (code 1) - team provisioned but not alive', + livenessKind: 'confirmed_bootstrap', + runtimeDiagnostic: 'Runtime process crashed', + runtimeDiagnosticSeverity: 'error', + updatedAt: '2026-05-25T20:14:02.147Z', + }, + }) + ).toBe(true); + }); + it('does not mark fresh OpenCode runtime candidates as relaunchable', () => { expect( isOpenCodeRelaunchActionable({ @@ -780,6 +930,214 @@ describe('memberHelpers spawn-aware presence', () => { }); }); + it('does not render bootstrap-confirmed provisioned-but-not-alive entries as failed or stale', () => { + expect( + buildMemberLaunchPresentation({ + member, + spawnStatus: provisionedButNotAliveSpawn.status, + spawnLaunchState: provisionedButNotAliveSpawn.launchState, + spawnLivenessSource: provisionedButNotAliveSpawn.livenessSource, + spawnRuntimeAlive: provisionedButNotAliveSpawn.runtimeAlive, + spawnBootstrapConfirmed: provisionedButNotAliveSpawn.bootstrapConfirmed, + spawnBootstrapStalled: provisionedButNotAliveSpawn.bootstrapStalled, + spawnAgentToolAccepted: provisionedButNotAliveSpawn.agentToolAccepted, + spawnHardFailure: provisionedButNotAliveSpawn.hardFailure, + spawnHardFailureReason: provisionedButNotAliveSpawn.hardFailureReason, + spawnError: provisionedButNotAliveSpawn.error, + spawnLivenessKind: provisionedButNotAliveSpawn.livenessKind, + runtimeEntry: processTableUnavailableRuntime, + runtimeAdvisory: undefined, + isLaunchSettling: false, + isTeamAlive: true, + isTeamProvisioning: false, + }) + ).toMatchObject({ + presenceLabel: 'idle', + launchVisualState: null, + launchStatusLabel: null, + spawnBadgeLabel: null, + }); + }); + + it('does not render spawn-only bootstrap-confirmed provisioned-but-not-alive entries as failed or stale', () => { + expect( + buildMemberLaunchPresentation({ + member, + spawnStatus: provisionedButNotAliveSpawn.status, + spawnLaunchState: provisionedButNotAliveSpawn.launchState, + spawnLivenessSource: provisionedButNotAliveSpawn.livenessSource, + spawnRuntimeAlive: provisionedButNotAliveSpawn.runtimeAlive, + spawnBootstrapConfirmed: provisionedButNotAliveSpawn.bootstrapConfirmed, + spawnBootstrapStalled: provisionedButNotAliveSpawn.bootstrapStalled, + spawnAgentToolAccepted: provisionedButNotAliveSpawn.agentToolAccepted, + spawnHardFailure: provisionedButNotAliveSpawn.hardFailure, + spawnHardFailureReason: provisionedButNotAliveSpawn.hardFailureReason, + spawnError: provisionedButNotAliveSpawn.error, + spawnLivenessKind: provisionedButNotAliveSpawn.livenessKind, + runtimeAdvisory: undefined, + isLaunchSettling: false, + isTeamAlive: true, + isTeamProvisioning: false, + }) + ).toMatchObject({ + presenceLabel: 'idle', + launchVisualState: null, + launchStatusLabel: null, + spawnBadgeLabel: null, + }); + }); + + it('does not leak safe process-table liveness into healed member visuals', () => { + expect( + buildMemberLaunchPresentation({ + member, + spawnStatus: provisionedButNotAliveSpawn.status, + spawnLaunchState: provisionedButNotAliveSpawn.launchState, + spawnLivenessSource: provisionedButNotAliveSpawn.livenessSource, + spawnRuntimeAlive: provisionedButNotAliveSpawn.runtimeAlive, + spawnBootstrapConfirmed: provisionedButNotAliveSpawn.bootstrapConfirmed, + spawnBootstrapStalled: provisionedButNotAliveSpawn.bootstrapStalled, + spawnAgentToolAccepted: provisionedButNotAliveSpawn.agentToolAccepted, + spawnHardFailure: provisionedButNotAliveSpawn.hardFailure, + spawnHardFailureReason: provisionedButNotAliveSpawn.hardFailureReason, + spawnError: provisionedButNotAliveSpawn.error, + spawnLivenessKind: provisionedButNotAliveSpawn.livenessKind, + runtimeEntry: { + ...processTableUnavailableRuntime, + livenessKind: 'registered_only', + }, + runtimeAdvisory: undefined, + isLaunchSettling: false, + isTeamAlive: true, + isTeamProvisioning: false, + }) + ).toMatchObject({ + presenceLabel: 'idle', + launchVisualState: null, + launchStatusLabel: null, + spawnBadgeLabel: null, + }); + }); + + it('recognizes provisioned-but-not-alive when the reason is only in runtime diagnostics', () => { + expect( + buildMemberLaunchPresentation({ + member, + spawnStatus: provisionedButNotAliveSpawn.status, + spawnLaunchState: provisionedButNotAliveSpawn.launchState, + spawnLivenessSource: provisionedButNotAliveSpawn.livenessSource, + spawnRuntimeAlive: provisionedButNotAliveSpawn.runtimeAlive, + spawnBootstrapConfirmed: provisionedButNotAliveSpawn.bootstrapConfirmed, + spawnBootstrapStalled: provisionedButNotAliveSpawn.bootstrapStalled, + spawnAgentToolAccepted: provisionedButNotAliveSpawn.agentToolAccepted, + spawnHardFailure: provisionedButNotAliveSpawn.hardFailure, + spawnRuntimeDiagnostic: provisionedButNotAliveSpawn.hardFailureReason, + spawnLivenessKind: provisionedButNotAliveSpawn.livenessKind, + runtimeEntry: processTableUnavailableRuntime, + runtimeAdvisory: undefined, + isLaunchSettling: false, + isTeamAlive: true, + isTeamProvisioning: false, + }) + ).toMatchObject({ + presenceLabel: 'idle', + launchVisualState: null, + launchStatusLabel: null, + spawnBadgeLabel: null, + }); + }); + + it('keeps runtime errors visible for bootstrap-confirmed provisioned-but-not-alive entries', () => { + expect( + buildMemberLaunchPresentation({ + member, + spawnStatus: provisionedButNotAliveSpawn.status, + spawnLaunchState: provisionedButNotAliveSpawn.launchState, + spawnLivenessSource: provisionedButNotAliveSpawn.livenessSource, + spawnRuntimeAlive: provisionedButNotAliveSpawn.runtimeAlive, + spawnBootstrapConfirmed: provisionedButNotAliveSpawn.bootstrapConfirmed, + spawnBootstrapStalled: provisionedButNotAliveSpawn.bootstrapStalled, + spawnAgentToolAccepted: provisionedButNotAliveSpawn.agentToolAccepted, + spawnHardFailure: provisionedButNotAliveSpawn.hardFailure, + spawnHardFailureReason: provisionedButNotAliveSpawn.hardFailureReason, + spawnError: provisionedButNotAliveSpawn.error, + spawnLivenessKind: provisionedButNotAliveSpawn.livenessKind, + spawnRuntimeDiagnosticSeverity: provisionedButNotAliveSpawn.runtimeDiagnosticSeverity, + runtimeEntry: { + ...processTableUnavailableRuntime, + runtimeDiagnostic: 'Runtime process crashed', + runtimeDiagnosticSeverity: 'error', + }, + runtimeAdvisory: undefined, + isLaunchSettling: false, + isTeamAlive: true, + isTeamProvisioning: false, + }) + ).toMatchObject({ + presenceLabel: 'spawn failed', + launchVisualState: 'error', + launchStatusLabel: 'failed', + spawnBadgeLabel: 'error', + }); + }); + + it('keeps spawn diagnostic errors visible for bootstrap-confirmed provisioned-but-not-alive entries', () => { + expect( + buildMemberLaunchPresentation({ + member, + spawnStatus: provisionedButNotAliveSpawn.status, + spawnLaunchState: provisionedButNotAliveSpawn.launchState, + spawnLivenessSource: provisionedButNotAliveSpawn.livenessSource, + spawnRuntimeAlive: provisionedButNotAliveSpawn.runtimeAlive, + spawnBootstrapConfirmed: provisionedButNotAliveSpawn.bootstrapConfirmed, + spawnBootstrapStalled: provisionedButNotAliveSpawn.bootstrapStalled, + spawnAgentToolAccepted: provisionedButNotAliveSpawn.agentToolAccepted, + spawnHardFailure: provisionedButNotAliveSpawn.hardFailure, + spawnHardFailureReason: provisionedButNotAliveSpawn.hardFailureReason, + spawnError: provisionedButNotAliveSpawn.error, + spawnLivenessKind: provisionedButNotAliveSpawn.livenessKind, + spawnRuntimeDiagnosticSeverity: 'error', + runtimeAdvisory: undefined, + isLaunchSettling: false, + isTeamAlive: true, + isTeamProvisioning: false, + }) + ).toMatchObject({ + presenceLabel: 'spawn failed', + launchVisualState: 'error', + launchStatusLabel: 'failed', + }); + }); + + it('keeps stopped runtime evidence failed for bootstrap-confirmed provisioned-but-not-alive entries', () => { + expect( + buildMemberLaunchPresentation({ + member, + spawnStatus: provisionedButNotAliveSpawn.status, + spawnLaunchState: provisionedButNotAliveSpawn.launchState, + spawnLivenessSource: provisionedButNotAliveSpawn.livenessSource, + spawnRuntimeAlive: provisionedButNotAliveSpawn.runtimeAlive, + spawnBootstrapConfirmed: provisionedButNotAliveSpawn.bootstrapConfirmed, + spawnBootstrapStalled: provisionedButNotAliveSpawn.bootstrapStalled, + spawnAgentToolAccepted: provisionedButNotAliveSpawn.agentToolAccepted, + spawnHardFailure: provisionedButNotAliveSpawn.hardFailure, + spawnHardFailureReason: provisionedButNotAliveSpawn.hardFailureReason, + spawnError: provisionedButNotAliveSpawn.error, + spawnLivenessKind: 'not_found', + spawnRuntimeDiagnosticSeverity: 'warning', + runtimeAdvisory: undefined, + isLaunchSettling: false, + isTeamAlive: true, + isTeamProvisioning: false, + }) + ).toMatchObject({ + presenceLabel: 'spawn failed', + launchVisualState: 'error', + launchStatusLabel: 'failed', + spawnBadgeLabel: 'error', + }); + }); + it('renders unified retry advisory labels for provider retries', () => { expect( getMemberRuntimeAdvisoryLabel( diff --git a/test/renderer/utils/memberLaunchDiagnostics.test.ts b/test/renderer/utils/memberLaunchDiagnostics.test.ts index 80b8ec8e..1117052e 100644 --- a/test/renderer/utils/memberLaunchDiagnostics.test.ts +++ b/test/renderer/utils/memberLaunchDiagnostics.test.ts @@ -1,5 +1,6 @@ import { buildMemberLaunchDiagnosticsPayload, + buildTeamMemberLaunchDiagnosticsPayloads, formatMemberLaunchDiagnosticsPayload, getMemberLaunchDiagnosticsErrorMessage, hasMemberLaunchDiagnosticsDetails, @@ -123,6 +124,360 @@ describe('member launch diagnostics', () => { expect(payload.runtimeDiagnostic).toBe('persisted runtime pid is not alive'); }); + it('does not surface bootstrap-confirmed provisioned-but-not-alive entries as card errors', () => { + const payload = buildMemberLaunchDiagnosticsPayload({ + teamName: 'signal-ops', + runId: 'bb64da3b-ed5e-4bae-813d-70e26418f9e5', + memberName: 'tom', + spawnEntry: { + status: 'error', + launchState: 'failed_to_start', + agentToolAccepted: true, + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: 'CLI process exited (code 1) \u2014 team provisioned but not alive', + livenessKind: 'confirmed_bootstrap', + runtimeDiagnostic: 'runtime pid could not be verified because process table is unavailable', + runtimeDiagnosticSeverity: 'warning', + firstSpawnAcceptedAt: '2026-05-25T20:13:46.326Z', + lastHeartbeatAt: '2026-05-25T20:13:56.110Z', + updatedAt: '2026-05-25T20:14:02.147Z', + }, + runtimeEntry: { + memberName: 'tom', + alive: false, + restartable: true, + livenessKind: 'confirmed_bootstrap', + runtimeDiagnostic: 'runtime pid could not be verified because process table is unavailable', + runtimeDiagnosticSeverity: 'warning', + updatedAt: '2026-05-25T20:14:03.317Z', + }, + }); + + expect(payload).toMatchObject({ + launchState: 'confirmed_alive', + spawnStatus: 'online', + runtimeAlive: true, + bootstrapConfirmed: true, + hardFailure: false, + }); + expect(payload.memberCardError).toBeUndefined(); + expect(payload.probableCause).toBeUndefined(); + expect(hasMemberLaunchDiagnosticsError(payload)).toBe(false); + expect(getMemberLaunchDiagnosticsErrorMessage(payload)).toBeUndefined(); + }); + + it('does not surface spawn-only safe bootstrap-confirmed provisioned-but-not-alive entries as card errors', () => { + const payload = buildMemberLaunchDiagnosticsPayload({ + teamName: 'signal-ops', + runId: 'bb64da3b-ed5e-4bae-813d-70e26418f9e5', + memberName: 'tom', + spawnEntry: { + status: 'error', + launchState: 'failed_to_start', + agentToolAccepted: true, + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: 'CLI process exited (code 1) - team provisioned but not alive', + livenessKind: 'confirmed_bootstrap', + firstSpawnAcceptedAt: '2026-05-25T20:13:46.326Z', + lastHeartbeatAt: '2026-05-25T20:13:56.110Z', + updatedAt: '2026-05-25T20:14:02.147Z', + }, + }); + + expect(payload).toMatchObject({ + launchState: 'confirmed_alive', + spawnStatus: 'online', + runtimeAlive: true, + bootstrapConfirmed: true, + hardFailure: false, + }); + expect(payload.memberCardError).toBeUndefined(); + expect(hasMemberLaunchDiagnosticsError(payload)).toBe(false); + expect(getMemberLaunchDiagnosticsErrorMessage(payload)).toBeUndefined(); + }); + + it('keeps runtime errors visible for bootstrap-confirmed provisioned-but-not-alive entries', () => { + const payload = buildMemberLaunchDiagnosticsPayload({ + teamName: 'signal-ops', + runId: 'bb64da3b-ed5e-4bae-813d-70e26418f9e5', + memberName: 'tom', + spawnEntry: { + status: 'error', + launchState: 'failed_to_start', + agentToolAccepted: true, + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: 'CLI process exited (code 1) - team provisioned but not alive', + livenessKind: 'confirmed_bootstrap', + firstSpawnAcceptedAt: '2026-05-25T20:13:46.326Z', + lastHeartbeatAt: '2026-05-25T20:13:56.110Z', + updatedAt: '2026-05-25T20:14:02.147Z', + }, + runtimeEntry: { + memberName: 'tom', + alive: false, + restartable: true, + livenessKind: 'confirmed_bootstrap', + runtimeDiagnostic: 'Runtime process crashed', + runtimeDiagnosticSeverity: 'error', + updatedAt: '2026-05-25T20:14:03.317Z', + }, + }); + + expect(payload).toMatchObject({ + launchState: 'failed_to_start', + spawnStatus: 'error', + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + memberCardError: 'Runtime process crashed', + runtimeDiagnostic: 'Runtime process crashed', + runtimeDiagnosticSeverity: 'error', + }); + expect(hasMemberLaunchDiagnosticsError(payload)).toBe(true); + expect(getMemberLaunchDiagnosticsErrorMessage(payload)).toBe('Runtime process crashed'); + }); + + it('keeps spawn errors visible when runtime evidence is only warning severity', () => { + const payload = buildMemberLaunchDiagnosticsPayload({ + teamName: 'signal-ops', + runId: 'bb64da3b-ed5e-4bae-813d-70e26418f9e5', + memberName: 'tom', + spawnEntry: { + status: 'error', + launchState: 'failed_to_start', + agentToolAccepted: true, + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: 'CLI process exited (code 1) - team provisioned but not alive', + livenessKind: 'confirmed_bootstrap', + runtimeDiagnostic: 'Runtime process crashed', + runtimeDiagnosticSeverity: 'error', + firstSpawnAcceptedAt: '2026-05-25T20:13:46.326Z', + lastHeartbeatAt: '2026-05-25T20:13:56.110Z', + updatedAt: '2026-05-25T20:14:02.147Z', + }, + runtimeEntry: { + memberName: 'tom', + alive: false, + restartable: true, + livenessKind: 'confirmed_bootstrap', + runtimeDiagnostic: 'runtime pid could not be verified because process table is unavailable', + runtimeDiagnosticSeverity: 'warning', + updatedAt: '2026-05-25T20:14:03.317Z', + }, + }); + + expect(payload).toMatchObject({ + launchState: 'failed_to_start', + spawnStatus: 'error', + runtimeAlive: false, + hardFailure: true, + memberCardError: 'Runtime process crashed', + runtimeDiagnostic: 'Runtime process crashed', + runtimeDiagnosticSeverity: 'error', + }); + expect(payload.diagnostics).toContain('Runtime process crashed'); + expect(hasMemberLaunchDiagnosticsError(payload)).toBe(true); + }); + + it('keeps spawn diagnostics for bootstrap-confirmed provisioned-but-not-alive entries without runtime evidence', () => { + const payload = buildMemberLaunchDiagnosticsPayload({ + teamName: 'signal-ops', + runId: 'bb64da3b-ed5e-4bae-813d-70e26418f9e5', + memberName: 'tom', + spawnEntry: { + status: 'error', + launchState: 'failed_to_start', + agentToolAccepted: true, + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: 'CLI process exited (code 1) - team provisioned but not alive', + livenessKind: 'confirmed_bootstrap', + runtimeDiagnostic: 'Runtime process crashed', + runtimeDiagnosticSeverity: 'error', + firstSpawnAcceptedAt: '2026-05-25T20:13:46.326Z', + lastHeartbeatAt: '2026-05-25T20:13:56.110Z', + updatedAt: '2026-05-25T20:14:02.147Z', + }, + }); + + expect(payload).toMatchObject({ + launchState: 'failed_to_start', + spawnStatus: 'error', + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + memberCardError: 'Runtime process crashed', + runtimeDiagnostic: 'Runtime process crashed', + runtimeDiagnosticSeverity: 'error', + }); + expect(payload.diagnostics).toContain('Runtime process crashed'); + expect(hasMemberLaunchDiagnosticsError(payload)).toBe(true); + }); + + it('does not heal stopped liveness evidence for bootstrap-confirmed provisioned-but-not-alive entries', () => { + const payload = buildMemberLaunchDiagnosticsPayload({ + teamName: 'signal-ops', + runId: 'bb64da3b-ed5e-4bae-813d-70e26418f9e5', + memberName: 'tom', + spawnEntry: { + status: 'error', + launchState: 'failed_to_start', + agentToolAccepted: true, + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: 'CLI process exited (code 1) - team provisioned but not alive', + livenessKind: 'not_found', + runtimeDiagnostic: 'Runtime is no longer registered', + runtimeDiagnosticSeverity: 'warning', + firstSpawnAcceptedAt: '2026-05-25T20:13:46.326Z', + lastHeartbeatAt: '2026-05-25T20:13:56.110Z', + updatedAt: '2026-05-25T20:14:02.147Z', + }, + }); + + expect(payload).toMatchObject({ + launchState: 'failed_to_start', + spawnStatus: 'error', + runtimeAlive: false, + hardFailure: true, + runtimeDiagnostic: 'Runtime is no longer registered', + runtimeDiagnosticSeverity: 'warning', + }); + }); + + it('keeps unsafe spawn diagnostics over benign runtime warnings for provisioned-but-not-alive entries', () => { + const payload = buildMemberLaunchDiagnosticsPayload({ + teamName: 'signal-ops', + runId: 'bb64da3b-ed5e-4bae-813d-70e26418f9e5', + memberName: 'tom', + spawnEntry: { + status: 'error', + launchState: 'failed_to_start', + agentToolAccepted: true, + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: 'CLI process exited (code 1) - team provisioned but not alive', + livenessKind: 'not_found', + runtimeDiagnostic: 'Runtime is no longer registered', + runtimeDiagnosticSeverity: 'warning', + updatedAt: '2026-05-25T20:14:02.147Z', + }, + runtimeEntry: { + memberName: 'tom', + alive: false, + restartable: true, + livenessKind: 'registered_only', + runtimeDiagnostic: 'runtime pid could not be verified because process table is unavailable', + runtimeDiagnosticSeverity: 'warning', + updatedAt: '2026-05-25T20:14:03.317Z', + }, + }); + + expect(payload).toMatchObject({ + launchState: 'failed_to_start', + spawnStatus: 'error', + runtimeAlive: false, + hardFailure: true, + runtimeDiagnostic: 'Runtime is no longer registered', + runtimeDiagnosticSeverity: 'warning', + }); + expect(payload.diagnostics).toContain('Runtime is no longer registered'); + }); + + it('prefers stopped runtime liveness over stale spawn liveness in copy diagnostics', () => { + const payload = buildMemberLaunchDiagnosticsPayload({ + teamName: 'signal-ops', + runId: 'bb64da3b-ed5e-4bae-813d-70e26418f9e5', + memberName: 'tom', + spawnEntry: { + status: 'error', + launchState: 'failed_to_start', + agentToolAccepted: true, + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: 'CLI process exited (code 1) - team provisioned but not alive', + livenessKind: 'confirmed_bootstrap', + runtimeDiagnostic: + 'runtime pid could not be verified because process table is unavailable', + runtimeDiagnosticSeverity: 'warning', + updatedAt: '2026-05-25T20:14:02.147Z', + }, + runtimeEntry: { + memberName: 'tom', + alive: false, + restartable: true, + livenessKind: 'not_found', + runtimeDiagnostic: 'Runtime is no longer registered', + runtimeDiagnosticSeverity: 'warning', + updatedAt: '2026-05-25T20:14:03.317Z', + }, + }); + + expect(payload).toMatchObject({ + livenessKind: 'not_found', + runtimeAlive: false, + runtimeDiagnostic: 'Runtime is no longer registered', + }); + }); + + it('prefers newer healed snapshots over unsafe live provisioned-but-not-alive diagnostics', () => { + const [payload] = buildTeamMemberLaunchDiagnosticsPayloads({ + teamName: 'signal-ops', + runId: 'run-42', + members: [{ name: 'tom', providerId: 'anthropic' }], + memberSpawnStatuses: { + tom: { + status: 'error', + launchState: 'failed_to_start', + agentToolAccepted: true, + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: 'CLI process exited (code 1) - team provisioned but not alive', + livenessKind: 'not_found', + runtimeDiagnostic: 'Runtime is no longer registered', + runtimeDiagnosticSeverity: 'warning', + updatedAt: '2026-05-25T20:14:02.147Z', + }, + }, + memberSpawnSnapshot: { + updatedAt: '2026-05-25T20:14:10.000Z', + statuses: { + tom: { + status: 'online', + launchState: 'confirmed_alive', + agentToolAccepted: true, + runtimeAlive: true, + bootstrapConfirmed: true, + hardFailure: false, + livenessKind: 'confirmed_bootstrap', + updatedAt: '2026-05-25T20:14:10.000Z', + }, + }, + }, + }); + + expect(payload).toMatchObject({ + memberName: 'tom', + launchState: 'confirmed_alive', + spawnStatus: 'online', + runtimeAlive: true, + hardFailure: false, + }); + }); + it('includes runtime advisory evidence in copy diagnostics', () => { const payload = buildMemberLaunchDiagnosticsPayload({ memberName: 'alice', diff --git a/test/renderer/utils/teamProvisioningPresentation.test.ts b/test/renderer/utils/teamProvisioningPresentation.test.ts index c5a40dae..3a8a5ecc 100644 --- a/test/renderer/utils/teamProvisioningPresentation.test.ts +++ b/test/renderer/utils/teamProvisioningPresentation.test.ts @@ -924,6 +924,127 @@ describe('buildTeamProvisioningPresentation', () => { expect(presentation?.currentStepIndex).toBe(2); }); + it('does not present bootstrap-confirmed provisioned-but-not-alive entries as failed', () => { + const presentation = buildTeamProvisioningPresentation({ + progress: { + runId: 'run-signal-ops', + teamName: 'signal-ops', + state: 'ready', + startedAt: '2026-05-25T20:13:40.000Z', + updatedAt: '2026-05-25T20:14:05.411Z', + message: 'Team provisioned', + messageSeverity: undefined, + pid: 27036, + cliLogsTail: '', + assistantOutput: '', + }, + members: [ + { + name: 'team-lead', + agentType: 'team-lead', + providerId: 'anthropic', + status: 'active', + currentTaskId: null, + taskCount: 0, + lastActiveAt: null, + messageCount: 0, + }, + { + name: 'tom', + providerId: 'anthropic', + laneKind: 'primary', + status: 'active', + currentTaskId: null, + taskCount: 0, + lastActiveAt: null, + messageCount: 0, + }, + ], + memberSpawnStatuses: { + tom: { + status: 'error', + launchState: 'failed_to_start', + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: 'CLI process exited (code 1) \u2014 team provisioned but not alive', + livenessKind: 'confirmed_bootstrap', + runtimeDiagnostic: + 'runtime pid could not be verified because process table is unavailable', + runtimeDiagnosticSeverity: 'warning', + updatedAt: '2026-05-25T20:14:02.147Z', + }, + }, + memberRuntimeEntries: { + tom: { + memberName: 'tom', + alive: false, + restartable: true, + livenessKind: 'confirmed_bootstrap', + runtimeDiagnostic: + 'runtime pid could not be verified because process table is unavailable', + runtimeDiagnosticSeverity: 'warning', + updatedAt: '2026-05-25T20:14:03.317Z', + }, + }, + }); + + expect(presentation?.isFailed).toBe(false); + expect(presentation?.failedSpawnCount).toBe(0); + expect(presentation?.heartbeatConfirmedCount).toBe(1); + expect(presentation?.panelTone).not.toBe('error'); + expect(presentation?.compactTone).not.toBe('error'); + }); + + it('presents unsafe bootstrap-confirmed provisioned-but-not-alive entries as failed', () => { + const presentation = buildTeamProvisioningPresentation({ + progress: { + runId: 'run-signal-ops', + teamName: 'signal-ops', + state: 'ready', + startedAt: '2026-05-25T20:13:40.000Z', + updatedAt: '2026-05-25T20:14:05.411Z', + message: 'Team provisioned', + messageSeverity: undefined, + pid: 27036, + cliLogsTail: '', + assistantOutput: '', + }, + members: [ + { + name: 'tom', + providerId: 'anthropic', + laneKind: 'primary', + status: 'active', + currentTaskId: null, + taskCount: 0, + lastActiveAt: null, + messageCount: 0, + }, + ], + memberSpawnStatuses: { + tom: { + status: 'error', + launchState: 'failed_to_start', + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: 'CLI process exited (code 1) - team provisioned but not alive', + livenessKind: 'not_found', + runtimeDiagnostic: 'Runtime is no longer registered', + runtimeDiagnosticSeverity: 'warning', + updatedAt: '2026-05-25T20:14:02.147Z', + }, + }, + }); + + expect(presentation?.isFailed).toBe(false); + expect(presentation?.failedSpawnCount).toBe(1); + expect(presentation?.heartbeatConfirmedCount).toBe(0); + expect(presentation?.successMessageSeverity).toBe('warning'); + expect(presentation?.compactTone).toBe('warning'); + }); + it('does not show core team ready while a primary member is still joining', () => { const presentation = buildTeamProvisioningPresentation({ progress: { diff --git a/test/shared/utils/teamLaunchFailureReason.test.ts b/test/shared/utils/teamLaunchFailureReason.test.ts new file mode 100644 index 00000000..af11a4f9 --- /dev/null +++ b/test/shared/utils/teamLaunchFailureReason.test.ts @@ -0,0 +1,149 @@ +import { describe, expect, it } from 'vitest'; + +import { + hasUnsafeProvisionedButNotAliveRuntimeEvidence, + hasUnsafeProvisionedButNotAliveRuntimeEvidenceWithSpawnContext, + isBootstrapConfirmedProvisionedButNotAliveFailure, +} from '@shared/utils/teamLaunchFailureReason'; + +describe('teamLaunchFailureReason', () => { + it('treats runtime process candidates as unsafe provisioned-but-not-alive evidence', () => { + expect( + hasUnsafeProvisionedButNotAliveRuntimeEvidence({ + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: 'CLI process exited (code 1) - team provisioned but not alive', + launchState: 'failed_to_start', + livenessKind: 'runtime_process_candidate', + runtimeDiagnostic: + 'OpenCode runtime process detected, but teammate bootstrap is not confirmed', + runtimeDiagnosticSeverity: 'warning', + status: 'error', + }) + ).toBe(true); + }); + + it('treats permission-blocked runtime liveness as unsafe provisioned-but-not-alive evidence', () => { + expect( + hasUnsafeProvisionedButNotAliveRuntimeEvidence({ + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: 'CLI process exited (code 1) - team provisioned but not alive', + launchState: 'failed_to_start', + livenessKind: 'permission_blocked', + runtimeDiagnostic: 'runtime is waiting for permission approval', + runtimeDiagnosticSeverity: 'warning', + status: 'error', + }) + ).toBe(true); + }); + + it('keeps process-table-unavailable registered metadata safe for bootstrap healing', () => { + expect( + hasUnsafeProvisionedButNotAliveRuntimeEvidence({ + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: 'CLI process exited (code 1) - team provisioned but not alive', + launchState: 'failed_to_start', + livenessKind: 'registered_only', + runtimeDiagnostic: 'runtime pid could not be verified because process table is unavailable', + runtimeDiagnosticSeverity: 'warning', + status: 'error', + }) + ).toBe(false); + }); + + it('treats missing liveness without process-table evidence as unsafe', () => { + expect( + hasUnsafeProvisionedButNotAliveRuntimeEvidence({ + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: 'CLI process exited (code 1) - team provisioned but not alive', + launchState: 'failed_to_start', + status: 'error', + }) + ).toBe(true); + }); + + it('keeps missing liveness safe when process-table evidence is explicit', () => { + expect( + hasUnsafeProvisionedButNotAliveRuntimeEvidence({ + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: + 'CLI process exited (code 1) - team provisioned but not alive; process table unavailable', + launchState: 'failed_to_start', + status: 'error', + }) + ).toBe(false); + }); + + it('uses spawn process-table evidence for registered runtime metadata without diagnostics', () => { + expect( + hasUnsafeProvisionedButNotAliveRuntimeEvidenceWithSpawnContext( + { + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: + 'CLI process exited (code 1) - team provisioned but not alive; process table unavailable', + launchState: 'failed_to_start', + status: 'error', + }, + { + livenessKind: 'registered_only', + runtimeDiagnosticSeverity: 'warning', + } + ) + ).toBe(false); + }); + + it('uses spawn process-table evidence for runtime metadata without liveness or diagnostics', () => { + expect( + hasUnsafeProvisionedButNotAliveRuntimeEvidenceWithSpawnContext( + { + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: + 'CLI process exited (code 1) - team provisioned but not alive; process table unavailable', + launchState: 'failed_to_start', + status: 'error', + }, + { + runtimeDiagnosticSeverity: 'warning', + } + ) + ).toBe(false); + }); + + it('keeps registered runtime metadata unsafe when runtime diagnostics contradict spawn proof', () => { + expect( + hasUnsafeProvisionedButNotAliveRuntimeEvidenceWithSpawnContext( + { + bootstrapConfirmed: true, + hardFailure: true, + hardFailureReason: + 'CLI process exited (code 1) - team provisioned but not alive; process table unavailable', + launchState: 'failed_to_start', + status: 'error', + }, + { + livenessKind: 'registered_only', + runtimeDiagnostic: 'Runtime heartbeat is not alive', + runtimeDiagnosticSeverity: 'warning', + } + ) + ).toBe(true); + }); + + it('recognizes runtime-diagnostic-only provisioned-but-not-alive failures', () => { + expect( + isBootstrapConfirmedProvisionedButNotAliveFailure({ + bootstrapConfirmed: true, + hardFailure: true, + launchState: 'failed_to_start', + runtimeDiagnostic: 'CLI process exited (code 1) - team provisioned but not alive', + status: 'error', + }) + ).toBe(true); + }); +});