diff --git a/src/features/agent-graph/renderer/adapters/TeamGraphAdapter.ts b/src/features/agent-graph/renderer/adapters/TeamGraphAdapter.ts index eefda424..e9602ce6 100644 --- a/src/features/agent-graph/renderer/adapters/TeamGraphAdapter.ts +++ b/src/features/agent-graph/renderer/adapters/TeamGraphAdapter.ts @@ -72,6 +72,12 @@ export interface TeamGraphData extends TeamViewSnapshot { messageFeed: InboxMessage[]; } +function toGraphLaunchVisualState( + visualState: ReturnType['launchVisualState'] | undefined +): GraphNode['launchVisualState'] { + return visualState === 'bootstrap_stalled' ? 'runtime_pending' : (visualState ?? undefined); +} + export class TeamGraphAdapter { // ─── ES #private fields ────────────────────────────────────────────────── #lastTeamName = ''; @@ -430,6 +436,7 @@ export class TeamGraphAdapter { spawnLaunchState: undefined, spawnLivenessSource: undefined, spawnRuntimeAlive: undefined, + spawnBootstrapStalled: undefined, runtimeAdvisory: leadMember.runtimeAdvisory, isLaunchSettling: false, isTeamAlive: data.isAlive, @@ -462,7 +469,7 @@ export class TeamGraphAdapter { leadMember?.model, leadMember?.effort ), - launchVisualState: leadLaunchPresentation?.launchVisualState ?? undefined, + launchVisualState: toGraphLaunchVisualState(leadLaunchPresentation?.launchVisualState), launchStatusLabel: leadLaunchPresentation?.launchStatusLabel ?? undefined, contextUsage: percent != null ? Math.max(0, Math.min(1, percent / 100)) : undefined, avatarUrl: leadMember @@ -538,6 +545,7 @@ export class TeamGraphAdapter { spawnLaunchState: spawn?.launchState, spawnLivenessSource: spawn?.livenessSource, spawnRuntimeAlive: spawn?.runtimeAlive, + spawnBootstrapStalled: spawn?.bootstrapStalled, runtimeAdvisory: member.runtimeAdvisory, isLaunchSettling, isTeamAlive: data.isAlive, @@ -562,7 +570,7 @@ export class TeamGraphAdapter { ), spawnStatus: isTeamVisualOnline ? spawn?.status : undefined, launchVisualState: isTeamVisualOnline - ? (launchPresentation.launchVisualState ?? undefined) + ? toGraphLaunchVisualState(launchPresentation.launchVisualState) : undefined, launchStatusLabel: isTeamVisualOnline ? (launchPresentation.launchStatusLabel ?? undefined) diff --git a/src/features/agent-graph/renderer/ui/GraphNodePopover.tsx b/src/features/agent-graph/renderer/ui/GraphNodePopover.tsx index f6794aa1..ca412410 100644 --- a/src/features/agent-graph/renderer/ui/GraphNodePopover.tsx +++ b/src/features/agent-graph/renderer/ui/GraphNodePopover.tsx @@ -325,6 +325,7 @@ const MemberPopoverContent = ({ spawnLaunchState: spawnEntry?.launchState, spawnLivenessSource: spawnEntry?.livenessSource, spawnRuntimeAlive: spawnEntry?.runtimeAlive, + spawnBootstrapStalled: spawnEntry?.bootstrapStalled, runtimeAdvisory: member.runtimeAdvisory, isLaunchSettling: provisioningPresentation?.hasMembersStillJoining ?? false, isTeamAlive: teamData?.isAlive, diff --git a/src/features/team-runtime-lanes/core/domain/buildMixedPersistedLaunchSnapshot.ts b/src/features/team-runtime-lanes/core/domain/buildMixedPersistedLaunchSnapshot.ts index ffbe971d..94263c35 100644 --- a/src/features/team-runtime-lanes/core/domain/buildMixedPersistedLaunchSnapshot.ts +++ b/src/features/team-runtime-lanes/core/domain/buildMixedPersistedLaunchSnapshot.ts @@ -47,6 +47,7 @@ export interface MixedSecondaryLaneMemberStateInput { pidSource?: TeamAgentRuntimePidSource; runtimeDiagnostic?: string; runtimeDiagnosticSeverity?: TeamAgentRuntimeDiagnosticSeverity; + bootstrapStalled?: boolean; diagnostics?: string[]; } | null; pendingReason?: string; @@ -96,6 +97,7 @@ function buildDiagnostics( | 'hardFailureReason' | 'sources' | 'pendingPermissionRequestIds' + | 'bootstrapStalled' > ): string[] { const diagnostics: string[] = []; @@ -104,6 +106,8 @@ function buildDiagnostics( if (member.bootstrapConfirmed) diagnostics.push('late heartbeat received'); if ((member.pendingPermissionRequestIds?.length ?? 0) > 0) { diagnostics.push('waiting for permission approval'); + } else if (member.bootstrapStalled) { + diagnostics.push('opencode_bootstrap_stalled'); } else if (member.runtimeAlive && !member.bootstrapConfirmed) { diagnostics.push('waiting for teammate check-in'); } @@ -268,6 +272,15 @@ function createSecondaryLaneMemberState( pidSource: evidence?.pidSource, runtimeDiagnostic: evidence?.runtimeDiagnostic, runtimeDiagnosticSeverity: evidence?.runtimeDiagnosticSeverity, + bootstrapStalled: + providerId === 'opencode' && + evidence?.bootstrapStalled === true && + launchState === 'runtime_pending_bootstrap' && + strongRuntimeAlive && + evidence.bootstrapConfirmed !== true && + hardFailure !== true + ? true + : undefined, firstSpawnAcceptedAt: evidence?.agentToolAccepted ? params.updatedAt : undefined, lastHeartbeatAt: evidence?.bootstrapConfirmed ? params.updatedAt : undefined, runtimeLastSeenAt: strongRuntimeAlive ? params.updatedAt : undefined, diff --git a/src/main/services/team/TeamLaunchStateEvaluator.ts b/src/main/services/team/TeamLaunchStateEvaluator.ts index 1dfc33ed..f19c8b74 100644 --- a/src/main/services/team/TeamLaunchStateEvaluator.ts +++ b/src/main/services/team/TeamLaunchStateEvaluator.ts @@ -46,6 +46,7 @@ type RuntimeMemberSpawnState = Pick< | 'livenessKind' | 'runtimeDiagnostic' | 'runtimeDiagnosticSeverity' + | 'bootstrapStalled' | 'livenessLastCheckedAt' | 'firstSpawnAcceptedAt' | 'lastHeartbeatAt' @@ -100,6 +101,47 @@ function preservesStrongRuntimeAlive( ); } +function isOpenCodeSecondaryBootstrapPending( + member: Pick< + PersistedTeamLaunchMemberState, + | 'providerId' + | 'laneKind' + | 'laneOwnerProviderId' + | 'launchState' + | 'bootstrapConfirmed' + | 'hardFailure' + > +): boolean { + return ( + member.providerId === 'opencode' && + member.laneKind === 'secondary' && + member.laneOwnerProviderId === 'opencode' && + member.launchState === 'runtime_pending_bootstrap' && + member.bootstrapConfirmed !== true && + member.hardFailure !== true + ); +} + +function isPersistedBootstrapStalled( + member: Pick< + PersistedTeamLaunchMemberState, + | 'providerId' + | 'laneKind' + | 'laneOwnerProviderId' + | 'launchState' + | 'runtimeAlive' + | 'bootstrapConfirmed' + | 'hardFailure' + | 'bootstrapStalled' + > +): boolean { + return ( + member.bootstrapStalled === true && + isOpenCodeSecondaryBootstrapPending(member) && + member.runtimeAlive === true + ); +} + function normalizePidSource(value: unknown): TeamAgentRuntimePidSource | undefined { return value === 'lead_process' || value === 'tmux_pane' || @@ -198,6 +240,7 @@ function buildDiagnostics( | 'skipReason' | 'sources' | 'pendingPermissionRequestIds' + | 'bootstrapStalled' > ): string[] { const diagnostics: string[] = []; @@ -206,6 +249,8 @@ function buildDiagnostics( if (member.bootstrapConfirmed) diagnostics.push('late heartbeat received'); if ((member.pendingPermissionRequestIds?.length ?? 0) > 0) { diagnostics.push('waiting for permission approval'); + } else if (member.bootstrapStalled) { + diagnostics.push('opencode_bootstrap_stalled'); } else if (member.runtimeAlive && !member.bootstrapConfirmed) { diagnostics.push('waiting for teammate check-in'); } @@ -545,6 +590,7 @@ function normalizePersistedMemberState( pidSource: normalizePidSource(parsed.pidSource), runtimeDiagnostic: normalizeOptionalString(parsed.runtimeDiagnostic), runtimeDiagnosticSeverity: normalizeDiagnosticSeverity(parsed.runtimeDiagnosticSeverity), + bootstrapStalled: toBoolean(parsed.bootstrapStalled), runtimeLastSeenAt: normalizeOptionalString(parsed.runtimeLastSeenAt), firstSpawnAcceptedAt: typeof parsed.firstSpawnAcceptedAt === 'string' ? parsed.firstSpawnAcceptedAt : undefined, @@ -571,6 +617,9 @@ function normalizePersistedMemberState( ? parsed.launchState : deriveMemberLaunchState(next); next.launchState = launchState; + if (!isPersistedBootstrapStalled(next)) { + next.bootstrapStalled = undefined; + } next.diagnostics = next.diagnostics?.length ? next.diagnostics : buildDiagnostics(next); return next; } @@ -714,6 +763,7 @@ export function snapshotFromRuntimeMemberStatuses(params: { livenessKind: runtime?.livenessKind, runtimeDiagnostic: runtime?.runtimeDiagnostic, runtimeDiagnosticSeverity: runtime?.runtimeDiagnosticSeverity, + bootstrapStalled: runtime?.bootstrapStalled === true, runtimeLastSeenAt: runtime?.livenessLastCheckedAt, firstSpawnAcceptedAt: runtime?.firstSpawnAcceptedAt, lastHeartbeatAt: runtime?.lastHeartbeatAt, @@ -723,6 +773,9 @@ export function snapshotFromRuntimeMemberStatuses(params: { diagnostics: undefined, }; entry.launchState = deriveMemberLaunchState(entry); + if (!isPersistedBootstrapStalled(entry)) { + entry.bootstrapStalled = undefined; + } entry.diagnostics = buildDiagnostics(entry); members[name] = entry; } @@ -756,6 +809,8 @@ export function snapshotToMemberSpawnStatuses( const skippedForLaunch = entry.launchState === 'skipped_for_launch' || entry.skippedForLaunch === true; const runtimeAlive = skippedForLaunch ? false : preservesStrongRuntimeAlive(entry); + const openCodeBootstrapPending = isOpenCodeSecondaryBootstrapPending(entry); + const bootstrapStalled = isPersistedBootstrapStalled(entry); if (entry.launchState === 'failed_to_start') { status = 'error'; } else if (entry.launchState === 'skipped_for_launch') { @@ -767,8 +822,10 @@ export function snapshotToMemberSpawnStatuses( entry.launchState === 'runtime_pending_permission' || entry.launchState === 'runtime_pending_bootstrap' ) { - status = runtimeAlive ? 'online' : 'waiting'; - livenessSource = runtimeAlive ? 'process' : undefined; + status = + runtimeAlive && !openCodeBootstrapPending && !bootstrapStalled ? 'online' : 'waiting'; + livenessSource = + runtimeAlive && !openCodeBootstrapPending && !bootstrapStalled ? 'process' : undefined; } else { status = entry.agentToolAccepted ? 'waiting' : 'spawning'; } @@ -789,6 +846,7 @@ export function snapshotToMemberSpawnStatuses( livenessKind: entry.livenessKind, runtimeDiagnostic: entry.runtimeDiagnostic, runtimeDiagnosticSeverity: entry.runtimeDiagnosticSeverity, + bootstrapStalled, livenessLastCheckedAt: entry.runtimeLastSeenAt ?? entry.lastEvaluatedAt, firstSpawnAcceptedAt: entry.firstSpawnAcceptedAt, lastHeartbeatAt: entry.lastHeartbeatAt, diff --git a/src/main/services/team/TeamProvisioningService.ts b/src/main/services/team/TeamProvisioningService.ts index 0f764b0a..a824e498 100644 --- a/src/main/services/team/TeamProvisioningService.ts +++ b/src/main/services/team/TeamProvisioningService.ts @@ -4075,6 +4075,18 @@ function buildLaunchDiagnosticsFromRun( }); continue; } + if (entry.bootstrapStalled === true) { + items.push({ + id: `${memberName}:bootstrap_stalled`, + memberName, + severity: 'warning', + code: 'bootstrap_stalled', + label: `${memberName} - bootstrap stalled`, + detail: entry.runtimeDiagnostic, + observedAt, + }); + continue; + } if (mentionsProcessTableUnavailable(entry.runtimeDiagnostic)) { items.push({ id: `${memberName}:process_table_unavailable`, @@ -9283,6 +9295,7 @@ export class TeamProvisioningService { runtimeAlive: true, bootstrapConfirmed: true, hardFailure: false, + bootstrapStalled: undefined, runtimePid, runtimeRunId: input.runId, runtimeSessionId: input.runtimeSessionId, @@ -9432,6 +9445,7 @@ export class TeamProvisioningService { runtimeAlive: true, bootstrapConfirmed: true, hardFailure: false, + bootstrapStalled: undefined, pendingPermissionRequestIds: undefined, firstSpawnAcceptedAt: previousStatus.firstSpawnAcceptedAt ?? input.observedAt, lastHeartbeatAt: input.observedAt, @@ -10247,6 +10261,7 @@ export class TeamProvisioningService { next.runtimeAlive = false; next.bootstrapConfirmed = false; next.hardFailure = false; + next.bootstrapStalled = undefined; next.error = undefined; next.hardFailureReason = undefined; next.livenessSource = undefined; @@ -10265,6 +10280,7 @@ export class TeamProvisioningService { next.runtimeAlive = false; next.bootstrapConfirmed = false; next.hardFailure = false; + next.bootstrapStalled = undefined; next.error = undefined; next.hardFailureReason = undefined; next.livenessSource = undefined; @@ -10294,6 +10310,7 @@ export class TeamProvisioningService { : prev.lastHeartbeatAt; } next.hardFailure = false; + next.bootstrapStalled = undefined; next.error = undefined; next.hardFailureReason = undefined; next.launchState = deriveMemberLaunchState(next); @@ -10303,6 +10320,7 @@ export class TeamProvisioningService { next.skippedAt = undefined; next.error = error; next.hardFailure = true; + next.bootstrapStalled = undefined; next.hardFailureReason = error; next.launchState = 'failed_to_start'; } else if (status === 'skipped') { @@ -10314,6 +10332,7 @@ export class TeamProvisioningService { next.runtimeAlive = false; next.bootstrapConfirmed = false; next.hardFailure = false; + next.bootstrapStalled = undefined; next.error = undefined; next.hardFailureReason = undefined; next.livenessSource = undefined; @@ -10357,6 +10376,7 @@ export class TeamProvisioningService { prev.livenessKind === next.livenessKind && prev.runtimeDiagnostic === next.runtimeDiagnostic && prev.runtimeDiagnosticSeverity === next.runtimeDiagnosticSeverity && + prev.bootstrapStalled === next.bootstrapStalled && prev.firstSpawnAcceptedAt === next.firstSpawnAcceptedAt && prev.lastHeartbeatAt === next.lastHeartbeatAt ) { @@ -10440,6 +10460,7 @@ export class TeamProvisioningService { runtimeAlive: prev.runtimeAlive === true, bootstrapConfirmed: true, hardFailure: false, + bootstrapStalled: undefined, error: undefined, hardFailureReason: undefined, livenessSource: prev.livenessSource ?? 'process', @@ -10460,6 +10481,7 @@ export class TeamProvisioningService { prev.runtimeAlive === next.runtimeAlive && prev.bootstrapConfirmed === next.bootstrapConfirmed && prev.hardFailure === next.hardFailure && + prev.bootstrapStalled === next.bootstrapStalled && prev.firstSpawnAcceptedAt === next.firstSpawnAcceptedAt && prev.lastHeartbeatAt === next.lastHeartbeatAt ) { @@ -10493,7 +10515,10 @@ export class TeamProvisioningService { }> { const readPersistedStatuses = async (resolvedRunId: string | null) => { const { snapshot, statuses } = await this.reconcilePersistedLaunchState(teamName); - const nextStatuses = await this.attachLiveRuntimeMetadataToStatuses(teamName, statuses); + const nextStatuses = await this.attachLiveRuntimeMetadataToStatuses(teamName, statuses, { + openCodeSecondaryBootstrapPendingMembers: + this.getOpenCodeSecondaryBootstrapPendingMemberNames(snapshot), + }); const expectedMembers = snapshot ? this.getPersistedLaunchMemberNames(snapshot) : undefined; const summary = expectedMembers ? summarizeMemberSpawnStatusRecord(expectedMembers, nextStatuses) @@ -10605,7 +10630,11 @@ export class TeamProvisioningService { const launchSnapshot = this.filterRemovedMembersFromLaunchSnapshot(rawSnapshot, metaMembers); const statuses = await this.attachLiveRuntimeMetadataToStatuses( teamName, - snapshotToMemberSpawnStatuses(launchSnapshot) + snapshotToMemberSpawnStatuses(launchSnapshot), + { + openCodeSecondaryBootstrapPendingMembers: + this.getOpenCodeSecondaryBootstrapPendingMemberNames(launchSnapshot), + } ); const expectedMembers = this.getPersistedLaunchMemberNames(launchSnapshot); const summary = summarizeMemberSpawnStatusRecord(expectedMembers, statuses); @@ -11910,14 +11939,27 @@ export class TeamProvisioningService { const elapsedMs = Number.isFinite(acceptedAtMs) ? Date.now() - acceptedAtMs : Infinity; const runtimeDiagnostic = metadata?.runtimeDiagnostic; if (metadata?.livenessKind === 'runtime_process') { - if (elapsedMs >= MEMBER_BOOTSTRAP_STALL_MS) { - run.memberSpawnStatuses.set(memberName, { - ...refreshed, - livenessKind: metadata.livenessKind, - runtimeDiagnostic: 'Runtime process is alive, but no bootstrap check-in after 5 min.', - runtimeDiagnosticSeverity: 'warning', - livenessLastCheckedAt: nowIso(), + if (this.isOpenCodeSecondaryLaneMemberInRun(run, memberName)) { + this.setOpenCodeRuntimePendingBootstrapStatus(run, memberName, refreshed, { + bootstrapStalled: elapsedMs >= MEMBER_BOOTSTRAP_STALL_MS, + runtimeDiagnostic: + elapsedMs >= MEMBER_BOOTSTRAP_STALL_MS + ? 'Runtime process is alive, but no bootstrap check-in after 5 min.' + : (runtimeDiagnostic ?? + 'OpenCode runtime process is alive, waiting for bootstrap check-in.'), + runtimeDiagnosticSeverity: + elapsedMs >= MEMBER_BOOTSTRAP_STALL_MS + ? 'warning' + : (metadata.runtimeDiagnosticSeverity ?? 'info'), }); + if (elapsedMs < MEMBER_BOOTSTRAP_STALL_MS) { + this.scheduleOpenCodeBootstrapStallReevaluation( + run, + memberName, + refreshedFirstSpawnAcceptedAt + ); + } + return; } this.setMemberSpawnStatus(run, memberName, 'online', undefined, 'process'); return; @@ -11988,6 +12030,98 @@ export class TeamProvisioningService { this.setMemberSpawnStatus(run, memberName, 'error', strictReason); } + private setOpenCodeRuntimePendingBootstrapStatus( + run: ProvisioningRun, + memberName: string, + current: MemberSpawnStatusEntry, + options: { + bootstrapStalled: boolean; + runtimeDiagnostic: string; + runtimeDiagnosticSeverity: TeamAgentRuntimeDiagnosticSeverity; + } + ): void { + const observedAt = nowIso(); + const wasBootstrapStalled = current.bootstrapStalled === true; + const next: MemberSpawnStatusEntry = { + ...current, + status: 'waiting', + launchState: 'runtime_pending_bootstrap', + agentToolAccepted: true, + runtimeAlive: true, + bootstrapConfirmed: false, + hardFailure: false, + error: undefined, + hardFailureReason: undefined, + livenessSource: undefined, + livenessKind: 'runtime_process', + runtimeDiagnostic: options.runtimeDiagnostic, + runtimeDiagnosticSeverity: options.runtimeDiagnosticSeverity, + bootstrapStalled: options.bootstrapStalled ? true : undefined, + livenessLastCheckedAt: observedAt, + firstSpawnAcceptedAt: current.firstSpawnAcceptedAt ?? observedAt, + updatedAt: observedAt, + }; + + run.memberSpawnStatuses.set(memberName, next); + const launchDiagnostics = boundLaunchDiagnostics(buildLaunchDiagnosticsFromRun(run)); + if (launchDiagnostics) { + run.progress = { + ...run.progress, + updatedAt: observedAt, + launchDiagnostics, + }; + run.onProgress(run.progress); + } + + if (options.bootstrapStalled && !wasBootstrapStalled) { + this.appendMemberBootstrapDiagnostic(run, memberName, 'opencode_bootstrap_stalled'); + } else if ( + !options.bootstrapStalled && + (current.status !== 'waiting' || current.livenessKind !== 'runtime_process') + ) { + this.appendMemberBootstrapDiagnostic( + run, + memberName, + 'runtime process is alive, teammate check-in not yet received' + ); + } + if (!this.isCurrentTrackedRun(run)) return; + this.emitMemberSpawnChange(run, memberName); + if (run.isLaunch) { + void this.persistLaunchStateSnapshot(run, run.provisioningComplete ? 'finished' : 'active'); + } + } + + private scheduleOpenCodeBootstrapStallReevaluation( + run: ProvisioningRun, + memberName: string, + firstSpawnAcceptedAt: string + ): void { + const acceptedAtMs = Date.parse(firstSpawnAcceptedAt); + if (!Number.isFinite(acceptedAtMs)) { + return; + } + const stallDelayMs = Math.max(1_000, acceptedAtMs + MEMBER_BOOTSTRAP_STALL_MS - Date.now()); + const stallKey = `${this.getMemberLaunchGraceKey(run, memberName)}:bootstrap-stall`; + if (this.pendingTimeouts.has(stallKey)) { + return; + } + const timer = setTimeout(() => { + this.pendingTimeouts.delete(stallKey); + void this.reevaluateMemberLaunchStatus(run, memberName); + }, stallDelayMs); + timer.unref?.(); + this.pendingTimeouts.set(stallKey, timer); + } + + private isOpenCodeBootstrapStallWindowElapsed(firstSpawnAcceptedAt: string | undefined): boolean { + if (!firstSpawnAcceptedAt) { + return false; + } + const acceptedAtMs = Date.parse(firstSpawnAcceptedAt); + return Number.isFinite(acceptedAtMs) && Date.now() - acceptedAtMs >= MEMBER_BOOTSTRAP_STALL_MS; + } + private shouldSkipMemberSpawnAudit(run: ProvisioningRun): boolean { if (!run.expectedMembers || run.expectedMembers.length === 0) { return true; @@ -17343,6 +17477,23 @@ export class TeamProvisioningService { // registered the runtime and the OS process is still alive, treat it as // process-confirmed running. Keep this distinct from heartbeat-confirmed online. if (runtimeAlive) { + if (this.isOpenCodeSecondaryLaneMemberInRun(run, expected)) { + const base = current ?? createInitialMemberSpawnStatusEntry(); + const bootstrapStalled = + base.bootstrapStalled === true || + this.isOpenCodeBootstrapStallWindowElapsed(base.firstSpawnAcceptedAt); + this.setOpenCodeRuntimePendingBootstrapStatus(run, expected, base, { + bootstrapStalled, + runtimeDiagnostic: bootstrapStalled + ? 'Runtime process is alive, but no bootstrap check-in after 5 min.' + : (base.runtimeDiagnostic ?? + 'OpenCode runtime process is alive, waiting for bootstrap check-in.'), + runtimeDiagnosticSeverity: bootstrapStalled + ? 'warning' + : (base.runtimeDiagnosticSeverity ?? 'info'), + }); + continue; + } this.setMemberSpawnStatus(run, expected, 'online', undefined, 'process'); continue; } @@ -17431,7 +17582,10 @@ export class TeamProvisioningService { private async attachLiveRuntimeMetadataToStatuses( teamName: string, - statuses: Record + statuses: Record, + options?: { + openCodeSecondaryBootstrapPendingMembers?: ReadonlySet; + } ): Promise> { const runtimeByMember = await this.getLiveTeamAgentRuntimeMetadata(teamName); const nextStatuses = { ...statuses }; @@ -17452,6 +17606,15 @@ export class TeamProvisioningService { if (!current) { continue; } + const openCodeSecondaryBootstrapPending = + options?.openCodeSecondaryBootstrapPendingMembers?.has(resolvedStatusKey) === true && + current.launchState === 'runtime_pending_bootstrap' && + current.bootstrapConfirmed !== true && + current.hardFailure !== true; + const openCodeBootstrapStalled = + openCodeSecondaryBootstrapPending && + (current.bootstrapStalled === true || + this.isOpenCodeBootstrapStallWindowElapsed(current.firstSpawnAcceptedAt)); if (current.launchState === 'skipped_for_launch' || current.skippedForLaunch === true) { nextStatuses[resolvedStatusKey] = { ...current, @@ -17487,6 +17650,8 @@ export class TeamProvisioningService { current.bootstrapConfirmed !== true; if ( hasStrongEvidence && + !openCodeSecondaryBootstrapPending && + current.bootstrapStalled !== true && current.hardFailure !== true && current.launchState !== 'failed_to_start' ) { @@ -17499,6 +17664,27 @@ export class TeamProvisioningService { nextEntry.livenessSource = current.bootstrapConfirmed ? current.livenessSource : 'process'; nextEntry.launchState = deriveMemberLaunchState(nextEntry); } + if ( + (current.bootstrapStalled === true || openCodeSecondaryBootstrapPending) && + hasStrongEvidence && + current.bootstrapConfirmed !== true && + current.launchState !== 'failed_to_start' + ) { + nextEntry.status = 'waiting'; + nextEntry.agentToolAccepted = true; + nextEntry.runtimeAlive = true; + nextEntry.hardFailure = false; + nextEntry.hardFailureReason = undefined; + nextEntry.error = undefined; + nextEntry.livenessSource = undefined; + nextEntry.bootstrapStalled = openCodeBootstrapStalled ? true : undefined; + if (openCodeBootstrapStalled) { + nextEntry.runtimeDiagnostic = + 'Runtime process is alive, but no bootstrap check-in after 5 min.'; + nextEntry.runtimeDiagnosticSeverity = 'warning'; + } + nextEntry.launchState = deriveMemberLaunchState(nextEntry); + } if ( hasStrongEvidence && current.launchState === 'failed_to_start' && @@ -17538,6 +17724,27 @@ export class TeamProvisioningService { return nextStatuses; } + private getOpenCodeSecondaryBootstrapPendingMemberNames( + snapshot: PersistedTeamLaunchSnapshot | null | undefined + ): ReadonlySet { + if (!snapshot) { + return new Set(); + } + const names = Object.entries(snapshot.members) + .filter(([, member]) => { + return ( + member.providerId === 'opencode' && + member.laneKind === 'secondary' && + member.laneOwnerProviderId === 'opencode' && + member.launchState === 'runtime_pending_bootstrap' && + member.bootstrapConfirmed !== true && + member.hardFailure !== true + ); + }) + .map(([name]) => name); + return new Set(names); + } + private async getLiveTeamAgentNames(teamName: string): Promise> { const runtimeByMember = await this.getLiveTeamAgentRuntimeMetadata(teamName); return new Set( diff --git a/src/renderer/components/team/members/MemberCard.tsx b/src/renderer/components/team/members/MemberCard.tsx index 6c9453e3..4f66def2 100644 --- a/src/renderer/components/team/members/MemberCard.tsx +++ b/src/renderer/components/team/members/MemberCard.tsx @@ -140,6 +140,7 @@ export const MemberCard = ({ spawnLaunchState, spawnLivenessSource, spawnRuntimeAlive, + spawnBootstrapStalled: spawnEntry?.bootstrapStalled, runtimeEntry, runtimeAdvisory: member.runtimeAdvisory, isLaunchSettling, @@ -157,6 +158,7 @@ export const MemberCard = ({ const launchStatusLabel = launchPresentation.launchStatusLabel; const displayPresenceLabel = launchVisualState === 'queued' || + launchVisualState === 'bootstrap_stalled' || launchVisualState === 'runtime_pending' || launchVisualState === 'permission_pending' || launchVisualState === 'shell_only' || diff --git a/src/renderer/components/team/members/MemberDetailDialog.tsx b/src/renderer/components/team/members/MemberDetailDialog.tsx index 8ab479ef..625069dd 100644 --- a/src/renderer/components/team/members/MemberDetailDialog.tsx +++ b/src/renderer/components/team/members/MemberDetailDialog.tsx @@ -50,6 +50,8 @@ import type { const OPENCODE_NO_RUNTIME_EVIDENCE_MESSAGE = 'No OpenCode runtime session was recorded. Relaunch this teammate to start a fresh OpenCode session.'; +const OPENCODE_BOOTSTRAP_STALLED_MESSAGE = + 'OpenCode process is alive, but bootstrap did not confirm. Relaunch this teammate to start a fresh OpenCode session.'; function hasOpenCodeRuntimeEvidence(runtimeEntry: TeamAgentRuntimeEntry | undefined): boolean { const hasPid = @@ -202,12 +204,17 @@ export const MemberDetailDialog = ({ const launchErrorMessage = launchDiagnosticsPayload ? getMemberLaunchDiagnosticsErrorMessage(launchDiagnosticsPayload) : undefined; + const openCodeBootstrapStalled = + member?.providerId === 'opencode' && spawnEntry?.bootstrapStalled === true; const openCodeNoRuntimeEvidence = member ? isOpenCodeNoRuntimeEvidenceFailure(member, spawnEntry, runtimeEntry) : false; const effectiveLaunchErrorMessage = openCodeNoRuntimeEvidence ? OPENCODE_NO_RUNTIME_EVIDENCE_MESSAGE : launchErrorMessage; + const effectiveLaunchInfoMessage = openCodeBootstrapStalled + ? OPENCODE_BOOTSTRAP_STALLED_MESSAGE + : undefined; const restartButtonLabel = openCodeNoRuntimeEvidence || openCodeRelaunchActionable ? 'Relaunch OpenCode' : 'Restart'; @@ -245,6 +252,7 @@ export const MemberDetailDialog = ({ spawnLaunchState={spawnEntry?.launchState} spawnLivenessSource={spawnEntry?.livenessSource} spawnRuntimeAlive={spawnEntry?.runtimeAlive} + spawnBootstrapStalled={spawnEntry?.bootstrapStalled} runtimeEntry={runtimeEntry} isLaunchSettling={isLaunchSettling} onUpdateRole={ @@ -351,6 +359,12 @@ export const MemberDetailDialog = ({ /> ) : null} + ) : effectiveLaunchInfoMessage ? ( +
+ + {effectiveLaunchInfoMessage} + +
) : runtimeEntry?.pid ? (
PID {runtimeEntry.pid} diff --git a/src/renderer/components/team/members/MemberDetailHeader.tsx b/src/renderer/components/team/members/MemberDetailHeader.tsx index 7d2563b4..cb82f777 100644 --- a/src/renderer/components/team/members/MemberDetailHeader.tsx +++ b/src/renderer/components/team/members/MemberDetailHeader.tsx @@ -38,6 +38,7 @@ interface MemberDetailHeaderProps { spawnLaunchState?: MemberLaunchState; spawnLivenessSource?: MemberSpawnLivenessSource; spawnRuntimeAlive?: boolean; + spawnBootstrapStalled?: boolean; isLaunchSettling?: boolean; onUpdateRole?: (newRole: string | undefined) => Promise | void; updatingRole?: boolean; @@ -54,6 +55,7 @@ export const MemberDetailHeader = ({ spawnLaunchState, spawnLivenessSource, spawnRuntimeAlive, + spawnBootstrapStalled, isLaunchSettling, onUpdateRole, updatingRole, @@ -79,6 +81,7 @@ export const MemberDetailHeader = ({ spawnLaunchState, spawnLivenessSource, spawnRuntimeAlive, + spawnBootstrapStalled, runtimeEntry, runtimeAdvisory: member.runtimeAdvisory, isLaunchSettling, @@ -96,7 +99,8 @@ export const MemberDetailHeader = ({ const badgeLabel = runtimeAdvisoryTone === 'error' && runtimeAdvisoryLabel ? runtimeAdvisoryLabel - : launchVisualState === 'runtime_pending' || + : launchVisualState === 'bootstrap_stalled' || + launchVisualState === 'runtime_pending' || launchVisualState === 'permission_pending' || launchVisualState === 'shell_only' || launchVisualState === 'runtime_candidate' || diff --git a/src/renderer/components/team/members/MemberHoverCard.tsx b/src/renderer/components/team/members/MemberHoverCard.tsx index 92a48c25..e701ea27 100644 --- a/src/renderer/components/team/members/MemberHoverCard.tsx +++ b/src/renderer/components/team/members/MemberHoverCard.tsx @@ -130,6 +130,7 @@ export const MemberHoverCard = ({ spawnLaunchState: spawnEntry?.launchState, spawnLivenessSource: spawnEntry?.livenessSource, spawnRuntimeAlive: spawnEntry?.runtimeAlive, + spawnBootstrapStalled: spawnEntry?.bootstrapStalled, runtimeEntry, runtimeAdvisory: member.runtimeAdvisory, isLaunchSettling, @@ -147,7 +148,8 @@ export const MemberHoverCard = ({ const badgeLabel = runtimeAdvisoryTone === 'error' && runtimeAdvisoryLabel ? runtimeAdvisoryLabel - : launchVisualState === 'runtime_pending' || + : launchVisualState === 'bootstrap_stalled' || + launchVisualState === 'runtime_pending' || launchVisualState === 'permission_pending' || launchVisualState === 'shell_only' || launchVisualState === 'runtime_candidate' || diff --git a/src/renderer/components/team/provisioningSteps.ts b/src/renderer/components/team/provisioningSteps.ts index 8d99764e..776eaecc 100644 --- a/src/renderer/components/team/provisioningSteps.ts +++ b/src/renderer/components/team/provisioningSteps.ts @@ -65,7 +65,11 @@ function isFailedSpawnEntry(entry: MemberSpawnStatusEntry | undefined): boolean } function isStrongRuntimeProcessSpawnEntry(entry: MemberSpawnStatusEntry): boolean { - return entry.runtimeAlive === true && entry.livenessKind === 'runtime_process'; + return ( + entry.runtimeAlive === true && + entry.livenessKind === 'runtime_process' && + entry.bootstrapStalled !== true + ); } function shouldPreferSnapshotEntryOverLive( diff --git a/src/renderer/utils/memberHelpers.ts b/src/renderer/utils/memberHelpers.ts index 4ce026e0..af2d9a1d 100644 --- a/src/renderer/utils/memberHelpers.ts +++ b/src/renderer/utils/memberHelpers.ts @@ -550,6 +550,7 @@ export type MemberLaunchVisualState = | 'waiting' | 'spawning' | 'permission_pending' + | 'bootstrap_stalled' | 'runtime_pending' | 'shell_only' | 'runtime_candidate' @@ -582,6 +583,8 @@ export function getMemberLaunchStatusLabel(visualState: MemberLaunchVisualState) return 'starting'; case 'permission_pending': return 'awaiting permission'; + case 'bootstrap_stalled': + return 'bootstrap stalled'; case 'runtime_pending': return 'waiting for bootstrap'; case 'shell_only': @@ -608,6 +611,7 @@ function getLaunchVisualStateDotClass(visualState: MemberLaunchVisualState): str case 'queued': return SPAWN_DOT_COLORS.waiting; case 'permission_pending': + case 'bootstrap_stalled': case 'runtime_pending': case 'runtime_candidate': return 'bg-amber-400 animate-pulse'; @@ -710,6 +714,7 @@ export function isOpenCodeRelaunchActionable({ nowMs ); const hasExplicitBootstrapStall = + spawnEntry?.bootstrapStalled === true || hasBootstrapStallDiagnostic(spawnEntry?.runtimeDiagnostic) || hasBootstrapStallDiagnostic(runtimeEntry?.runtimeDiagnostic); const launchIsNoLongerFresh = @@ -725,6 +730,9 @@ export function isOpenCodeRelaunchActionable({ ) { return launchIsNoLongerFresh; } + if (livenessKind === 'runtime_process') { + return hasExplicitBootstrapStall; + } if (livenessKind !== 'runtime_process_candidate') { return false; } @@ -738,6 +746,7 @@ export function buildMemberLaunchPresentation({ spawnLaunchState, spawnLivenessSource, spawnRuntimeAlive, + spawnBootstrapStalled, runtimeAdvisory, runtimeEntry, isLaunchSettling = false, @@ -750,6 +759,7 @@ export function buildMemberLaunchPresentation({ spawnLaunchState: MemberLaunchState | undefined; spawnLivenessSource: MemberSpawnLivenessSource | undefined; spawnRuntimeAlive: boolean | undefined; + spawnBootstrapStalled?: boolean; runtimeAdvisory: MemberRuntimeAdvisory | undefined; runtimeEntry?: TeamAgentRuntimeEntry; isLaunchSettling?: boolean; @@ -800,6 +810,8 @@ export function buildMemberLaunchPresentation({ launchVisualState = 'skipped'; } else if (spawnLaunchState === 'runtime_pending_permission') { launchVisualState = 'permission_pending'; + } else if (spawnBootstrapStalled === true) { + launchVisualState = 'bootstrap_stalled'; } else if (runtimeEntry?.livenessKind === 'shell_only') { launchVisualState = 'shell_only'; } else if (runtimeEntry?.livenessKind === 'runtime_process_candidate') { @@ -851,6 +863,7 @@ export function buildMemberLaunchPresentation({ const shouldShowLaunchStatusAsPresence = launchVisualState === 'queued' || launchVisualState === 'permission_pending' || + launchVisualState === 'bootstrap_stalled' || launchVisualState === 'runtime_pending' || launchVisualState === 'shell_only' || launchVisualState === 'runtime_candidate' || diff --git a/src/renderer/utils/memberLaunchDiagnostics.ts b/src/renderer/utils/memberLaunchDiagnostics.ts index 97f04069..b09b70b1 100644 --- a/src/renderer/utils/memberLaunchDiagnostics.ts +++ b/src/renderer/utils/memberLaunchDiagnostics.ts @@ -27,6 +27,7 @@ export interface MemberLaunchDiagnosticsPayload { runtimeSessionId?: string; runtimeDiagnostic?: string; runtimeDiagnosticSeverity?: TeamAgentRuntimeDiagnosticSeverity; + bootstrapStalled?: boolean; diagnostics?: string[]; updatedAt?: string; } @@ -139,6 +140,7 @@ export function buildMemberLaunchDiagnosticsPayload(params: { spawnEntry?.runtimeDiagnosticSeverity ?? runtimeEntry?.runtimeDiagnosticSeverity, } : {}), + ...(spawnEntry?.bootstrapStalled === true ? { bootstrapStalled: true } : {}), ...(diagnostics ? { diagnostics } : {}), ...(boundedString(spawnEntry?.updatedAt ?? runtimeEntry?.updatedAt) ? { updatedAt: boundedString(spawnEntry?.updatedAt ?? runtimeEntry?.updatedAt) } @@ -159,6 +161,7 @@ export function hasMemberLaunchDiagnosticsDetails( return Boolean( (payload.launchState && payload.launchState !== 'confirmed_alive') || (payload.spawnStatus && payload.spawnStatus !== 'online') || + payload.bootstrapStalled === true || weakLiveness || payload.runtimeDiagnostic || payload.diagnostics?.length diff --git a/src/renderer/utils/teamProvisioningPresentation.ts b/src/renderer/utils/teamProvisioningPresentation.ts index f51b6e46..58b22116 100644 --- a/src/renderer/utils/teamProvisioningPresentation.ts +++ b/src/renderer/utils/teamProvisioningPresentation.ts @@ -42,6 +42,7 @@ interface SkippedSpawnDetail { } type PendingDiagnosticBucket = + | 'bootstrapStalled' | 'shellOnly' | 'runtimeProcess' | 'runtimeCandidate' @@ -182,6 +183,7 @@ function getPendingDiagnosticNameGroups(params: { memberSpawnSnapshotUpdatedAt?: string; }): PendingDiagnosticNameGroups { const groups: PendingDiagnosticNameGroups = { + bootstrapStalled: [], shellOnly: [], runtimeProcess: [], runtimeCandidate: [], @@ -215,6 +217,10 @@ function getPendingDiagnosticNameGroups(params: { groups.permission.push(name); continue; } + if (entry.bootstrapStalled === true) { + groups.bootstrapStalled.push(name); + continue; + } if (entry.livenessKind === 'shell_only') { groups.shellOnly.push(name); } else if (entry.livenessKind === 'runtime_process') { @@ -288,9 +294,24 @@ function buildOpenCodeSecondaryWaitPhrase(params: { const pendingOnlyOpenCodeSecondary = pendingNames.every((name) => isOpenCodeSecondaryMember(memberByName.get(name)) ); - return pendingOnlyOpenCodeSecondary - ? `Waiting for OpenCode: ${formatMemberNameList(pendingNames)}` - : null; + if (!pendingOnlyOpenCodeSecondary) { + return null; + } + + const groups = getPendingDiagnosticNameGroups({ + memberSpawnStatuses: params.memberSpawnStatuses, + memberSpawnSnapshotStatuses: params.memberSpawnSnapshotStatuses, + memberSpawnSnapshotUpdatedAt: params.memberSpawnSnapshotUpdatedAt, + }); + if (groups.bootstrapStalled.length === 0) { + return `Waiting for OpenCode: ${formatMemberNameList(pendingNames)}`; + } + + const stalled = `Bootstrap stalled: ${formatMemberNameList(groups.bootstrapStalled)}`; + const waitingNames = pendingNames.filter((name) => !groups.bootstrapStalled.includes(name)); + return waitingNames.length > 0 + ? `${stalled}; Waiting for OpenCode: ${formatMemberNameList(waitingNames)}` + : stalled; } function formatNamedPendingDiagnostic(label: string, names: readonly string[]): string | null { @@ -323,6 +344,7 @@ function buildPendingDiagnosticPhrase({ memberSpawnSnapshotUpdatedAt, }); const namedParts = [ + formatNamedPendingDiagnostic('Bootstrap stalled', groups.bootstrapStalled), formatNamedPendingDiagnostic('Shell-only', groups.shellOnly), formatNamedPendingDiagnostic('Waiting for bootstrap', groups.runtimeProcess), formatNamedPendingDiagnostic('Bootstrap unconfirmed', groups.runtimeCandidate), diff --git a/src/shared/types/team.ts b/src/shared/types/team.ts index d1371c26..63bdd748 100644 --- a/src/shared/types/team.ts +++ b/src/shared/types/team.ts @@ -1022,6 +1022,8 @@ export interface PersistedTeamLaunchMemberState { pidSource?: TeamAgentRuntimePidSource; runtimeDiagnostic?: string; runtimeDiagnosticSeverity?: TeamAgentRuntimeDiagnosticSeverity; + /** True when a live runtime process missed the bounded bootstrap check-in window. */ + bootstrapStalled?: boolean; runtimeLastSeenAt?: string; firstSpawnAcceptedAt?: string; lastHeartbeatAt?: string; @@ -1199,6 +1201,8 @@ export interface MemberSpawnStatusEntry { runtimeDiagnostic?: string; /** Visual severity for runtimeDiagnostic. */ runtimeDiagnosticSeverity?: TeamAgentRuntimeDiagnosticSeverity; + /** Process is alive, but bootstrap did not confirm before the bounded OpenCode deadline. */ + bootstrapStalled?: boolean; /** ISO timestamp of the last liveness evaluation. */ livenessLastCheckedAt?: string; /** ISO timestamp of the last status change. */ diff --git a/test/main/services/team/TeamAgentLaunchMatrix.safe-e2e.test.ts b/test/main/services/team/TeamAgentLaunchMatrix.safe-e2e.test.ts index f32a1e75..753a7a5d 100644 --- a/test/main/services/team/TeamAgentLaunchMatrix.safe-e2e.test.ts +++ b/test/main/services/team/TeamAgentLaunchMatrix.safe-e2e.test.ts @@ -14796,7 +14796,7 @@ describe('Team agent launch matrix safe e2e', () => { expect(adapter.launchInputs).toHaveLength(0); expect(run.mixedSecondaryLanes.map((lane: { state: string }) => lane.state)).toEqual([ 'launching', - 'launching', + 'queued', ]); expect(run.mixedSecondaryLanes.map((lane: { runId: string | null }) => lane.runId)).toEqual( firstLaneRunIds @@ -14847,7 +14847,7 @@ describe('Team agent launch matrix safe e2e', () => { expect(adapter.launchInputs).toHaveLength(0); expect(run.mixedSecondaryLanes.map((lane: { state: string }) => lane.state)).toEqual([ 'launching', - 'launching', + 'queued', ]); expect(run.mixedSecondaryLanes.map((lane: { runId: string | null }) => lane.runId)).toEqual( firstLaneRunIds diff --git a/test/main/services/team/TeamLaunchStateEvaluator.test.ts b/test/main/services/team/TeamLaunchStateEvaluator.test.ts index 106e2663..70051180 100644 --- a/test/main/services/team/TeamLaunchStateEvaluator.test.ts +++ b/test/main/services/team/TeamLaunchStateEvaluator.test.ts @@ -223,6 +223,86 @@ describe('TeamLaunchStateEvaluator', () => { }); }); + it('keeps bootstrap-stalled runtime processes pending instead of online', () => { + const snapshot = normalizePersistedLaunchSnapshot('my-team', { + version: 2, + teamName: 'my-team', + updatedAt: '2026-04-23T00:00:00.000Z', + launchPhase: 'active', + expectedMembers: ['alice'], + members: { + alice: { + name: 'alice', + providerId: 'opencode', + laneKind: 'secondary', + laneOwnerProviderId: 'opencode', + laneId: 'secondary:opencode:alice', + launchState: 'runtime_pending_bootstrap', + agentToolAccepted: true, + runtimeAlive: true, + bootstrapConfirmed: false, + hardFailure: false, + livenessKind: 'runtime_process', + bootstrapStalled: true, + runtimeDiagnostic: 'Runtime process is alive, but no bootstrap check-in after 5 min.', + runtimeDiagnosticSeverity: 'warning', + lastEvaluatedAt: '2026-04-23T00:00:00.000Z', + }, + }, + }); + + expect(snapshot?.members.alice.bootstrapStalled).toBe(true); + expect(snapshot?.teamLaunchState).toBe('partial_pending'); + + const statuses = snapshotToMemberSpawnStatuses(snapshot); + expect(statuses.alice).toMatchObject({ + status: 'waiting', + launchState: 'runtime_pending_bootstrap', + runtimeAlive: true, + livenessSource: undefined, + livenessKind: 'runtime_process', + bootstrapStalled: true, + }); + }); + + it('keeps OpenCode secondary runtime processes pending before bootstrap stalls', () => { + const snapshot = normalizePersistedLaunchSnapshot('my-team', { + version: 2, + teamName: 'my-team', + updatedAt: '2026-04-23T00:00:00.000Z', + launchPhase: 'active', + expectedMembers: ['alice'], + members: { + alice: { + name: 'alice', + providerId: 'opencode', + laneKind: 'secondary', + laneOwnerProviderId: 'opencode', + laneId: 'secondary:opencode:alice', + launchState: 'runtime_pending_bootstrap', + agentToolAccepted: true, + runtimeAlive: true, + bootstrapConfirmed: false, + hardFailure: false, + livenessKind: 'runtime_process', + runtimeDiagnostic: 'OpenCode runtime process detected', + runtimeDiagnosticSeverity: 'info', + lastEvaluatedAt: '2026-04-23T00:00:00.000Z', + }, + }, + }); + + const statuses = snapshotToMemberSpawnStatuses(snapshot); + expect(statuses.alice).toMatchObject({ + status: 'waiting', + launchState: 'runtime_pending_bootstrap', + runtimeAlive: true, + livenessSource: undefined, + livenessKind: 'runtime_process', + bootstrapStalled: false, + }); + }); + it('normalizes stale persisted runtimeAlive to false without strong liveness evidence', () => { const snapshot = normalizePersistedLaunchSnapshot('demo', { version: 2, diff --git a/test/main/services/team/TeamProvisioningService.test.ts b/test/main/services/team/TeamProvisioningService.test.ts index 4c89c11f..2bbe8eaa 100644 --- a/test/main/services/team/TeamProvisioningService.test.ts +++ b/test/main/services/team/TeamProvisioningService.test.ts @@ -431,6 +431,7 @@ function createMemberSpawnRun(params?: { expectedMembers?: string[]; memberSpawnStatuses?: Map>; memberSpawnLeadInboxCursorByMember?: Map; + mixedSecondaryLanes?: Array<{ providerId: string; member: { name: string } }>; }) { const teamName = params?.teamName ?? 'member-spawn-team'; const expectedMembers = params?.expectedMembers ?? ['alice']; @@ -452,6 +453,7 @@ function createMemberSpawnRun(params?: { request: { members: [], }, + mixedSecondaryLanes: params?.mixedSecondaryLanes ?? [], expectedMembers, memberSpawnStatuses, memberSpawnToolUseIds: new Map(), @@ -9202,6 +9204,7 @@ describe('TeamProvisioningService', () => { const run = createMemberSpawnRun({ teamName: 'codex-team', expectedMembers: ['bob'], + mixedSecondaryLanes: [{ providerId: 'opencode', member: { name: 'bob' } }], memberSpawnStatuses: new Map([ [ 'bob', @@ -9235,14 +9238,67 @@ describe('TeamProvisioningService', () => { await (svc as any).reevaluateMemberLaunchStatus(run, 'bob'); expect(run.memberSpawnStatuses.get('bob')).toMatchObject({ - status: 'online', + status: 'waiting', launchState: 'runtime_pending_bootstrap', runtimeAlive: true, bootstrapConfirmed: false, - livenessSource: 'process', + livenessSource: undefined, livenessKind: 'runtime_process', runtimeDiagnostic: 'Runtime process is alive, but no bootstrap check-in after 5 min.', runtimeDiagnosticSeverity: 'warning', + bootstrapStalled: true, + hardFailure: false, + }); + }); + + it('keeps OpenCode runtime process pending before the bootstrap stall window', async () => { + const svc = new TeamProvisioningService(); + const run = createMemberSpawnRun({ + teamName: 'codex-team', + expectedMembers: ['bob'], + mixedSecondaryLanes: [{ providerId: 'opencode', member: { name: 'bob' } }], + memberSpawnStatuses: new Map([ + [ + 'bob', + createMemberSpawnStatusEntry({ + status: 'waiting', + launchState: 'runtime_pending_bootstrap', + agentToolAccepted: true, + runtimeAlive: false, + bootstrapConfirmed: false, + firstSpawnAcceptedAt: new Date(Date.now() - 60_000).toISOString(), + }), + ], + ]), + }); + (svc as any).refreshMemberSpawnStatusesFromLeadInbox = vi.fn(async () => {}); + (svc as any).maybeAuditMemberSpawnStatuses = vi.fn(async () => {}); + (svc as any).getLiveTeamAgentRuntimeMetadata = vi.fn( + async () => + new Map([ + [ + 'bob', + { + alive: true, + livenessKind: 'runtime_process', + runtimeDiagnostic: 'OpenCode runtime process detected', + }, + ], + ]) + ); + + await (svc as any).reevaluateMemberLaunchStatus(run, 'bob'); + + expect(run.memberSpawnStatuses.get('bob')).toMatchObject({ + status: 'waiting', + launchState: 'runtime_pending_bootstrap', + runtimeAlive: true, + bootstrapConfirmed: false, + livenessSource: undefined, + livenessKind: 'runtime_process', + runtimeDiagnostic: 'OpenCode runtime process detected', + runtimeDiagnosticSeverity: 'info', + bootstrapStalled: undefined, hardFailure: false, }); }); @@ -12894,6 +12950,101 @@ describe('TeamProvisioningService', () => { }); }); + it('keeps OpenCode secondary pending-bootstrap status waiting when live runtime process is attached', async () => { + const svc = new TeamProvisioningService(); + (svc as any).getLiveTeamAgentRuntimeMetadata = vi.fn( + async () => + new Map([ + [ + 'tom', + { + alive: true, + model: 'openrouter/minimax/minimax-m2.5', + livenessKind: 'runtime_process', + providerId: 'opencode', + runtimeDiagnostic: 'OpenCode runtime process detected', + runtimeDiagnosticSeverity: 'info', + }, + ], + ]) + ); + + const result = await (svc as any).attachLiveRuntimeMetadataToStatuses( + 'beacon-desk-4', + { + tom: createMemberSpawnStatusEntry({ + status: 'waiting', + launchState: 'runtime_pending_bootstrap', + agentToolAccepted: true, + runtimeAlive: true, + bootstrapConfirmed: false, + hardFailure: false, + }), + }, + { openCodeSecondaryBootstrapPendingMembers: new Set(['tom']) } + ); + + expect(result.tom).toMatchObject({ + status: 'waiting', + launchState: 'runtime_pending_bootstrap', + runtimeAlive: true, + bootstrapConfirmed: false, + livenessSource: undefined, + livenessKind: 'runtime_process', + runtimeModel: 'openrouter/minimax/minimax-m2.5', + runtimeDiagnostic: 'OpenCode runtime process detected', + runtimeDiagnosticSeverity: 'info', + }); + }); + + it('marks stale OpenCode secondary pending-bootstrap status stalled when live runtime is attached after restart', async () => { + const svc = new TeamProvisioningService(); + (svc as any).getLiveTeamAgentRuntimeMetadata = vi.fn( + async () => + new Map([ + [ + 'tom', + { + alive: true, + model: 'openrouter/minimax/minimax-m2.5', + livenessKind: 'runtime_process', + providerId: 'opencode', + runtimeDiagnostic: 'OpenCode runtime process detected', + runtimeDiagnosticSeverity: 'info', + }, + ], + ]) + ); + + const result = await (svc as any).attachLiveRuntimeMetadataToStatuses( + 'beacon-desk-4', + { + tom: createMemberSpawnStatusEntry({ + status: 'waiting', + launchState: 'runtime_pending_bootstrap', + agentToolAccepted: true, + runtimeAlive: true, + bootstrapConfirmed: false, + hardFailure: false, + firstSpawnAcceptedAt: new Date(Date.now() - 6 * 60_000).toISOString(), + }), + }, + { openCodeSecondaryBootstrapPendingMembers: new Set(['tom']) } + ); + + expect(result.tom).toMatchObject({ + status: 'waiting', + launchState: 'runtime_pending_bootstrap', + runtimeAlive: true, + bootstrapConfirmed: false, + livenessSource: undefined, + livenessKind: 'runtime_process', + bootstrapStalled: true, + runtimeDiagnostic: 'Runtime process is alive, but no bootstrap check-in after 5 min.', + runtimeDiagnosticSeverity: 'warning', + }); + }); + it('keeps process table diagnostics visible when live metadata has no primary diagnostic', async () => { const svc = new TeamProvisioningService(); (svc as any).getLiveTeamAgentRuntimeMetadata = vi.fn( diff --git a/test/renderer/components/team/members/MemberDetailDialog.test.ts b/test/renderer/components/team/members/MemberDetailDialog.test.ts index 1243c6b7..78f905f3 100644 --- a/test/renderer/components/team/members/MemberDetailDialog.test.ts +++ b/test/renderer/components/team/members/MemberDetailDialog.test.ts @@ -413,4 +413,82 @@ describe('MemberDetailDialog activity count', () => { await Promise.resolve(); }); }); + + it('shows Relaunch OpenCode copy for stalled OpenCode bootstrap', async () => { + const member: ResolvedTeamMember = { + name: 'tom', + status: 'active', + currentTaskId: null, + taskCount: 0, + lastActiveAt: null, + messageCount: 0, + providerId: 'opencode', + }; + const onRestartMember = vi.fn(async () => undefined); + const host = document.createElement('div'); + document.body.appendChild(host); + const root = createRoot(host); + + await act(async () => { + root.render( + React.createElement(MemberDetailDialog, { + open: true, + member, + teamName: 'demo-team', + members: [member], + tasks: [], + isTeamAlive: true, + spawnEntry: { + status: 'waiting', + launchState: 'runtime_pending_bootstrap', + runtimeAlive: true, + bootstrapConfirmed: false, + hardFailure: false, + agentToolAccepted: true, + livenessKind: 'runtime_process', + runtimeDiagnostic: 'Runtime process is alive, but no bootstrap check-in after 5 min.', + runtimeDiagnosticSeverity: 'warning', + bootstrapStalled: true, + updatedAt: '2026-04-24T12:05:00.000Z', + }, + runtimeEntry: { + memberName: 'tom', + alive: true, + restartable: true, + providerId: 'opencode', + livenessKind: 'runtime_process', + runtimeDiagnostic: 'OpenCode runtime process detected', + runtimeDiagnosticSeverity: 'info', + updatedAt: '2026-04-24T12:05:01.000Z', + }, + onClose: () => undefined, + onSendMessage: () => undefined, + onAssignTask: () => undefined, + onTaskClick: () => undefined, + onRestartMember, + }) + ); + await Promise.resolve(); + }); + + expect(host.textContent).toContain( + 'OpenCode process is alive, but bootstrap did not confirm. Relaunch this teammate to start a fresh OpenCode session.' + ); + const relaunchButton = Array.from(host.querySelectorAll('button')).find((button) => + button.textContent?.includes('Relaunch OpenCode') + ); + expect(relaunchButton).not.toBeUndefined(); + + await act(async () => { + relaunchButton?.click(); + await Promise.resolve(); + }); + + expect(onRestartMember).toHaveBeenCalledWith('tom'); + + await act(async () => { + root.unmount(); + await Promise.resolve(); + }); + }); }); diff --git a/test/renderer/components/team/provisioningSteps.test.ts b/test/renderer/components/team/provisioningSteps.test.ts index b4574066..4d8830bd 100644 --- a/test/renderer/components/team/provisioningSteps.test.ts +++ b/test/renderer/components/team/provisioningSteps.test.ts @@ -65,6 +65,25 @@ describe('getLaunchJoinMilestonesFromMembers', () => { expect(milestones.pendingSpawnCount).toBe(4); }); + it('keeps bootstrap-stalled runtime processes out of process-alive progress', () => { + const milestones = getLaunchJoinMilestonesFromMembers({ + members, + memberSpawnStatuses: { + alice: { + status: 'waiting', + launchState: 'runtime_pending_bootstrap', + runtimeAlive: true, + livenessKind: 'runtime_process', + bootstrapStalled: true, + updatedAt: '2026-04-24T12:05:00.000Z', + }, + }, + }); + + expect(milestones.processOnlyAliveCount).toBe(0); + expect(milestones.pendingSpawnCount).toBe(4); + }); + it('uses runtimeProcessPendingCount instead of legacy runtimeAlivePendingCount for snapshot pending math', () => { const milestones = getLaunchJoinMilestonesFromMembers({ members, diff --git a/test/renderer/utils/memberHelpers.test.ts b/test/renderer/utils/memberHelpers.test.ts index 8ccc48ac..5cb99d59 100644 --- a/test/renderer/utils/memberHelpers.test.ts +++ b/test/renderer/utils/memberHelpers.test.ts @@ -303,6 +303,26 @@ describe('memberHelpers spawn-aware presence', () => { expect(permissionPending.cardClass).toContain('member-waiting-shimmer'); }); + it('surfaces bootstrap-stalled OpenCode teammates as actionable pending state', () => { + const bootstrapStalled = buildMemberLaunchPresentation({ + member: { ...member, providerId: 'opencode' }, + spawnStatus: 'waiting', + spawnLaunchState: 'runtime_pending_bootstrap', + spawnLivenessSource: undefined, + spawnRuntimeAlive: true, + spawnBootstrapStalled: true, + runtimeAdvisory: undefined, + isLaunchSettling: false, + isTeamAlive: true, + isTeamProvisioning: false, + }); + + expect(bootstrapStalled.presenceLabel).toBe('bootstrap stalled'); + expect(bootstrapStalled.launchVisualState).toBe('bootstrap_stalled'); + expect(bootstrapStalled.launchStatusLabel).toBe('bootstrap stalled'); + expect(bootstrapStalled.dotClass).toContain('bg-amber-400'); + }); + it('surfaces strict runtime liveness diagnostics as launch labels', () => { expect( buildMemberLaunchPresentation({ diff --git a/test/renderer/utils/teamProvisioningPresentation.test.ts b/test/renderer/utils/teamProvisioningPresentation.test.ts index a07043b8..38f03e30 100644 --- a/test/renderer/utils/teamProvisioningPresentation.test.ts +++ b/test/renderer/utils/teamProvisioningPresentation.test.ts @@ -667,6 +667,84 @@ describe('buildTeamProvisioningPresentation', () => { expect(presentation?.currentStepIndex).toBe(2); }); + it('shows stalled OpenCode secondaries separately from normal bootstrap waiting', () => { + const presentation = buildTeamProvisioningPresentation({ + progress: { + runId: 'run-opencode-secondary-stalled', + teamName: 'mixed-team', + state: 'ready', + startedAt: '2026-04-13T10:00:00.000Z', + updatedAt: '2026-04-13T10:05:08.000Z', + message: 'Team provisioned - waiting for secondary runtime lane: tom', + messageSeverity: undefined, + pid: 4321, + cliLogsTail: '', + assistantOutput: '', + }, + members: [ + { + name: 'team-lead', + agentType: 'team-lead', + providerId: 'codex', + status: 'active', + currentTaskId: null, + taskCount: 0, + lastActiveAt: null, + messageCount: 0, + }, + { + name: 'alice', + providerId: 'codex', + laneKind: 'primary', + status: 'active', + currentTaskId: null, + taskCount: 0, + lastActiveAt: null, + messageCount: 0, + }, + { + name: 'tom', + providerId: 'opencode', + laneId: 'secondary:opencode:tom', + laneKind: 'secondary', + laneOwnerProviderId: 'opencode', + status: 'unknown', + currentTaskId: null, + taskCount: 0, + lastActiveAt: null, + messageCount: 0, + }, + ], + memberSpawnStatuses: { + alice: { + status: 'online', + launchState: 'confirmed_alive', + updatedAt: '2026-04-13T10:00:05.000Z', + runtimeAlive: true, + bootstrapConfirmed: true, + hardFailure: false, + agentToolAccepted: true, + }, + tom: { + status: 'waiting', + launchState: 'runtime_pending_bootstrap', + updatedAt: '2026-04-13T10:05:07.000Z', + runtimeAlive: true, + livenessKind: 'runtime_process', + bootstrapConfirmed: false, + hardFailure: false, + agentToolAccepted: true, + bootstrapStalled: true, + }, + }, + }); + + expect(presentation?.successMessage).toBe('Core team ready'); + expect(presentation?.panelMessage).toBe('Bootstrap stalled: tom'); + expect(presentation?.compactDetail).toBe('Bootstrap stalled: tom'); + expect(presentation?.currentStepIndex).toBe(2); + }); + it('does not show core team ready while a primary member is still joining', () => { const presentation = buildTeamProvisioningPresentation({ progress: {