Refine launch status and Gemini staging flow

This commit is contained in:
iliya 2026-04-05 12:08:13 +03:00
parent e9c08c4b6d
commit 2959a3d074
7 changed files with 397 additions and 70 deletions

View file

@ -106,6 +106,7 @@ import type {
CrossTeamSendResult, CrossTeamSendResult,
InboxMessage, InboxMessage,
LeadContextUsage, LeadContextUsage,
MemberLaunchState,
MemberSpawnStatus, MemberSpawnStatus,
MemberSpawnLivenessSource, MemberSpawnLivenessSource,
MemberSpawnStatusEntry, MemberSpawnStatusEntry,
@ -591,8 +592,16 @@ interface ProvisioningRun {
string, string,
{ {
status: MemberSpawnStatus; status: MemberSpawnStatus;
launchState: MemberLaunchState;
error?: string; error?: string;
hardFailureReason?: string;
livenessSource?: MemberSpawnLivenessSource; livenessSource?: MemberSpawnLivenessSource;
agentToolAccepted?: boolean;
runtimeAlive?: boolean;
bootstrapConfirmed?: boolean;
hardFailure?: boolean;
firstSpawnAcceptedAt?: string;
lastHeartbeatAt?: string;
updatedAt: string; updatedAt: string;
} }
>; >;
@ -620,10 +629,44 @@ interface PromptSizeSummary {
lines: number; lines: number;
} }
const MEMBER_LAUNCH_GRACE_MS = 90_000;
function nowIso(): string { function nowIso(): string {
return new Date().toISOString(); return new Date().toISOString();
} }
function createInitialMemberSpawnStatusEntry(): MemberSpawnStatusEntry {
const updatedAt = nowIso();
return {
status: 'offline',
launchState: 'starting',
agentToolAccepted: false,
runtimeAlive: false,
bootstrapConfirmed: false,
hardFailure: false,
updatedAt,
};
}
function deriveMemberLaunchState(entry: {
status: MemberSpawnStatus;
agentToolAccepted?: boolean;
runtimeAlive?: boolean;
bootstrapConfirmed?: boolean;
hardFailure?: boolean;
}): MemberLaunchState {
if (entry.hardFailure || entry.status === 'error') {
return 'failed_to_start';
}
if (entry.bootstrapConfirmed) {
return 'confirmed_alive';
}
if (entry.runtimeAlive || entry.agentToolAccepted || entry.status === 'waiting') {
return 'runtime_pending_bootstrap';
}
return 'starting';
}
function sleep(ms: number): Promise<void> { function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms)); return new Promise((resolve) => setTimeout(resolve, ms));
} }
@ -1720,15 +1763,17 @@ function buildGeminiPostLaunchHydrationPrompt(
.map((member) => { .map((member) => {
const status = run.memberSpawnStatuses.get(member.name); const status = run.memberSpawnStatuses.get(member.name);
const label = const label =
status?.status === 'error' status?.launchState === 'failed_to_start'
? `failed to start${status.error ? `${status.error}` : ''}` ? `failed to start${status.hardFailureReason ? `${status.hardFailureReason}` : status.error ? `${status.error}` : ''}`
: status?.status === 'online' : status?.launchState === 'confirmed_alive'
? 'bootstrap confirmed' ? 'bootstrap confirmed'
: status?.status === 'waiting' : status?.runtimeAlive
? 'runtime started, bootstrap pending' ? 'runtime started, bootstrap pending'
: status?.status === 'spawning' : status?.launchState === 'runtime_pending_bootstrap'
? 'spawn in progress' ? 'spawn accepted, bootstrap pending'
: 'runtime state unclear'; : status?.status === 'spawning'
? 'spawn in progress'
: 'runtime state unclear';
return `- @${member.name}: ${label}`; return `- @${member.name}: ${label}`;
}) })
.join('\n')}\n` .join('\n')}\n`
@ -2580,6 +2625,44 @@ export class TeamProvisioningService {
} }
} }
private async refreshMemberSpawnStatusesFromLeadInbox(run: ProvisioningRun): Promise<void> {
const leadName = this.getRunLeadName(run);
let leadInboxMessages: Awaited<ReturnType<TeamInboxReader['getMessagesFor']>> = [];
try {
leadInboxMessages = await this.inboxReader.getMessagesFor(run.teamName, leadName);
} catch {
return;
}
const runStartedAtMs = Date.parse(run.startedAt);
const teammateMessages = leadInboxMessages
.filter((message) => {
const from = typeof message.from === 'string' ? message.from.trim() : '';
if (!from || from === leadName || from === 'user' || from === 'system') return false;
if (!run.expectedMembers.includes(from)) return false;
const messageTs = Date.parse(message.timestamp);
if (
Number.isFinite(messageTs) &&
Number.isFinite(runStartedAtMs) &&
messageTs < runStartedAtMs
) {
return false;
}
return typeof message.text === 'string' && message.text.trim().length > 0;
})
.sort((a, b) => Date.parse(a.timestamp) - Date.parse(b.timestamp));
for (const message of teammateMessages) {
const from = message.from.trim();
const reason = extractBootstrapFailureReason(message.text);
if (reason) {
this.setMemberSpawnStatus(run, from, 'error', reason);
continue;
}
this.setMemberSpawnStatus(run, from, 'online', undefined, 'heartbeat');
}
}
private persistSentMessage(teamName: string, message: InboxMessage): void { private persistSentMessage(teamName: string, message: InboxMessage): void {
try { try {
createController({ createController({
@ -3008,20 +3091,65 @@ export class TeamProvisioningService {
error?: string, error?: string,
livenessSource?: MemberSpawnLivenessSource livenessSource?: MemberSpawnLivenessSource
): void { ): void {
const prev = run.memberSpawnStatuses.get(memberName); const prev = run.memberSpawnStatuses.get(memberName) ?? createInitialMemberSpawnStatusEntry();
const updatedAt = nowIso();
const next: MemberSpawnStatusEntry = {
...prev,
status,
updatedAt,
};
if (status === 'spawning') {
next.launchState = 'starting';
} else if (status === 'waiting') {
next.agentToolAccepted = true;
next.hardFailure = false;
next.error = undefined;
next.hardFailureReason = undefined;
next.firstSpawnAcceptedAt = prev.firstSpawnAcceptedAt ?? updatedAt;
next.launchState = 'runtime_pending_bootstrap';
} else if (status === 'online') {
next.agentToolAccepted = true;
next.runtimeAlive = true;
next.livenessSource = livenessSource;
next.firstSpawnAcceptedAt = prev.firstSpawnAcceptedAt ?? updatedAt;
if (livenessSource === 'heartbeat') {
next.bootstrapConfirmed = true;
next.lastHeartbeatAt = updatedAt;
}
next.hardFailure = false;
next.error = undefined;
next.hardFailureReason = undefined;
next.launchState = deriveMemberLaunchState(next);
} else if (status === 'error') {
next.error = error;
next.hardFailure = true;
next.hardFailureReason = error;
next.launchState = 'failed_to_start';
} else if (status === 'offline') {
Object.assign(next, createInitialMemberSpawnStatusEntry(), { updatedAt });
}
next.launchState = deriveMemberLaunchState(next);
if ( if (
prev?.status === status && prev.status === next.status &&
prev?.error === error && prev.launchState === next.launchState &&
prev?.livenessSource === livenessSource prev.error === next.error &&
prev.hardFailureReason === next.hardFailureReason &&
prev.livenessSource === next.livenessSource &&
prev.agentToolAccepted === next.agentToolAccepted &&
prev.runtimeAlive === next.runtimeAlive &&
prev.bootstrapConfirmed === next.bootstrapConfirmed &&
prev.hardFailure === next.hardFailure &&
prev.firstSpawnAcceptedAt === next.firstSpawnAcceptedAt &&
prev.lastHeartbeatAt === next.lastHeartbeatAt
) { ) {
return; return;
} }
run.memberSpawnStatuses.set(memberName, {
status, run.memberSpawnStatuses.set(memberName, next);
error, this.syncMemberLaunchGraceCheck(run, memberName, next);
livenessSource,
updatedAt: nowIso(),
});
if (status === 'spawning') { if (status === 'spawning') {
this.appendMemberBootstrapDiagnostic(run, memberName, 'Agent tool invoked'); this.appendMemberBootstrapDiagnostic(run, memberName, 'Agent tool invoked');
} else if (status === 'waiting') { } else if (status === 'waiting') {
@ -3074,14 +3202,96 @@ export class TeamProvisioningService {
for (const [name, entry] of run.memberSpawnStatuses) { for (const [name, entry] of run.memberSpawnStatuses) {
result[name] = { result[name] = {
status: entry.status, status: entry.status,
launchState: entry.launchState,
error: entry.error, error: entry.error,
hardFailureReason: entry.hardFailureReason,
livenessSource: entry.livenessSource, livenessSource: entry.livenessSource,
agentToolAccepted: entry.agentToolAccepted,
runtimeAlive: entry.runtimeAlive,
bootstrapConfirmed: entry.bootstrapConfirmed,
hardFailure: entry.hardFailure,
firstSpawnAcceptedAt: entry.firstSpawnAcceptedAt,
lastHeartbeatAt: entry.lastHeartbeatAt,
updatedAt: entry.updatedAt, updatedAt: entry.updatedAt,
}; };
} }
return { statuses: result, runId }; return { statuses: result, runId };
} }
private getMemberLaunchGraceKey(run: ProvisioningRun, memberName: string): string {
return `member-launch-grace:${run.runId}:${memberName}`;
}
private syncMemberLaunchGraceCheck(
run: ProvisioningRun,
memberName: string,
entry: MemberSpawnStatusEntry
): void {
const key = this.getMemberLaunchGraceKey(run, memberName);
const existing = this.pendingTimeouts.get(key);
if (entry.launchState === 'failed_to_start' || entry.launchState === 'confirmed_alive') {
if (existing) {
clearTimeout(existing);
this.pendingTimeouts.delete(key);
}
return;
}
if (!entry.firstSpawnAcceptedAt) {
return;
}
const remainingMs =
Date.parse(entry.firstSpawnAcceptedAt) + MEMBER_LAUNCH_GRACE_MS - Date.now();
if (remainingMs <= 0) {
if (existing) {
clearTimeout(existing);
this.pendingTimeouts.delete(key);
}
void this.reevaluateMemberLaunchStatus(run, memberName);
return;
}
if (existing) {
return;
}
const timer = setTimeout(() => {
this.pendingTimeouts.delete(key);
void this.reevaluateMemberLaunchStatus(run, memberName);
}, remainingMs);
timer.unref?.();
this.pendingTimeouts.set(key, timer);
}
private async reevaluateMemberLaunchStatus(
run: ProvisioningRun,
memberName: string
): Promise<void> {
const current = run.memberSpawnStatuses.get(memberName);
if (!current) return;
if (
current.launchState === 'failed_to_start' ||
current.launchState === 'confirmed_alive' ||
!current.firstSpawnAcceptedAt
) {
return;
}
await this.refreshMemberSpawnStatusesFromLeadInbox(run);
await this.auditMemberSpawnStatuses(run);
const refreshed = run.memberSpawnStatuses.get(memberName);
if (!refreshed) return;
if (
refreshed.launchState === 'failed_to_start' ||
refreshed.launchState === 'confirmed_alive' ||
refreshed.runtimeAlive
) {
return;
}
this.setMemberSpawnStatus(
run,
memberName,
'error',
'Teammate did not join within the launch grace window.'
);
}
private static readonly CONTEXT_EMIT_THROTTLE_MS = 2000; private static readonly CONTEXT_EMIT_THROTTLE_MS = 2000;
private static readonly LEAD_TEXT_EMIT_THROTTLE_MS = 2000; private static readonly LEAD_TEXT_EMIT_THROTTLE_MS = 2000;
@ -4063,7 +4273,7 @@ export class TeamProvisioningService {
geminiPostLaunchHydrationSent: false, geminiPostLaunchHydrationSent: false,
suppressGeminiPostLaunchHydrationOutput: false, suppressGeminiPostLaunchHydrationOutput: false,
memberSpawnStatuses: new Map( memberSpawnStatuses: new Map(
request.members.map((m) => [m.name, { status: 'offline' as const, updatedAt: nowIso() }]) request.members.map((m) => [m.name, createInitialMemberSpawnStatusEntry()])
), ),
memberSpawnToolUseIds: new Map(), memberSpawnToolUseIds: new Map(),
progress: { progress: {
@ -4545,7 +4755,7 @@ export class TeamProvisioningService {
geminiPostLaunchHydrationSent: false, geminiPostLaunchHydrationSent: false,
suppressGeminiPostLaunchHydrationOutput: false, suppressGeminiPostLaunchHydrationOutput: false,
memberSpawnStatuses: new Map( memberSpawnStatuses: new Map(
expectedMembers.map((name) => [name, { status: 'offline' as const, updatedAt: nowIso() }]) expectedMembers.map((name) => [name, createInitialMemberSpawnStatusEntry()])
), ),
memberSpawnToolUseIds: new Map(), memberSpawnToolUseIds: new Map(),
progress: { progress: {
@ -5155,6 +5365,8 @@ export class TeamProvisioningService {
return 0; return 0;
} }
await this.refreshMemberSpawnStatusesFromLeadInbox(run);
const unread = leadInboxMessages const unread = leadInboxMessages
.filter((m): m is InboxMessage & { messageId: string } => { .filter((m): m is InboxMessage & { messageId: string } => {
if (m.read) return false; if (m.read) return false;
@ -5731,7 +5943,12 @@ export class TeamProvisioningService {
// Flag any expected member not found in config.json (excluding the lead) // Flag any expected member not found in config.json (excluding the lead)
for (const expected of run.expectedMembers) { for (const expected of run.expectedMembers) {
const current = run.memberSpawnStatuses.get(expected); const current = run.memberSpawnStatuses.get(expected);
if (current?.status === 'error' || current?.status === 'online') continue; if (
current?.launchState === 'failed_to_start' ||
current?.launchState === 'confirmed_alive'
) {
continue;
}
const matchedRuntimeNames = [...registeredNames].filter((name) => { const matchedRuntimeNames = [...registeredNames].filter((name) => {
if (name === expected) return true; if (name === expected) return true;
@ -5739,30 +5956,50 @@ export class TeamProvisioningService {
return parsed !== null && parsed.suffix >= 2 && parsed.base === expected; return parsed !== null && parsed.suffix >= 2 && parsed.base === expected;
}); });
// A teammate may intentionally stay silent after bootstrap. If Claude Code const runtimeAlive =
// registered the runtime and the OS process is still alive, treat it as
// process-confirmed running. Keep this distinct from heartbeat-confirmed online.
if (
matchedRuntimeNames.length > 0 && matchedRuntimeNames.length > 0 &&
matchedRuntimeNames.some((runtimeName) => matchedRuntimeNames.some((runtimeName) =>
this.hasLiveTeamAgentProcess(run.teamName, runtimeName) this.hasLiveTeamAgentProcess(run.teamName, runtimeName)
) );
) {
// A teammate may intentionally stay silent after bootstrap. If Claude Code
// registered the runtime and the OS process is still alive, treat it as
// process-confirmed running. Keep this distinct from heartbeat-confirmed online.
if (runtimeAlive) {
this.setMemberSpawnStatus(run, expected, 'online', undefined, 'process'); this.setMemberSpawnStatus(run, expected, 'online', undefined, 'process');
continue; continue;
} }
if (matchedRuntimeNames.length > 0) continue; if (matchedRuntimeNames.length > 0) {
if (current?.agentToolAccepted) {
this.setMemberSpawnStatus(run, expected, 'waiting');
}
continue;
}
const acceptedAtMs =
current?.firstSpawnAcceptedAt != null ? Date.parse(current.firstSpawnAcceptedAt) : NaN;
const graceExpired =
current?.agentToolAccepted === true &&
Number.isFinite(acceptedAtMs) &&
Date.now() - acceptedAtMs >= MEMBER_LAUNCH_GRACE_MS;
if (current?.agentToolAccepted && !graceExpired) {
this.setMemberSpawnStatus(run, expected, 'waiting');
continue;
}
logger.warn( logger.warn(
`[${run.teamName}] Member "${expected}" not found in config.json members after provisioning` `[${run.teamName}] Member "${expected}" not found in config.json members after provisioning`
); );
this.setMemberSpawnStatus( if (graceExpired) {
run, this.setMemberSpawnStatus(
expected, run,
'error', expected,
'Teammate not registered after provisioning — spawned incorrectly. Restart the team to fix.' 'error',
); 'Teammate not registered after provisioning within the launch grace window.'
);
}
} }
} }
@ -5802,11 +6039,43 @@ export class TeamProvisioningService {
run: ProvisioningRun run: ProvisioningRun
): { name: string; error?: string; updatedAt: string }[] { ): { name: string; error?: string; updatedAt: string }[] {
return [...run.memberSpawnStatuses.entries()] return [...run.memberSpawnStatuses.entries()]
.filter(([, entry]) => entry.status === 'error') .filter(([, entry]) => entry.launchState === 'failed_to_start')
.map(([name, entry]) => ({ name, error: entry.error, updatedAt: entry.updatedAt })) .map(([name, entry]) => ({
name,
error: entry.hardFailureReason ?? entry.error,
updatedAt: entry.updatedAt,
}))
.sort((a, b) => a.name.localeCompare(b.name)); .sort((a, b) => a.name.localeCompare(b.name));
} }
private getMemberLaunchSummary(run: ProvisioningRun): {
confirmedCount: number;
pendingCount: number;
failedCount: number;
runtimeAlivePendingCount: number;
} {
let confirmedCount = 0;
let pendingCount = 0;
let failedCount = 0;
let runtimeAlivePendingCount = 0;
for (const expected of run.expectedMembers) {
const entry = run.memberSpawnStatuses.get(expected) ?? createInitialMemberSpawnStatusEntry();
if (entry.launchState === 'confirmed_alive') {
confirmedCount += 1;
continue;
}
if (entry.launchState === 'failed_to_start') {
failedCount += 1;
continue;
}
pendingCount += 1;
if (entry.runtimeAlive) {
runtimeAlivePendingCount += 1;
}
}
return { confirmedCount, pendingCount, failedCount, runtimeAlivePendingCount };
}
private async persistPartialLaunchState(run: ProvisioningRun): Promise<void> { private async persistPartialLaunchState(run: ProvisioningRun): Promise<void> {
if (!run.isLaunch || !run.expectedMembers || run.expectedMembers.length === 0) { if (!run.isLaunch || !run.expectedMembers || run.expectedMembers.length === 0) {
if (run.isLaunch) { if (run.isLaunch) {
@ -8140,18 +8409,24 @@ export class TeamProvisioningService {
} }
// Audit: flag any expected member not registered in config.json after launch. // Audit: flag any expected member not registered in config.json after launch.
await this.refreshMemberSpawnStatusesFromLeadInbox(run);
await this.auditMemberSpawnStatuses(run); await this.auditMemberSpawnStatuses(run);
await this.persistPartialLaunchState(run); await this.persistPartialLaunchState(run);
const failedSpawnMembers = this.getFailedSpawnMembers(run); const failedSpawnMembers = this.getFailedSpawnMembers(run);
const launchSummary = this.getMemberLaunchSummary(run);
const hasSpawnFailures = failedSpawnMembers.length > 0; const hasSpawnFailures = failedSpawnMembers.length > 0;
const hasPendingBootstrap =
!hasSpawnFailures && launchSummary.pendingCount > 0 && run.expectedMembers.length > 0;
const readyMessage = hasSpawnFailures const readyMessage = hasSpawnFailures
? `Launch completed with teammate errors — ${failedSpawnMembers ? `Launch completed with teammate errors — ${failedSpawnMembers
.map((member) => member.name) .map((member) => member.name)
.join(', ')} failed to start` .join(', ')} failed to start`
: 'Team launched — process alive and ready'; : hasPendingBootstrap
? `Launch completed — ${launchSummary.confirmedCount}/${run.expectedMembers.length} teammates confirmed alive, bootstrap still pending`
: 'Team launched — process alive and ready';
const progress = updateProgress(run, 'ready', readyMessage, { const progress = updateProgress(run, 'ready', readyMessage, {
cliLogsTail: extractCliLogsFromRun(run), cliLogsTail: extractCliLogsFromRun(run),
messageSeverity: hasSpawnFailures ? 'warning' : undefined, messageSeverity: hasSpawnFailures || hasPendingBootstrap ? 'warning' : undefined,
}); });
run.onProgress(progress); run.onProgress(progress);
this.provisioningRunByTeam.delete(run.teamName); this.provisioningRunByTeam.delete(run.teamName);
@ -8171,7 +8446,7 @@ export class TeamProvisioningService {
detail: 'lead-session-sync', detail: 'lead-session-sync',
}); });
if (!hasSpawnFailures) { if (!hasSpawnFailures && !hasPendingBootstrap) {
// Fire "Team Launched" notification only for clean launches. // Fire "Team Launched" notification only for clean launches.
void this.fireTeamLaunchedNotification(run); void this.fireTeamLaunchedNotification(run);
} }
@ -8293,10 +8568,14 @@ export class TeamProvisioningService {
await this.teamMetaStore.deleteMeta(run.teamName).catch(() => {}); await this.teamMetaStore.deleteMeta(run.teamName).catch(() => {});
// Audit: flag any expected member not registered in config.json after provisioning. // Audit: flag any expected member not registered in config.json after provisioning.
await this.refreshMemberSpawnStatusesFromLeadInbox(run);
await this.auditMemberSpawnStatuses(run); await this.auditMemberSpawnStatuses(run);
await this.clearPartialLaunchState(run.teamName); await this.clearPartialLaunchState(run.teamName);
const failedSpawnMembers = this.getFailedSpawnMembers(run); const failedSpawnMembers = this.getFailedSpawnMembers(run);
const launchSummary = this.getMemberLaunchSummary(run);
const hasSpawnFailures = failedSpawnMembers.length > 0; const hasSpawnFailures = failedSpawnMembers.length > 0;
const hasPendingBootstrap =
!hasSpawnFailures && launchSummary.pendingCount > 0 && run.expectedMembers.length > 0;
const progress = updateProgress( const progress = updateProgress(
run, run,
'ready', 'ready',
@ -8304,10 +8583,12 @@ export class TeamProvisioningService {
? `Provisioning completed with teammate errors — ${failedSpawnMembers ? `Provisioning completed with teammate errors — ${failedSpawnMembers
.map((member) => member.name) .map((member) => member.name)
.join(', ')} failed to start` .join(', ')} failed to start`
: 'Team provisioned — process alive and ready', : hasPendingBootstrap
? `Team provisioned — ${launchSummary.confirmedCount}/${run.expectedMembers.length} teammates confirmed alive, bootstrap still pending`
: 'Team provisioned — process alive and ready',
{ {
cliLogsTail: extractCliLogsFromRun(run), cliLogsTail: extractCliLogsFromRun(run),
messageSeverity: hasSpawnFailures ? 'warning' : undefined, messageSeverity: hasSpawnFailures || hasPendingBootstrap ? 'warning' : undefined,
} }
); );
run.onProgress(progress); run.onProgress(progress);
@ -8328,7 +8609,7 @@ export class TeamProvisioningService {
detail: 'lead-session-sync', detail: 'lead-session-sync',
}); });
if (!hasSpawnFailures) { if (!hasSpawnFailures && !hasPendingBootstrap) {
// Fire "Team Launched" notification only for clean launches. // Fire "Team Launched" notification only for clean launches.
void this.fireTeamLaunchedNotification(run); void this.fireTeamLaunchedNotification(run);
} }
@ -8702,6 +8983,14 @@ export class TeamProvisioningService {
this.pendingTimeouts.delete(key); this.pendingTimeouts.delete(key);
} }
} }
for (const memberName of run.memberSpawnStatuses.keys()) {
const key = this.getMemberLaunchGraceKey(run, memberName);
const timer = this.pendingTimeouts.get(key);
if (timer) {
clearTimeout(timer);
this.pendingTimeouts.delete(key);
}
}
run.activeCrossTeamReplyHints = []; run.activeCrossTeamReplyHints = [];
run.pendingInboxRelayCandidates = []; run.pendingInboxRelayCandidates = [];
for (const key of Array.from(this.memberInboxRelayInFlight.keys())) { for (const key of Array.from(this.memberInboxRelayInFlight.keys())) {

View file

@ -577,14 +577,20 @@ export const TeamDetailView = ({
string, string,
{ {
status: MemberSpawnStatusEntry['status']; status: MemberSpawnStatusEntry['status'];
launchState: MemberSpawnStatusEntry['launchState'];
error?: string; error?: string;
updatedAt: string;
runtimeAlive?: boolean;
livenessSource?: MemberSpawnStatusEntry['livenessSource']; livenessSource?: MemberSpawnStatusEntry['livenessSource'];
} }
>(); >();
for (const [name, entry] of Object.entries(memberSpawnStatuses)) { for (const [name, entry] of Object.entries(memberSpawnStatuses)) {
map.set(name, { map.set(name, {
status: entry.status, status: entry.status,
launchState: entry.launchState,
error: entry.error, error: entry.error,
updatedAt: entry.updatedAt,
runtimeAlive: entry.runtimeAlive,
livenessSource: entry.livenessSource, livenessSource: entry.livenessSource,
}); });
} }

View file

@ -106,20 +106,23 @@ export const TeamProvisioningBanner = ({
const teammates = (teamMembers ?? []).filter((member) => !isLeadMember(member)); const teammates = (teamMembers ?? []).filter((member) => !isLeadMember(member));
const failedSpawnEntries = Object.entries(memberSpawnStatuses ?? {}).filter( const failedSpawnEntries = Object.entries(memberSpawnStatuses ?? {}).filter(
([, entry]) => entry.status === 'error' ([, entry]) => entry.launchState === 'failed_to_start'
); );
const failedSpawnCount = failedSpawnEntries.length; const failedSpawnCount = failedSpawnEntries.length;
const heartbeatConfirmedCount = teammates.filter((member) => { const heartbeatConfirmedCount = teammates.filter((member) => {
const entry = memberSpawnStatuses?.[member.name]; const entry = memberSpawnStatuses?.[member.name];
return entry?.status === 'online' && entry.livenessSource === 'heartbeat'; return entry?.launchState === 'confirmed_alive';
}).length; }).length;
const processOnlyAliveCount = teammates.filter((member) => { const processOnlyAliveCount = teammates.filter((member) => {
const entry = memberSpawnStatuses?.[member.name]; const entry = memberSpawnStatuses?.[member.name];
return entry?.status === 'online' && entry.livenessSource === 'process'; return entry?.launchState === 'runtime_pending_bootstrap' && entry.runtimeAlive === true;
}).length; }).length;
const awaitingHeartbeatCount = teammates.filter((member) => { const pendingSpawnCount = teammates.filter((member) => {
const entry = memberSpawnStatuses?.[member.name]; const entry = memberSpawnStatuses?.[member.name];
return entry?.status === 'waiting'; return (
entry?.launchState === 'starting' ||
(entry?.launchState === 'runtime_pending_bootstrap' && entry.runtimeAlive !== true)
);
}).length; }).length;
const allTeammatesConfirmedAlive = const allTeammatesConfirmedAlive =
teammates.length > 0 && failedSpawnCount === 0 && heartbeatConfirmedCount === teammates.length; teammates.length > 0 && failedSpawnCount === 0 && heartbeatConfirmedCount === teammates.length;
@ -132,8 +135,8 @@ export const TeamProvisioningBanner = ({
? 'Team launched — lead online' ? 'Team launched — lead online'
: allTeammatesConfirmedAlive : allTeammatesConfirmedAlive
? `Team launched — all ${teammates.length} teammates confirmed alive` ? `Team launched — all ${teammates.length} teammates confirmed alive`
: processOnlyAliveCount > 0 || awaitingHeartbeatCount > 0 : processOnlyAliveCount > 0 || pendingSpawnCount > 0
? `Team launched — ${heartbeatConfirmedCount}/${teammates.length} teammates confirmed alive${processOnlyAliveCount > 0 ? `, ${processOnlyAliveCount} runtime${processOnlyAliveCount === 1 ? '' : 's'} alive but bootstrap still pending` : ''}${awaitingHeartbeatCount > 0 ? `${processOnlyAliveCount > 0 ? ', ' : ', '}${awaitingHeartbeatCount} awaiting first heartbeat` : ''}` ? `Team launched — ${heartbeatConfirmedCount}/${teammates.length} teammates confirmed alive${processOnlyAliveCount > 0 ? `, ${processOnlyAliveCount} runtime${processOnlyAliveCount === 1 ? '' : 's'} alive but bootstrap still pending` : ''}${pendingSpawnCount > 0 ? `${processOnlyAliveCount > 0 ? ', ' : ', '}${pendingSpawnCount} still starting` : ''}`
: 'Team launched — teammate liveness is still being confirmed'; : 'Team launched — teammate liveness is still being confirmed';
return ( return (
@ -151,7 +154,11 @@ export const TeamProvisioningBanner = ({
defaultLiveOutputOpen={false} defaultLiveOutputOpen={false}
onCancel={null} onCancel={null}
successMessage={readyMessage} successMessage={readyMessage}
successMessageSeverity={failedSpawnCount > 0 ? 'warning' : 'success'} successMessageSeverity={
failedSpawnCount > 0 || processOnlyAliveCount > 0 || pendingSpawnCount > 0
? 'warning'
: 'success'
}
onDismiss={() => setDismissed(true)} onDismiss={() => setDismissed(true)}
/> />
</div> </div>

View file

@ -18,6 +18,7 @@ import { CurrentTaskIndicator } from './CurrentTaskIndicator';
import type { TaskStatusCounts } from '@renderer/utils/pathNormalize'; import type { TaskStatusCounts } from '@renderer/utils/pathNormalize';
import type { import type {
LeadActivityState, LeadActivityState,
MemberLaunchState,
MemberSpawnLivenessSource, MemberSpawnLivenessSource,
MemberSpawnStatus, MemberSpawnStatus,
ResolvedTeamMember, ResolvedTeamMember,
@ -39,6 +40,8 @@ interface MemberCardProps {
spawnStatus?: MemberSpawnStatus; spawnStatus?: MemberSpawnStatus;
spawnError?: string; spawnError?: string;
spawnLivenessSource?: MemberSpawnLivenessSource; spawnLivenessSource?: MemberSpawnLivenessSource;
spawnLaunchState?: MemberLaunchState;
spawnRuntimeAlive?: boolean;
onOpenTask?: () => void; onOpenTask?: () => void;
onOpenReviewTask?: () => void; onOpenReviewTask?: () => void;
onClick?: () => void; onClick?: () => void;
@ -61,6 +64,8 @@ export const MemberCard = ({
spawnStatus, spawnStatus,
spawnError, spawnError,
spawnLivenessSource, spawnLivenessSource,
spawnLaunchState,
spawnRuntimeAlive,
onOpenTask, onOpenTask,
onOpenReviewTask, onOpenReviewTask,
onClick, onClick,
@ -75,6 +80,7 @@ export const MemberCard = ({
const dotClass = getSpawnAwareDotClass( const dotClass = getSpawnAwareDotClass(
member, member,
spawnStatus, spawnStatus,
spawnLaunchState,
isTeamAlive, isTeamAlive,
isTeamProvisioning, isTeamProvisioning,
leadActivity leadActivity
@ -82,7 +88,9 @@ export const MemberCard = ({
const presenceLabel = getSpawnAwarePresenceLabel( const presenceLabel = getSpawnAwarePresenceLabel(
member, member,
spawnStatus, spawnStatus,
spawnLaunchState,
spawnLivenessSource, spawnLivenessSource,
spawnRuntimeAlive,
isTeamAlive, isTeamAlive,
isTeamProvisioning, isTeamProvisioning,
leadActivity leadActivity

View file

@ -15,24 +15,17 @@ import { MemberCard } from './MemberCard';
import type { TaskStatusCounts } from '@renderer/utils/pathNormalize'; import type { TaskStatusCounts } from '@renderer/utils/pathNormalize';
import type { import type {
LeadActivityState, LeadActivityState,
MemberSpawnLivenessSource, MemberSpawnStatusEntry,
MemberSpawnStatus,
ResolvedTeamMember, ResolvedTeamMember,
TeamTaskWithKanban, TeamTaskWithKanban,
} from '@shared/types'; } from '@shared/types';
export interface MemberSpawnEntry {
status: MemberSpawnStatus;
error?: string;
livenessSource?: MemberSpawnLivenessSource;
}
interface MemberListProps { interface MemberListProps {
members: ResolvedTeamMember[]; members: ResolvedTeamMember[];
memberTaskCounts?: Map<string, TaskStatusCounts>; memberTaskCounts?: Map<string, TaskStatusCounts>;
taskMap?: Map<string, TeamTaskWithKanban>; taskMap?: Map<string, TeamTaskWithKanban>;
pendingRepliesByMember?: Record<string, number>; pendingRepliesByMember?: Record<string, number>;
memberSpawnStatuses?: Map<string, MemberSpawnEntry>; memberSpawnStatuses?: Map<string, MemberSpawnStatusEntry>;
isTeamAlive?: boolean; isTeamAlive?: boolean;
isTeamProvisioning?: boolean; isTeamProvisioning?: boolean;
leadActivity?: LeadActivityState; leadActivity?: LeadActivityState;
@ -137,6 +130,8 @@ export const MemberList = ({
spawnStatus={isRemoved ? undefined : spawnEntry?.status} spawnStatus={isRemoved ? undefined : spawnEntry?.status}
spawnError={isRemoved ? undefined : spawnEntry?.error} spawnError={isRemoved ? undefined : spawnEntry?.error}
spawnLivenessSource={isRemoved ? undefined : spawnEntry?.livenessSource} spawnLivenessSource={isRemoved ? undefined : spawnEntry?.livenessSource}
spawnLaunchState={isRemoved ? undefined : spawnEntry?.launchState}
spawnRuntimeAlive={isRemoved ? undefined : spawnEntry?.runtimeAlive}
onOpenTask={!isRemoved && currentTask ? () => onOpenTask?.(currentTask) : undefined} onOpenTask={!isRemoved && currentTask ? () => onOpenTask?.(currentTask) : undefined}
onOpenReviewTask={!isRemoved && reviewTask ? () => onOpenTask?.(reviewTask) : undefined} onOpenReviewTask={!isRemoved && reviewTask ? () => onOpenTask?.(reviewTask) : undefined}
onClick={() => onMemberClick?.(member)} onClick={() => onMemberClick?.(member)}

View file

@ -7,6 +7,7 @@ import { isLeadMember } from '@shared/utils/leadDetection';
import type { import type {
LeadActivityState, LeadActivityState,
MemberLaunchState,
MemberSpawnLivenessSource, MemberSpawnLivenessSource,
MemberSpawnStatus, MemberSpawnStatus,
MemberStatus, MemberStatus,
@ -99,7 +100,7 @@ export const SPAWN_DOT_COLORS: Record<MemberSpawnStatus, string> = {
export const SPAWN_PRESENCE_LABELS: Record<MemberSpawnStatus, string> = { export const SPAWN_PRESENCE_LABELS: Record<MemberSpawnStatus, string> = {
offline: 'offline', offline: 'offline',
waiting: 'awaiting heartbeat', waiting: 'bootstrap pending',
spawning: 'starting', spawning: 'starting',
online: 'ready', online: 'ready',
error: 'spawn failed', error: 'spawn failed',
@ -112,13 +113,17 @@ export const SPAWN_PRESENCE_LABELS: Record<MemberSpawnStatus, string> = {
export function getSpawnAwareDotClass( export function getSpawnAwareDotClass(
member: ResolvedTeamMember, member: ResolvedTeamMember,
spawnStatus: MemberSpawnStatus | undefined, spawnStatus: MemberSpawnStatus | undefined,
spawnLaunchState: MemberLaunchState | undefined,
isTeamAlive?: boolean, isTeamAlive?: boolean,
isTeamProvisioning?: boolean, isTeamProvisioning?: boolean,
leadActivity?: LeadActivityState leadActivity?: LeadActivityState
): string { ): string {
if (spawnStatus === 'error') { if (spawnLaunchState === 'failed_to_start' || spawnStatus === 'error') {
return SPAWN_DOT_COLORS.error; return SPAWN_DOT_COLORS.error;
} }
if (spawnLaunchState === 'runtime_pending_bootstrap' && spawnStatus === 'online') {
return SPAWN_DOT_COLORS.spawning;
}
if (spawnStatus === 'waiting') { if (spawnStatus === 'waiting') {
return SPAWN_DOT_COLORS.waiting; return SPAWN_DOT_COLORS.waiting;
} }
@ -140,22 +145,27 @@ export function getSpawnAwareDotClass(
export function getSpawnAwarePresenceLabel( export function getSpawnAwarePresenceLabel(
member: ResolvedTeamMember, member: ResolvedTeamMember,
spawnStatus: MemberSpawnStatus | undefined, spawnStatus: MemberSpawnStatus | undefined,
spawnLaunchState: MemberLaunchState | undefined,
livenessSource: MemberSpawnLivenessSource | undefined, livenessSource: MemberSpawnLivenessSource | undefined,
runtimeAlive: boolean | undefined,
isTeamAlive?: boolean, isTeamAlive?: boolean,
isTeamProvisioning?: boolean, isTeamProvisioning?: boolean,
leadActivity?: LeadActivityState leadActivity?: LeadActivityState
): string { ): string {
if (spawnStatus === 'error') { if (spawnLaunchState === 'failed_to_start' || spawnStatus === 'error') {
return SPAWN_PRESENCE_LABELS.error; return SPAWN_PRESENCE_LABELS.error;
} }
if (spawnStatus === 'offline' && isTeamProvisioning) { if (spawnStatus === 'offline' && isTeamProvisioning) {
return 'waiting for Agent'; return 'waiting for Agent';
} }
if (spawnLaunchState === 'runtime_pending_bootstrap' && runtimeAlive) {
return 'bootstrap pending';
}
if (spawnStatus === 'waiting') { if (spawnStatus === 'waiting') {
return SPAWN_PRESENCE_LABELS.waiting; return SPAWN_PRESENCE_LABELS.waiting;
} }
if (spawnStatus === 'online' && livenessSource === 'process') { if (spawnStatus === 'online' && livenessSource === 'process') {
return 'running'; return 'bootstrap pending';
} }
if (spawnStatus && isTeamProvisioning) { if (spawnStatus && isTeamProvisioning) {
return SPAWN_PRESENCE_LABELS[spawnStatus]; return SPAWN_PRESENCE_LABELS[spawnStatus];

View file

@ -440,15 +440,12 @@ export interface AddTaskCommentRequest {
export type MemberStatus = 'active' | 'idle' | 'terminated' | 'unknown'; export type MemberStatus = 'active' | 'idle' | 'terminated' | 'unknown';
/**
* Spawn lifecycle status for a team member during team launch/reconnect.
* - offline: queued, Agent tool_use not sent yet
* - spawning: Agent tool_use sent, awaiting tool_result
* - waiting: teammate process accepted by runtime, awaiting first heartbeat/inbox signal
* - online: first heartbeat/inbox signal received
* - error: spawn failed (tool_result with error)
*/
export type MemberSpawnStatus = 'offline' | 'waiting' | 'spawning' | 'online' | 'error'; export type MemberSpawnStatus = 'offline' | 'waiting' | 'spawning' | 'online' | 'error';
export type MemberLaunchState =
| 'starting'
| 'runtime_pending_bootstrap'
| 'confirmed_alive'
| 'failed_to_start';
export type KanbanColumnId = 'todo' | 'in_progress' | 'done' | 'review' | 'approved'; export type KanbanColumnId = 'todo' | 'in_progress' | 'done' | 'review' | 'approved';
@ -610,14 +607,29 @@ export interface ProjectBranchChangeEvent {
/** Per-member spawn status entry, exposed to renderer via IPC. */ /** Per-member spawn status entry, exposed to renderer via IPC. */
export interface MemberSpawnStatusEntry { export interface MemberSpawnStatusEntry {
status: MemberSpawnStatus; status: MemberSpawnStatus;
launchState: MemberLaunchState;
/** Error message when status === 'error'. */ /** Error message when status === 'error'. */
error?: string; error?: string;
/** Hard failure reason for failed_to_start. */
hardFailureReason?: string;
/** /**
* Optional provenance for `online`. * Optional provenance for `online`.
* - heartbeat: teammate sent a real inbox/native message after bootstrap * - heartbeat: teammate sent a real inbox/native message after bootstrap
* - process: runtime process is alive, but bootstrap/first reply is not yet confirmed * - process: runtime process is alive, but bootstrap/first reply is not yet confirmed
*/ */
livenessSource?: MemberSpawnLivenessSource; livenessSource?: MemberSpawnLivenessSource;
/** Agent tool_result confirmed the spawn request was accepted by the runtime. */
agentToolAccepted?: boolean;
/** Runtime process or registered teammate runtime is currently alive. */
runtimeAlive?: boolean;
/** A real teammate heartbeat/bootstrap confirmation was observed. */
bootstrapConfirmed?: boolean;
/** Hard failure observed from spawn/bootstrap/runtime evidence. */
hardFailure?: boolean;
/** ISO timestamp of the first accepted teammate spawn for this member. */
firstSpawnAcceptedAt?: string;
/** ISO timestamp of the latest confirmed heartbeat/bootstrap message. */
lastHeartbeatAt?: string;
/** ISO timestamp of the last status change. */ /** ISO timestamp of the last status change. */
updatedAt: string; updatedAt: string;
} }