Refine launch status and Gemini staging flow
This commit is contained in:
iliya
parent
e9c08c4b6d
commit
2959a3d074
7 changed files with 397 additions and 70 deletions
|
|
@ -106,6 +106,7 @@ import type {
|
|||
CrossTeamSendResult,
|
||||
InboxMessage,
|
||||
LeadContextUsage,
|
||||
MemberLaunchState,
|
||||
MemberSpawnStatus,
|
||||
MemberSpawnLivenessSource,
|
||||
MemberSpawnStatusEntry,
|
||||
|
|
@ -591,8 +592,16 @@ interface ProvisioningRun {
|
|||
string,
|
||||
{
|
||||
status: MemberSpawnStatus;
|
||||
launchState: MemberLaunchState;
|
||||
error?: string;
|
||||
hardFailureReason?: string;
|
||||
livenessSource?: MemberSpawnLivenessSource;
|
||||
agentToolAccepted?: boolean;
|
||||
runtimeAlive?: boolean;
|
||||
bootstrapConfirmed?: boolean;
|
||||
hardFailure?: boolean;
|
||||
firstSpawnAcceptedAt?: string;
|
||||
lastHeartbeatAt?: string;
|
||||
updatedAt: string;
|
||||
}
|
||||
>;
|
||||
|
|
@ -620,10 +629,44 @@ interface PromptSizeSummary {
|
|||
lines: number;
|
||||
}
|
||||
|
||||
const MEMBER_LAUNCH_GRACE_MS = 90_000;
|
||||
|
||||
function nowIso(): string {
|
||||
return new Date().toISOString();
|
||||
}
|
||||
|
||||
function createInitialMemberSpawnStatusEntry(): MemberSpawnStatusEntry {
|
||||
const updatedAt = nowIso();
|
||||
return {
|
||||
status: 'offline',
|
||||
launchState: 'starting',
|
||||
agentToolAccepted: false,
|
||||
runtimeAlive: false,
|
||||
bootstrapConfirmed: false,
|
||||
hardFailure: false,
|
||||
updatedAt,
|
||||
};
|
||||
}
|
||||
|
||||
function deriveMemberLaunchState(entry: {
|
||||
status: MemberSpawnStatus;
|
||||
agentToolAccepted?: boolean;
|
||||
runtimeAlive?: boolean;
|
||||
bootstrapConfirmed?: boolean;
|
||||
hardFailure?: boolean;
|
||||
}): MemberLaunchState {
|
||||
if (entry.hardFailure || entry.status === 'error') {
|
||||
return 'failed_to_start';
|
||||
}
|
||||
if (entry.bootstrapConfirmed) {
|
||||
return 'confirmed_alive';
|
||||
}
|
||||
if (entry.runtimeAlive || entry.agentToolAccepted || entry.status === 'waiting') {
|
||||
return 'runtime_pending_bootstrap';
|
||||
}
|
||||
return 'starting';
|
||||
}
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
|
@ -1720,15 +1763,17 @@ function buildGeminiPostLaunchHydrationPrompt(
|
|||
.map((member) => {
|
||||
const status = run.memberSpawnStatuses.get(member.name);
|
||||
const label =
|
||||
status?.status === 'error'
|
||||
? `failed to start${status.error ? ` — ${status.error}` : ''}`
|
||||
: status?.status === 'online'
|
||||
status?.launchState === 'failed_to_start'
|
||||
? `failed to start${status.hardFailureReason ? ` — ${status.hardFailureReason}` : status.error ? ` — ${status.error}` : ''}`
|
||||
: status?.launchState === 'confirmed_alive'
|
||||
? 'bootstrap confirmed'
|
||||
: status?.status === 'waiting'
|
||||
: status?.runtimeAlive
|
||||
? 'runtime started, bootstrap pending'
|
||||
: status?.status === 'spawning'
|
||||
? 'spawn in progress'
|
||||
: 'runtime state unclear';
|
||||
: status?.launchState === 'runtime_pending_bootstrap'
|
||||
? 'spawn accepted, bootstrap pending'
|
||||
: status?.status === 'spawning'
|
||||
? 'spawn in progress'
|
||||
: 'runtime state unclear';
|
||||
return `- @${member.name}: ${label}`;
|
||||
})
|
||||
.join('\n')}\n`
|
||||
|
|
@ -2580,6 +2625,44 @@ export class TeamProvisioningService {
|
|||
}
|
||||
}
|
||||
|
||||
private async refreshMemberSpawnStatusesFromLeadInbox(run: ProvisioningRun): Promise<void> {
|
||||
const leadName = this.getRunLeadName(run);
|
||||
let leadInboxMessages: Awaited<ReturnType<TeamInboxReader['getMessagesFor']>> = [];
|
||||
try {
|
||||
leadInboxMessages = await this.inboxReader.getMessagesFor(run.teamName, leadName);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
|
||||
const runStartedAtMs = Date.parse(run.startedAt);
|
||||
const teammateMessages = leadInboxMessages
|
||||
.filter((message) => {
|
||||
const from = typeof message.from === 'string' ? message.from.trim() : '';
|
||||
if (!from || from === leadName || from === 'user' || from === 'system') return false;
|
||||
if (!run.expectedMembers.includes(from)) return false;
|
||||
const messageTs = Date.parse(message.timestamp);
|
||||
if (
|
||||
Number.isFinite(messageTs) &&
|
||||
Number.isFinite(runStartedAtMs) &&
|
||||
messageTs < runStartedAtMs
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
return typeof message.text === 'string' && message.text.trim().length > 0;
|
||||
})
|
||||
.sort((a, b) => Date.parse(a.timestamp) - Date.parse(b.timestamp));
|
||||
|
||||
for (const message of teammateMessages) {
|
||||
const from = message.from.trim();
|
||||
const reason = extractBootstrapFailureReason(message.text);
|
||||
if (reason) {
|
||||
this.setMemberSpawnStatus(run, from, 'error', reason);
|
||||
continue;
|
||||
}
|
||||
this.setMemberSpawnStatus(run, from, 'online', undefined, 'heartbeat');
|
||||
}
|
||||
}
|
||||
|
||||
private persistSentMessage(teamName: string, message: InboxMessage): void {
|
||||
try {
|
||||
createController({
|
||||
|
|
@ -3008,20 +3091,65 @@ export class TeamProvisioningService {
|
|||
error?: string,
|
||||
livenessSource?: MemberSpawnLivenessSource
|
||||
): void {
|
||||
const prev = run.memberSpawnStatuses.get(memberName);
|
||||
const prev = run.memberSpawnStatuses.get(memberName) ?? createInitialMemberSpawnStatusEntry();
|
||||
const updatedAt = nowIso();
|
||||
const next: MemberSpawnStatusEntry = {
|
||||
...prev,
|
||||
status,
|
||||
updatedAt,
|
||||
};
|
||||
|
||||
if (status === 'spawning') {
|
||||
next.launchState = 'starting';
|
||||
} else if (status === 'waiting') {
|
||||
next.agentToolAccepted = true;
|
||||
next.hardFailure = false;
|
||||
next.error = undefined;
|
||||
next.hardFailureReason = undefined;
|
||||
next.firstSpawnAcceptedAt = prev.firstSpawnAcceptedAt ?? updatedAt;
|
||||
next.launchState = 'runtime_pending_bootstrap';
|
||||
} else if (status === 'online') {
|
||||
next.agentToolAccepted = true;
|
||||
next.runtimeAlive = true;
|
||||
next.livenessSource = livenessSource;
|
||||
next.firstSpawnAcceptedAt = prev.firstSpawnAcceptedAt ?? updatedAt;
|
||||
if (livenessSource === 'heartbeat') {
|
||||
next.bootstrapConfirmed = true;
|
||||
next.lastHeartbeatAt = updatedAt;
|
||||
}
|
||||
next.hardFailure = false;
|
||||
next.error = undefined;
|
||||
next.hardFailureReason = undefined;
|
||||
next.launchState = deriveMemberLaunchState(next);
|
||||
} else if (status === 'error') {
|
||||
next.error = error;
|
||||
next.hardFailure = true;
|
||||
next.hardFailureReason = error;
|
||||
next.launchState = 'failed_to_start';
|
||||
} else if (status === 'offline') {
|
||||
Object.assign(next, createInitialMemberSpawnStatusEntry(), { updatedAt });
|
||||
}
|
||||
|
||||
next.launchState = deriveMemberLaunchState(next);
|
||||
if (
|
||||
prev?.status === status &&
|
||||
prev?.error === error &&
|
||||
prev?.livenessSource === livenessSource
|
||||
prev.status === next.status &&
|
||||
prev.launchState === next.launchState &&
|
||||
prev.error === next.error &&
|
||||
prev.hardFailureReason === next.hardFailureReason &&
|
||||
prev.livenessSource === next.livenessSource &&
|
||||
prev.agentToolAccepted === next.agentToolAccepted &&
|
||||
prev.runtimeAlive === next.runtimeAlive &&
|
||||
prev.bootstrapConfirmed === next.bootstrapConfirmed &&
|
||||
prev.hardFailure === next.hardFailure &&
|
||||
prev.firstSpawnAcceptedAt === next.firstSpawnAcceptedAt &&
|
||||
prev.lastHeartbeatAt === next.lastHeartbeatAt
|
||||
) {
|
||||
return;
|
||||
}
|
||||
run.memberSpawnStatuses.set(memberName, {
|
||||
status,
|
||||
error,
|
||||
livenessSource,
|
||||
updatedAt: nowIso(),
|
||||
});
|
||||
|
||||
run.memberSpawnStatuses.set(memberName, next);
|
||||
this.syncMemberLaunchGraceCheck(run, memberName, next);
|
||||
|
||||
if (status === 'spawning') {
|
||||
this.appendMemberBootstrapDiagnostic(run, memberName, 'Agent tool invoked');
|
||||
} else if (status === 'waiting') {
|
||||
|
|
@ -3074,14 +3202,96 @@ export class TeamProvisioningService {
|
|||
for (const [name, entry] of run.memberSpawnStatuses) {
|
||||
result[name] = {
|
||||
status: entry.status,
|
||||
launchState: entry.launchState,
|
||||
error: entry.error,
|
||||
hardFailureReason: entry.hardFailureReason,
|
||||
livenessSource: entry.livenessSource,
|
||||
agentToolAccepted: entry.agentToolAccepted,
|
||||
runtimeAlive: entry.runtimeAlive,
|
||||
bootstrapConfirmed: entry.bootstrapConfirmed,
|
||||
hardFailure: entry.hardFailure,
|
||||
firstSpawnAcceptedAt: entry.firstSpawnAcceptedAt,
|
||||
lastHeartbeatAt: entry.lastHeartbeatAt,
|
||||
updatedAt: entry.updatedAt,
|
||||
};
|
||||
}
|
||||
return { statuses: result, runId };
|
||||
}
|
||||
|
||||
private getMemberLaunchGraceKey(run: ProvisioningRun, memberName: string): string {
|
||||
return `member-launch-grace:${run.runId}:${memberName}`;
|
||||
}
|
||||
|
||||
private syncMemberLaunchGraceCheck(
|
||||
run: ProvisioningRun,
|
||||
memberName: string,
|
||||
entry: MemberSpawnStatusEntry
|
||||
): void {
|
||||
const key = this.getMemberLaunchGraceKey(run, memberName);
|
||||
const existing = this.pendingTimeouts.get(key);
|
||||
if (entry.launchState === 'failed_to_start' || entry.launchState === 'confirmed_alive') {
|
||||
if (existing) {
|
||||
clearTimeout(existing);
|
||||
this.pendingTimeouts.delete(key);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (!entry.firstSpawnAcceptedAt) {
|
||||
return;
|
||||
}
|
||||
const remainingMs =
|
||||
Date.parse(entry.firstSpawnAcceptedAt) + MEMBER_LAUNCH_GRACE_MS - Date.now();
|
||||
if (remainingMs <= 0) {
|
||||
if (existing) {
|
||||
clearTimeout(existing);
|
||||
this.pendingTimeouts.delete(key);
|
||||
}
|
||||
void this.reevaluateMemberLaunchStatus(run, memberName);
|
||||
return;
|
||||
}
|
||||
if (existing) {
|
||||
return;
|
||||
}
|
||||
const timer = setTimeout(() => {
|
||||
this.pendingTimeouts.delete(key);
|
||||
void this.reevaluateMemberLaunchStatus(run, memberName);
|
||||
}, remainingMs);
|
||||
timer.unref?.();
|
||||
this.pendingTimeouts.set(key, timer);
|
||||
}
|
||||
|
||||
private async reevaluateMemberLaunchStatus(
|
||||
run: ProvisioningRun,
|
||||
memberName: string
|
||||
): Promise<void> {
|
||||
const current = run.memberSpawnStatuses.get(memberName);
|
||||
if (!current) return;
|
||||
if (
|
||||
current.launchState === 'failed_to_start' ||
|
||||
current.launchState === 'confirmed_alive' ||
|
||||
!current.firstSpawnAcceptedAt
|
||||
) {
|
||||
return;
|
||||
}
|
||||
await this.refreshMemberSpawnStatusesFromLeadInbox(run);
|
||||
await this.auditMemberSpawnStatuses(run);
|
||||
const refreshed = run.memberSpawnStatuses.get(memberName);
|
||||
if (!refreshed) return;
|
||||
if (
|
||||
refreshed.launchState === 'failed_to_start' ||
|
||||
refreshed.launchState === 'confirmed_alive' ||
|
||||
refreshed.runtimeAlive
|
||||
) {
|
||||
return;
|
||||
}
|
||||
this.setMemberSpawnStatus(
|
||||
run,
|
||||
memberName,
|
||||
'error',
|
||||
'Teammate did not join within the launch grace window.'
|
||||
);
|
||||
}
|
||||
|
||||
private static readonly CONTEXT_EMIT_THROTTLE_MS = 2000;
|
||||
private static readonly LEAD_TEXT_EMIT_THROTTLE_MS = 2000;
|
||||
|
||||
|
|
@ -4063,7 +4273,7 @@ export class TeamProvisioningService {
|
|||
geminiPostLaunchHydrationSent: false,
|
||||
suppressGeminiPostLaunchHydrationOutput: false,
|
||||
memberSpawnStatuses: new Map(
|
||||
request.members.map((m) => [m.name, { status: 'offline' as const, updatedAt: nowIso() }])
|
||||
request.members.map((m) => [m.name, createInitialMemberSpawnStatusEntry()])
|
||||
),
|
||||
memberSpawnToolUseIds: new Map(),
|
||||
progress: {
|
||||
|
|
@ -4545,7 +4755,7 @@ export class TeamProvisioningService {
|
|||
geminiPostLaunchHydrationSent: false,
|
||||
suppressGeminiPostLaunchHydrationOutput: false,
|
||||
memberSpawnStatuses: new Map(
|
||||
expectedMembers.map((name) => [name, { status: 'offline' as const, updatedAt: nowIso() }])
|
||||
expectedMembers.map((name) => [name, createInitialMemberSpawnStatusEntry()])
|
||||
),
|
||||
memberSpawnToolUseIds: new Map(),
|
||||
progress: {
|
||||
|
|
@ -5155,6 +5365,8 @@ export class TeamProvisioningService {
|
|||
return 0;
|
||||
}
|
||||
|
||||
await this.refreshMemberSpawnStatusesFromLeadInbox(run);
|
||||
|
||||
const unread = leadInboxMessages
|
||||
.filter((m): m is InboxMessage & { messageId: string } => {
|
||||
if (m.read) return false;
|
||||
|
|
@ -5731,7 +5943,12 @@ export class TeamProvisioningService {
|
|||
// Flag any expected member not found in config.json (excluding the lead)
|
||||
for (const expected of run.expectedMembers) {
|
||||
const current = run.memberSpawnStatuses.get(expected);
|
||||
if (current?.status === 'error' || current?.status === 'online') continue;
|
||||
if (
|
||||
current?.launchState === 'failed_to_start' ||
|
||||
current?.launchState === 'confirmed_alive'
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const matchedRuntimeNames = [...registeredNames].filter((name) => {
|
||||
if (name === expected) return true;
|
||||
|
|
@ -5739,30 +5956,50 @@ export class TeamProvisioningService {
|
|||
return parsed !== null && parsed.suffix >= 2 && parsed.base === expected;
|
||||
});
|
||||
|
||||
// A teammate may intentionally stay silent after bootstrap. If Claude Code
|
||||
// registered the runtime and the OS process is still alive, treat it as
|
||||
// process-confirmed running. Keep this distinct from heartbeat-confirmed online.
|
||||
if (
|
||||
const runtimeAlive =
|
||||
matchedRuntimeNames.length > 0 &&
|
||||
matchedRuntimeNames.some((runtimeName) =>
|
||||
this.hasLiveTeamAgentProcess(run.teamName, runtimeName)
|
||||
)
|
||||
) {
|
||||
);
|
||||
|
||||
// A teammate may intentionally stay silent after bootstrap. If Claude Code
|
||||
// registered the runtime and the OS process is still alive, treat it as
|
||||
// process-confirmed running. Keep this distinct from heartbeat-confirmed online.
|
||||
if (runtimeAlive) {
|
||||
this.setMemberSpawnStatus(run, expected, 'online', undefined, 'process');
|
||||
continue;
|
||||
}
|
||||
|
||||
if (matchedRuntimeNames.length > 0) continue;
|
||||
if (matchedRuntimeNames.length > 0) {
|
||||
if (current?.agentToolAccepted) {
|
||||
this.setMemberSpawnStatus(run, expected, 'waiting');
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
const acceptedAtMs =
|
||||
current?.firstSpawnAcceptedAt != null ? Date.parse(current.firstSpawnAcceptedAt) : NaN;
|
||||
const graceExpired =
|
||||
current?.agentToolAccepted === true &&
|
||||
Number.isFinite(acceptedAtMs) &&
|
||||
Date.now() - acceptedAtMs >= MEMBER_LAUNCH_GRACE_MS;
|
||||
|
||||
if (current?.agentToolAccepted && !graceExpired) {
|
||||
this.setMemberSpawnStatus(run, expected, 'waiting');
|
||||
continue;
|
||||
}
|
||||
|
||||
logger.warn(
|
||||
`[${run.teamName}] Member "${expected}" not found in config.json members after provisioning`
|
||||
);
|
||||
this.setMemberSpawnStatus(
|
||||
run,
|
||||
expected,
|
||||
'error',
|
||||
'Teammate not registered after provisioning — spawned incorrectly. Restart the team to fix.'
|
||||
);
|
||||
if (graceExpired) {
|
||||
this.setMemberSpawnStatus(
|
||||
run,
|
||||
expected,
|
||||
'error',
|
||||
'Teammate not registered after provisioning within the launch grace window.'
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -5802,11 +6039,43 @@ export class TeamProvisioningService {
|
|||
run: ProvisioningRun
|
||||
): { name: string; error?: string; updatedAt: string }[] {
|
||||
return [...run.memberSpawnStatuses.entries()]
|
||||
.filter(([, entry]) => entry.status === 'error')
|
||||
.map(([name, entry]) => ({ name, error: entry.error, updatedAt: entry.updatedAt }))
|
||||
.filter(([, entry]) => entry.launchState === 'failed_to_start')
|
||||
.map(([name, entry]) => ({
|
||||
name,
|
||||
error: entry.hardFailureReason ?? entry.error,
|
||||
updatedAt: entry.updatedAt,
|
||||
}))
|
||||
.sort((a, b) => a.name.localeCompare(b.name));
|
||||
}
|
||||
|
||||
private getMemberLaunchSummary(run: ProvisioningRun): {
|
||||
confirmedCount: number;
|
||||
pendingCount: number;
|
||||
failedCount: number;
|
||||
runtimeAlivePendingCount: number;
|
||||
} {
|
||||
let confirmedCount = 0;
|
||||
let pendingCount = 0;
|
||||
let failedCount = 0;
|
||||
let runtimeAlivePendingCount = 0;
|
||||
for (const expected of run.expectedMembers) {
|
||||
const entry = run.memberSpawnStatuses.get(expected) ?? createInitialMemberSpawnStatusEntry();
|
||||
if (entry.launchState === 'confirmed_alive') {
|
||||
confirmedCount += 1;
|
||||
continue;
|
||||
}
|
||||
if (entry.launchState === 'failed_to_start') {
|
||||
failedCount += 1;
|
||||
continue;
|
||||
}
|
||||
pendingCount += 1;
|
||||
if (entry.runtimeAlive) {
|
||||
runtimeAlivePendingCount += 1;
|
||||
}
|
||||
}
|
||||
return { confirmedCount, pendingCount, failedCount, runtimeAlivePendingCount };
|
||||
}
|
||||
|
||||
private async persistPartialLaunchState(run: ProvisioningRun): Promise<void> {
|
||||
if (!run.isLaunch || !run.expectedMembers || run.expectedMembers.length === 0) {
|
||||
if (run.isLaunch) {
|
||||
|
|
@ -8140,18 +8409,24 @@ export class TeamProvisioningService {
|
|||
}
|
||||
|
||||
// Audit: flag any expected member not registered in config.json after launch.
|
||||
await this.refreshMemberSpawnStatusesFromLeadInbox(run);
|
||||
await this.auditMemberSpawnStatuses(run);
|
||||
await this.persistPartialLaunchState(run);
|
||||
const failedSpawnMembers = this.getFailedSpawnMembers(run);
|
||||
const launchSummary = this.getMemberLaunchSummary(run);
|
||||
const hasSpawnFailures = failedSpawnMembers.length > 0;
|
||||
const hasPendingBootstrap =
|
||||
!hasSpawnFailures && launchSummary.pendingCount > 0 && run.expectedMembers.length > 0;
|
||||
const readyMessage = hasSpawnFailures
|
||||
? `Launch completed with teammate errors — ${failedSpawnMembers
|
||||
.map((member) => member.name)
|
||||
.join(', ')} failed to start`
|
||||
: 'Team launched — process alive and ready';
|
||||
: hasPendingBootstrap
|
||||
? `Launch completed — ${launchSummary.confirmedCount}/${run.expectedMembers.length} teammates confirmed alive, bootstrap still pending`
|
||||
: 'Team launched — process alive and ready';
|
||||
const progress = updateProgress(run, 'ready', readyMessage, {
|
||||
cliLogsTail: extractCliLogsFromRun(run),
|
||||
messageSeverity: hasSpawnFailures ? 'warning' : undefined,
|
||||
messageSeverity: hasSpawnFailures || hasPendingBootstrap ? 'warning' : undefined,
|
||||
});
|
||||
run.onProgress(progress);
|
||||
this.provisioningRunByTeam.delete(run.teamName);
|
||||
|
|
@ -8171,7 +8446,7 @@ export class TeamProvisioningService {
|
|||
detail: 'lead-session-sync',
|
||||
});
|
||||
|
||||
if (!hasSpawnFailures) {
|
||||
if (!hasSpawnFailures && !hasPendingBootstrap) {
|
||||
// Fire "Team Launched" notification only for clean launches.
|
||||
void this.fireTeamLaunchedNotification(run);
|
||||
}
|
||||
|
|
@ -8293,10 +8568,14 @@ export class TeamProvisioningService {
|
|||
await this.teamMetaStore.deleteMeta(run.teamName).catch(() => {});
|
||||
|
||||
// Audit: flag any expected member not registered in config.json after provisioning.
|
||||
await this.refreshMemberSpawnStatusesFromLeadInbox(run);
|
||||
await this.auditMemberSpawnStatuses(run);
|
||||
await this.clearPartialLaunchState(run.teamName);
|
||||
const failedSpawnMembers = this.getFailedSpawnMembers(run);
|
||||
const launchSummary = this.getMemberLaunchSummary(run);
|
||||
const hasSpawnFailures = failedSpawnMembers.length > 0;
|
||||
const hasPendingBootstrap =
|
||||
!hasSpawnFailures && launchSummary.pendingCount > 0 && run.expectedMembers.length > 0;
|
||||
const progress = updateProgress(
|
||||
run,
|
||||
'ready',
|
||||
|
|
@ -8304,10 +8583,12 @@ export class TeamProvisioningService {
|
|||
? `Provisioning completed with teammate errors — ${failedSpawnMembers
|
||||
.map((member) => member.name)
|
||||
.join(', ')} failed to start`
|
||||
: 'Team provisioned — process alive and ready',
|
||||
: hasPendingBootstrap
|
||||
? `Team provisioned — ${launchSummary.confirmedCount}/${run.expectedMembers.length} teammates confirmed alive, bootstrap still pending`
|
||||
: 'Team provisioned — process alive and ready',
|
||||
{
|
||||
cliLogsTail: extractCliLogsFromRun(run),
|
||||
messageSeverity: hasSpawnFailures ? 'warning' : undefined,
|
||||
messageSeverity: hasSpawnFailures || hasPendingBootstrap ? 'warning' : undefined,
|
||||
}
|
||||
);
|
||||
run.onProgress(progress);
|
||||
|
|
@ -8328,7 +8609,7 @@ export class TeamProvisioningService {
|
|||
detail: 'lead-session-sync',
|
||||
});
|
||||
|
||||
if (!hasSpawnFailures) {
|
||||
if (!hasSpawnFailures && !hasPendingBootstrap) {
|
||||
// Fire "Team Launched" notification only for clean launches.
|
||||
void this.fireTeamLaunchedNotification(run);
|
||||
}
|
||||
|
|
@ -8702,6 +8983,14 @@ export class TeamProvisioningService {
|
|||
this.pendingTimeouts.delete(key);
|
||||
}
|
||||
}
|
||||
for (const memberName of run.memberSpawnStatuses.keys()) {
|
||||
const key = this.getMemberLaunchGraceKey(run, memberName);
|
||||
const timer = this.pendingTimeouts.get(key);
|
||||
if (timer) {
|
||||
clearTimeout(timer);
|
||||
this.pendingTimeouts.delete(key);
|
||||
}
|
||||
}
|
||||
run.activeCrossTeamReplyHints = [];
|
||||
run.pendingInboxRelayCandidates = [];
|
||||
for (const key of Array.from(this.memberInboxRelayInFlight.keys())) {
|
||||
|
|
|
|||
|
|
@ -577,14 +577,20 @@ export const TeamDetailView = ({
|
|||
string,
|
||||
{
|
||||
status: MemberSpawnStatusEntry['status'];
|
||||
launchState: MemberSpawnStatusEntry['launchState'];
|
||||
error?: string;
|
||||
updatedAt: string;
|
||||
runtimeAlive?: boolean;
|
||||
livenessSource?: MemberSpawnStatusEntry['livenessSource'];
|
||||
}
|
||||
>();
|
||||
for (const [name, entry] of Object.entries(memberSpawnStatuses)) {
|
||||
map.set(name, {
|
||||
status: entry.status,
|
||||
launchState: entry.launchState,
|
||||
error: entry.error,
|
||||
updatedAt: entry.updatedAt,
|
||||
runtimeAlive: entry.runtimeAlive,
|
||||
livenessSource: entry.livenessSource,
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -106,20 +106,23 @@ export const TeamProvisioningBanner = ({
|
|||
|
||||
const teammates = (teamMembers ?? []).filter((member) => !isLeadMember(member));
|
||||
const failedSpawnEntries = Object.entries(memberSpawnStatuses ?? {}).filter(
|
||||
([, entry]) => entry.status === 'error'
|
||||
([, entry]) => entry.launchState === 'failed_to_start'
|
||||
);
|
||||
const failedSpawnCount = failedSpawnEntries.length;
|
||||
const heartbeatConfirmedCount = teammates.filter((member) => {
|
||||
const entry = memberSpawnStatuses?.[member.name];
|
||||
return entry?.status === 'online' && entry.livenessSource === 'heartbeat';
|
||||
return entry?.launchState === 'confirmed_alive';
|
||||
}).length;
|
||||
const processOnlyAliveCount = teammates.filter((member) => {
|
||||
const entry = memberSpawnStatuses?.[member.name];
|
||||
return entry?.status === 'online' && entry.livenessSource === 'process';
|
||||
return entry?.launchState === 'runtime_pending_bootstrap' && entry.runtimeAlive === true;
|
||||
}).length;
|
||||
const awaitingHeartbeatCount = teammates.filter((member) => {
|
||||
const pendingSpawnCount = teammates.filter((member) => {
|
||||
const entry = memberSpawnStatuses?.[member.name];
|
||||
return entry?.status === 'waiting';
|
||||
return (
|
||||
entry?.launchState === 'starting' ||
|
||||
(entry?.launchState === 'runtime_pending_bootstrap' && entry.runtimeAlive !== true)
|
||||
);
|
||||
}).length;
|
||||
const allTeammatesConfirmedAlive =
|
||||
teammates.length > 0 && failedSpawnCount === 0 && heartbeatConfirmedCount === teammates.length;
|
||||
|
|
@ -132,8 +135,8 @@ export const TeamProvisioningBanner = ({
|
|||
? 'Team launched — lead online'
|
||||
: allTeammatesConfirmedAlive
|
||||
? `Team launched — all ${teammates.length} teammates confirmed alive`
|
||||
: processOnlyAliveCount > 0 || awaitingHeartbeatCount > 0
|
||||
? `Team launched — ${heartbeatConfirmedCount}/${teammates.length} teammates confirmed alive${processOnlyAliveCount > 0 ? `, ${processOnlyAliveCount} runtime${processOnlyAliveCount === 1 ? '' : 's'} alive but bootstrap still pending` : ''}${awaitingHeartbeatCount > 0 ? `${processOnlyAliveCount > 0 ? ', ' : ', '}${awaitingHeartbeatCount} awaiting first heartbeat` : ''}`
|
||||
: processOnlyAliveCount > 0 || pendingSpawnCount > 0
|
||||
? `Team launched — ${heartbeatConfirmedCount}/${teammates.length} teammates confirmed alive${processOnlyAliveCount > 0 ? `, ${processOnlyAliveCount} runtime${processOnlyAliveCount === 1 ? '' : 's'} alive but bootstrap still pending` : ''}${pendingSpawnCount > 0 ? `${processOnlyAliveCount > 0 ? ', ' : ', '}${pendingSpawnCount} still starting` : ''}`
|
||||
: 'Team launched — teammate liveness is still being confirmed';
|
||||
|
||||
return (
|
||||
|
|
@ -151,7 +154,11 @@ export const TeamProvisioningBanner = ({
|
|||
defaultLiveOutputOpen={false}
|
||||
onCancel={null}
|
||||
successMessage={readyMessage}
|
||||
successMessageSeverity={failedSpawnCount > 0 ? 'warning' : 'success'}
|
||||
successMessageSeverity={
|
||||
failedSpawnCount > 0 || processOnlyAliveCount > 0 || pendingSpawnCount > 0
|
||||
? 'warning'
|
||||
: 'success'
|
||||
}
|
||||
onDismiss={() => setDismissed(true)}
|
||||
/>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ import { CurrentTaskIndicator } from './CurrentTaskIndicator';
|
|||
import type { TaskStatusCounts } from '@renderer/utils/pathNormalize';
|
||||
import type {
|
||||
LeadActivityState,
|
||||
MemberLaunchState,
|
||||
MemberSpawnLivenessSource,
|
||||
MemberSpawnStatus,
|
||||
ResolvedTeamMember,
|
||||
|
|
@ -39,6 +40,8 @@ interface MemberCardProps {
|
|||
spawnStatus?: MemberSpawnStatus;
|
||||
spawnError?: string;
|
||||
spawnLivenessSource?: MemberSpawnLivenessSource;
|
||||
spawnLaunchState?: MemberLaunchState;
|
||||
spawnRuntimeAlive?: boolean;
|
||||
onOpenTask?: () => void;
|
||||
onOpenReviewTask?: () => void;
|
||||
onClick?: () => void;
|
||||
|
|
@ -61,6 +64,8 @@ export const MemberCard = ({
|
|||
spawnStatus,
|
||||
spawnError,
|
||||
spawnLivenessSource,
|
||||
spawnLaunchState,
|
||||
spawnRuntimeAlive,
|
||||
onOpenTask,
|
||||
onOpenReviewTask,
|
||||
onClick,
|
||||
|
|
@ -75,6 +80,7 @@ export const MemberCard = ({
|
|||
const dotClass = getSpawnAwareDotClass(
|
||||
member,
|
||||
spawnStatus,
|
||||
spawnLaunchState,
|
||||
isTeamAlive,
|
||||
isTeamProvisioning,
|
||||
leadActivity
|
||||
|
|
@ -82,7 +88,9 @@ export const MemberCard = ({
|
|||
const presenceLabel = getSpawnAwarePresenceLabel(
|
||||
member,
|
||||
spawnStatus,
|
||||
spawnLaunchState,
|
||||
spawnLivenessSource,
|
||||
spawnRuntimeAlive,
|
||||
isTeamAlive,
|
||||
isTeamProvisioning,
|
||||
leadActivity
|
||||
|
|
|
|||
|
|
@ -15,24 +15,17 @@ import { MemberCard } from './MemberCard';
|
|||
import type { TaskStatusCounts } from '@renderer/utils/pathNormalize';
|
||||
import type {
|
||||
LeadActivityState,
|
||||
MemberSpawnLivenessSource,
|
||||
MemberSpawnStatus,
|
||||
MemberSpawnStatusEntry,
|
||||
ResolvedTeamMember,
|
||||
TeamTaskWithKanban,
|
||||
} from '@shared/types';
|
||||
|
||||
export interface MemberSpawnEntry {
|
||||
status: MemberSpawnStatus;
|
||||
error?: string;
|
||||
livenessSource?: MemberSpawnLivenessSource;
|
||||
}
|
||||
|
||||
interface MemberListProps {
|
||||
members: ResolvedTeamMember[];
|
||||
memberTaskCounts?: Map<string, TaskStatusCounts>;
|
||||
taskMap?: Map<string, TeamTaskWithKanban>;
|
||||
pendingRepliesByMember?: Record<string, number>;
|
||||
memberSpawnStatuses?: Map<string, MemberSpawnEntry>;
|
||||
memberSpawnStatuses?: Map<string, MemberSpawnStatusEntry>;
|
||||
isTeamAlive?: boolean;
|
||||
isTeamProvisioning?: boolean;
|
||||
leadActivity?: LeadActivityState;
|
||||
|
|
@ -137,6 +130,8 @@ export const MemberList = ({
|
|||
spawnStatus={isRemoved ? undefined : spawnEntry?.status}
|
||||
spawnError={isRemoved ? undefined : spawnEntry?.error}
|
||||
spawnLivenessSource={isRemoved ? undefined : spawnEntry?.livenessSource}
|
||||
spawnLaunchState={isRemoved ? undefined : spawnEntry?.launchState}
|
||||
spawnRuntimeAlive={isRemoved ? undefined : spawnEntry?.runtimeAlive}
|
||||
onOpenTask={!isRemoved && currentTask ? () => onOpenTask?.(currentTask) : undefined}
|
||||
onOpenReviewTask={!isRemoved && reviewTask ? () => onOpenTask?.(reviewTask) : undefined}
|
||||
onClick={() => onMemberClick?.(member)}
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import { isLeadMember } from '@shared/utils/leadDetection';
|
|||
|
||||
import type {
|
||||
LeadActivityState,
|
||||
MemberLaunchState,
|
||||
MemberSpawnLivenessSource,
|
||||
MemberSpawnStatus,
|
||||
MemberStatus,
|
||||
|
|
@ -99,7 +100,7 @@ export const SPAWN_DOT_COLORS: Record<MemberSpawnStatus, string> = {
|
|||
|
||||
export const SPAWN_PRESENCE_LABELS: Record<MemberSpawnStatus, string> = {
|
||||
offline: 'offline',
|
||||
waiting: 'awaiting heartbeat',
|
||||
waiting: 'bootstrap pending',
|
||||
spawning: 'starting',
|
||||
online: 'ready',
|
||||
error: 'spawn failed',
|
||||
|
|
@ -112,13 +113,17 @@ export const SPAWN_PRESENCE_LABELS: Record<MemberSpawnStatus, string> = {
|
|||
export function getSpawnAwareDotClass(
|
||||
member: ResolvedTeamMember,
|
||||
spawnStatus: MemberSpawnStatus | undefined,
|
||||
spawnLaunchState: MemberLaunchState | undefined,
|
||||
isTeamAlive?: boolean,
|
||||
isTeamProvisioning?: boolean,
|
||||
leadActivity?: LeadActivityState
|
||||
): string {
|
||||
if (spawnStatus === 'error') {
|
||||
if (spawnLaunchState === 'failed_to_start' || spawnStatus === 'error') {
|
||||
return SPAWN_DOT_COLORS.error;
|
||||
}
|
||||
if (spawnLaunchState === 'runtime_pending_bootstrap' && spawnStatus === 'online') {
|
||||
return SPAWN_DOT_COLORS.spawning;
|
||||
}
|
||||
if (spawnStatus === 'waiting') {
|
||||
return SPAWN_DOT_COLORS.waiting;
|
||||
}
|
||||
|
|
@ -140,22 +145,27 @@ export function getSpawnAwareDotClass(
|
|||
export function getSpawnAwarePresenceLabel(
|
||||
member: ResolvedTeamMember,
|
||||
spawnStatus: MemberSpawnStatus | undefined,
|
||||
spawnLaunchState: MemberLaunchState | undefined,
|
||||
livenessSource: MemberSpawnLivenessSource | undefined,
|
||||
runtimeAlive: boolean | undefined,
|
||||
isTeamAlive?: boolean,
|
||||
isTeamProvisioning?: boolean,
|
||||
leadActivity?: LeadActivityState
|
||||
): string {
|
||||
if (spawnStatus === 'error') {
|
||||
if (spawnLaunchState === 'failed_to_start' || spawnStatus === 'error') {
|
||||
return SPAWN_PRESENCE_LABELS.error;
|
||||
}
|
||||
if (spawnStatus === 'offline' && isTeamProvisioning) {
|
||||
return 'waiting for Agent';
|
||||
}
|
||||
if (spawnLaunchState === 'runtime_pending_bootstrap' && runtimeAlive) {
|
||||
return 'bootstrap pending';
|
||||
}
|
||||
if (spawnStatus === 'waiting') {
|
||||
return SPAWN_PRESENCE_LABELS.waiting;
|
||||
}
|
||||
if (spawnStatus === 'online' && livenessSource === 'process') {
|
||||
return 'running';
|
||||
return 'bootstrap pending';
|
||||
}
|
||||
if (spawnStatus && isTeamProvisioning) {
|
||||
return SPAWN_PRESENCE_LABELS[spawnStatus];
|
||||
|
|
|
|||
|
|
@ -440,15 +440,12 @@ export interface AddTaskCommentRequest {
|
|||
|
||||
export type MemberStatus = 'active' | 'idle' | 'terminated' | 'unknown';
|
||||
|
||||
/**
|
||||
* Spawn lifecycle status for a team member during team launch/reconnect.
|
||||
* - offline: queued, Agent tool_use not sent yet
|
||||
* - spawning: Agent tool_use sent, awaiting tool_result
|
||||
* - waiting: teammate process accepted by runtime, awaiting first heartbeat/inbox signal
|
||||
* - online: first heartbeat/inbox signal received
|
||||
* - error: spawn failed (tool_result with error)
|
||||
*/
|
||||
export type MemberSpawnStatus = 'offline' | 'waiting' | 'spawning' | 'online' | 'error';
|
||||
export type MemberLaunchState =
|
||||
| 'starting'
|
||||
| 'runtime_pending_bootstrap'
|
||||
| 'confirmed_alive'
|
||||
| 'failed_to_start';
|
||||
|
||||
export type KanbanColumnId = 'todo' | 'in_progress' | 'done' | 'review' | 'approved';
|
||||
|
||||
|
|
@ -610,14 +607,29 @@ export interface ProjectBranchChangeEvent {
|
|||
/** Per-member spawn status entry, exposed to renderer via IPC. */
|
||||
export interface MemberSpawnStatusEntry {
|
||||
status: MemberSpawnStatus;
|
||||
launchState: MemberLaunchState;
|
||||
/** Error message when status === 'error'. */
|
||||
error?: string;
|
||||
/** Hard failure reason for failed_to_start. */
|
||||
hardFailureReason?: string;
|
||||
/**
|
||||
* Optional provenance for `online`.
|
||||
* - heartbeat: teammate sent a real inbox/native message after bootstrap
|
||||
* - process: runtime process is alive, but bootstrap/first reply is not yet confirmed
|
||||
*/
|
||||
livenessSource?: MemberSpawnLivenessSource;
|
||||
/** Agent tool_result confirmed the spawn request was accepted by the runtime. */
|
||||
agentToolAccepted?: boolean;
|
||||
/** Runtime process or registered teammate runtime is currently alive. */
|
||||
runtimeAlive?: boolean;
|
||||
/** A real teammate heartbeat/bootstrap confirmation was observed. */
|
||||
bootstrapConfirmed?: boolean;
|
||||
/** Hard failure observed from spawn/bootstrap/runtime evidence. */
|
||||
hardFailure?: boolean;
|
||||
/** ISO timestamp of the first accepted teammate spawn for this member. */
|
||||
firstSpawnAcceptedAt?: string;
|
||||
/** ISO timestamp of the latest confirmed heartbeat/bootstrap message. */
|
||||
lastHeartbeatAt?: string;
|
||||
/** ISO timestamp of the last status change. */
|
||||
updatedAt: string;
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue