Persist launch-state and duplicate no-op handling

This commit is contained in:
iliya 2026-04-05 15:23:41 +03:00
parent 2959a3d074
commit 30fb2501d3
12 changed files with 1102 additions and 252 deletions

View file

@ -12,6 +12,7 @@ import * as path from 'path';
import { getTeamFsWorkerClient } from './TeamFsWorkerClient'; import { getTeamFsWorkerClient } from './TeamFsWorkerClient';
import { TeamMembersMetaStore } from './TeamMembersMetaStore'; import { TeamMembersMetaStore } from './TeamMembersMetaStore';
import { TeamMetaStore } from './TeamMetaStore'; import { TeamMetaStore } from './TeamMetaStore';
import { normalizePersistedLaunchSnapshot } from './TeamLaunchStateEvaluator';
import type { TeamConfig, TeamMember, TeamSummary, TeamSummaryMember } from '@shared/types'; import type { TeamConfig, TeamMember, TeamSummary, TeamSummaryMember } from '@shared/types';
@ -27,16 +28,20 @@ const MAX_PROJECT_PATH_HISTORY_IN_SUMMARY = 200;
const MAX_LAUNCH_STATE_BYTES = 32 * 1024; const MAX_LAUNCH_STATE_BYTES = 32 * 1024;
const TEAM_LAUNCH_STATE_FILE = 'launch-state.json'; const TEAM_LAUNCH_STATE_FILE = 'launch-state.json';
interface PartialLaunchStateSummary { interface LaunchStateSummary {
partialLaunchFailure: true; partialLaunchFailure?: true;
expectedMemberCount: number; expectedMemberCount?: number;
confirmedMemberCount: number; confirmedMemberCount?: number;
missingMembers: string[]; missingMembers?: string[];
teamLaunchState?: TeamSummary['teamLaunchState'];
launchUpdatedAt?: string;
confirmedCount?: number;
pendingCount?: number;
failedCount?: number;
runtimeAlivePendingCount?: number;
} }
async function readPartialLaunchStateSummary( async function readLaunchStateSummary(teamDir: string): Promise<LaunchStateSummary | null> {
teamDir: string
): Promise<PartialLaunchStateSummary | null> {
const launchStatePath = path.join(teamDir, TEAM_LAUNCH_STATE_FILE); const launchStatePath = path.join(teamDir, TEAM_LAUNCH_STATE_FILE);
try { try {
const stat = await fs.promises.stat(launchStatePath); const stat = await fs.promises.stat(launchStatePath);
@ -44,38 +49,31 @@ async function readPartialLaunchStateSummary(
return null; return null;
} }
const raw = await readFileUtf8WithTimeout(launchStatePath, PER_TEAM_READ_TIMEOUT_MS); const raw = await readFileUtf8WithTimeout(launchStatePath, PER_TEAM_READ_TIMEOUT_MS);
const parsed = JSON.parse(raw) as { const snapshot = normalizePersistedLaunchSnapshot(path.basename(teamDir), JSON.parse(raw));
state?: unknown; if (!snapshot) {
expectedMembers?: unknown;
confirmedMembers?: unknown;
missingMembers?: unknown;
};
if (parsed.state !== 'partial_launch_failure') {
return null;
}
const expectedMembers = Array.isArray(parsed.expectedMembers)
? parsed.expectedMembers.filter(
(name): name is string => typeof name === 'string' && name.trim().length > 0
)
: [];
const confirmedMembers = Array.isArray(parsed.confirmedMembers)
? parsed.confirmedMembers.filter(
(name): name is string => typeof name === 'string' && name.trim().length > 0
)
: [];
const missingMembers = Array.isArray(parsed.missingMembers)
? parsed.missingMembers.filter(
(name): name is string => typeof name === 'string' && name.trim().length > 0
)
: [];
if (expectedMembers.length === 0 || missingMembers.length === 0) {
return null; return null;
} }
const missingMembers = snapshot.expectedMembers.filter((name) => {
const member = snapshot.members[name];
return member?.launchState === 'failed_to_start';
});
return { return {
partialLaunchFailure: true, ...(snapshot.teamLaunchState === 'partial_failure'
expectedMemberCount: expectedMembers.length, ? { partialLaunchFailure: true as const }
confirmedMemberCount: confirmedMembers.length, : {}),
missingMembers, ...(snapshot.expectedMembers.length > 0
? { expectedMemberCount: snapshot.expectedMembers.length }
: {}),
...(snapshot.summary.confirmedCount > 0
? { confirmedMemberCount: snapshot.summary.confirmedCount }
: {}),
...(missingMembers.length > 0 ? { missingMembers } : {}),
teamLaunchState: snapshot.teamLaunchState,
launchUpdatedAt: snapshot.updatedAt,
confirmedCount: snapshot.summary.confirmedCount,
pendingCount: snapshot.summary.pendingCount,
failedCount: snapshot.summary.failedCount,
runtimeAlivePendingCount: snapshot.summary.runtimeAlivePendingCount,
}; };
} catch { } catch {
return null; return null;
@ -340,8 +338,8 @@ export class TeamConfigReader {
} }
const members = Array.from(memberMap.values()); const members = Array.from(memberMap.values());
const partialLaunchState = const launchStateSummary =
(await readPartialLaunchStateSummary(teamDir)) ?? (await readLaunchStateSummary(teamDir)) ??
(() => { (() => {
if ( if (
!leadSessionId || !leadSessionId ||
@ -377,7 +375,7 @@ export class TeamConfigReader {
...(projectPathHistory ? { projectPathHistory } : {}), ...(projectPathHistory ? { projectPathHistory } : {}),
...(sessionHistory ? { sessionHistory } : {}), ...(sessionHistory ? { sessionHistory } : {}),
...(deletedAt ? { deletedAt } : {}), ...(deletedAt ? { deletedAt } : {}),
...(partialLaunchState ?? {}), ...(launchStateSummary ?? {}),
}; };
return summary; return summary;
} catch { } catch {

View file

@ -0,0 +1,426 @@
import { isLeadMember } from '@shared/utils/leadDetection';
import type {
MemberLaunchState,
MemberSpawnLivenessSource,
MemberSpawnStatusEntry,
PersistedTeamLaunchMemberSources,
PersistedTeamLaunchMemberState,
PersistedTeamLaunchPhase,
PersistedTeamLaunchSnapshot,
PersistedTeamLaunchSummary,
TeamLaunchAggregateState,
} from '@shared/types';
interface LegacyPartialLaunchStateFile {
version?: unknown;
state?: unknown;
updatedAt?: unknown;
leadSessionId?: unknown;
expectedMembers?: unknown;
confirmedMembers?: unknown;
missingMembers?: unknown;
}
type RuntimeMemberSpawnState = Pick<
MemberSpawnStatusEntry,
| 'launchState'
| 'status'
| 'error'
| 'hardFailureReason'
| 'livenessSource'
| 'agentToolAccepted'
| 'runtimeAlive'
| 'bootstrapConfirmed'
| 'hardFailure'
| 'firstSpawnAcceptedAt'
| 'lastHeartbeatAt'
| 'updatedAt'
>;
function normalizeMemberName(name: string): string {
return name.trim();
}
function buildDiagnostics(
member: Pick<
PersistedTeamLaunchMemberState,
'agentToolAccepted' | 'runtimeAlive' | 'bootstrapConfirmed' | 'hardFailureReason' | 'sources'
>
): string[] {
const diagnostics: string[] = [];
if (member.agentToolAccepted) diagnostics.push('spawn accepted');
if (member.runtimeAlive) diagnostics.push('runtime alive');
if (member.bootstrapConfirmed) diagnostics.push('late heartbeat received');
if (member.runtimeAlive && !member.bootstrapConfirmed) diagnostics.push('waiting for bootstrap');
if (member.hardFailureReason)
diagnostics.push(`hard failure reason: ${member.hardFailureReason}`);
if (member.sources?.duplicateRespawnBlocked) diagnostics.push('respawn blocked as duplicate');
if (member.sources?.configDrift) diagnostics.push('config drift detected');
return diagnostics;
}
export function deriveTeamLaunchAggregateState(
summary: PersistedTeamLaunchSummary
): TeamLaunchAggregateState {
if (summary.failedCount > 0) {
return 'partial_failure';
}
if (summary.pendingCount > 0) {
return 'partial_pending';
}
return 'clean_success';
}
export function summarizePersistedLaunchMembers(
expectedMembers: readonly string[],
members: Record<string, PersistedTeamLaunchMemberState>
): PersistedTeamLaunchSummary {
let confirmedCount = 0;
let pendingCount = 0;
let failedCount = 0;
let runtimeAlivePendingCount = 0;
const normalizedExpected = expectedMembers.map(normalizeMemberName).filter(Boolean);
for (const memberName of normalizedExpected) {
const entry = members[memberName];
if (!entry) {
pendingCount += 1;
continue;
}
if (entry.launchState === 'confirmed_alive') {
confirmedCount += 1;
continue;
}
if (entry.launchState === 'failed_to_start') {
failedCount += 1;
continue;
}
pendingCount += 1;
if (entry.runtimeAlive) {
runtimeAlivePendingCount += 1;
}
}
return { confirmedCount, pendingCount, failedCount, runtimeAlivePendingCount };
}
function deriveMemberLaunchState(
member: Pick<
PersistedTeamLaunchMemberState,
'hardFailure' | 'bootstrapConfirmed' | 'runtimeAlive' | 'agentToolAccepted'
>
): MemberLaunchState {
if (member.hardFailure) {
return 'failed_to_start';
}
if (member.bootstrapConfirmed) {
return 'confirmed_alive';
}
if (member.runtimeAlive || member.agentToolAccepted) {
return 'runtime_pending_bootstrap';
}
return 'starting';
}
function toBoolean(value: unknown): boolean {
return value === true;
}
function normalizeSources(value: unknown): PersistedTeamLaunchMemberSources | undefined {
if (!value || typeof value !== 'object') {
return undefined;
}
const source = value as Record<string, unknown>;
const normalized: PersistedTeamLaunchMemberSources = {
inboxHeartbeat: toBoolean(source.inboxHeartbeat),
nativeHeartbeat: toBoolean(source.nativeHeartbeat),
processAlive: toBoolean(source.processAlive),
configRegistered: toBoolean(source.configRegistered),
configDrift: toBoolean(source.configDrift),
hardFailureSignal: toBoolean(source.hardFailureSignal),
duplicateRespawnBlocked: toBoolean(source.duplicateRespawnBlocked),
};
return Object.values(normalized).some(Boolean) ? normalized : undefined;
}
function normalizePersistedMemberState(
memberName: string,
value: unknown,
updatedAtFallback: string
): PersistedTeamLaunchMemberState | null {
if (!value || typeof value !== 'object') {
return null;
}
const parsed = value as Record<string, unknown>;
const normalizedName = normalizeMemberName(memberName);
if (!normalizedName || normalizedName === 'user' || isLeadMember({ name: normalizedName })) {
return null;
}
const next: PersistedTeamLaunchMemberState = {
name: normalizedName,
launchState: 'starting',
agentToolAccepted: toBoolean(parsed.agentToolAccepted),
runtimeAlive: toBoolean(parsed.runtimeAlive),
bootstrapConfirmed: toBoolean(parsed.bootstrapConfirmed),
hardFailure: toBoolean(parsed.hardFailure),
hardFailureReason:
typeof parsed.hardFailureReason === 'string' && parsed.hardFailureReason.trim().length > 0
? parsed.hardFailureReason.trim()
: undefined,
firstSpawnAcceptedAt:
typeof parsed.firstSpawnAcceptedAt === 'string' ? parsed.firstSpawnAcceptedAt : undefined,
lastHeartbeatAt:
typeof parsed.lastHeartbeatAt === 'string' ? parsed.lastHeartbeatAt : undefined,
lastRuntimeAliveAt:
typeof parsed.lastRuntimeAliveAt === 'string' ? parsed.lastRuntimeAliveAt : undefined,
lastEvaluatedAt:
typeof parsed.lastEvaluatedAt === 'string' ? parsed.lastEvaluatedAt : updatedAtFallback,
sources: normalizeSources(parsed.sources),
diagnostics: Array.isArray(parsed.diagnostics)
? parsed.diagnostics.filter(
(item): item is string => typeof item === 'string' && item.trim().length > 0
)
: undefined,
};
const launchState =
parsed.launchState === 'starting' ||
parsed.launchState === 'runtime_pending_bootstrap' ||
parsed.launchState === 'confirmed_alive' ||
parsed.launchState === 'failed_to_start'
? parsed.launchState
: deriveMemberLaunchState(next);
next.launchState = launchState;
next.diagnostics = next.diagnostics?.length ? next.diagnostics : buildDiagnostics(next);
return next;
}
export function createPersistedLaunchSnapshot(params: {
teamName: string;
expectedMembers: readonly string[];
leadSessionId?: string;
launchPhase?: PersistedTeamLaunchPhase;
members?: Record<string, PersistedTeamLaunchMemberState>;
updatedAt?: string;
}): PersistedTeamLaunchSnapshot {
const updatedAt = params.updatedAt ?? new Date().toISOString();
const expectedMembers = Array.from(
new Set(
params.expectedMembers
.map(normalizeMemberName)
.filter((name) => name.length > 0 && name !== 'user' && !isLeadMember({ name }))
)
);
const members = params.members ?? {};
const summary = summarizePersistedLaunchMembers(expectedMembers, members);
return {
version: 2,
teamName: params.teamName,
updatedAt,
...(params.leadSessionId ? { leadSessionId: params.leadSessionId } : {}),
launchPhase: params.launchPhase ?? 'active',
expectedMembers,
members,
summary,
teamLaunchState: deriveTeamLaunchAggregateState(summary),
};
}
export function snapshotFromRuntimeMemberStatuses(params: {
teamName: string;
expectedMembers: readonly string[];
leadSessionId?: string;
launchPhase?: PersistedTeamLaunchPhase;
statuses: Record<string, RuntimeMemberSpawnState>;
updatedAt?: string;
}): PersistedTeamLaunchSnapshot {
const updatedAt = params.updatedAt ?? new Date().toISOString();
const members: Record<string, PersistedTeamLaunchMemberState> = {};
for (const expected of params.expectedMembers) {
const name = normalizeMemberName(expected);
if (!name || name === 'user' || isLeadMember({ name })) continue;
const runtime = params.statuses[name];
const sources: PersistedTeamLaunchMemberSources = {};
if (runtime?.livenessSource === 'heartbeat') {
sources.nativeHeartbeat = true;
sources.inboxHeartbeat = true;
}
if (runtime?.livenessSource === 'process' || runtime?.runtimeAlive) {
sources.processAlive = true;
}
const entry: PersistedTeamLaunchMemberState = {
name,
launchState: runtime?.launchState ?? 'starting',
agentToolAccepted: runtime?.agentToolAccepted === true,
runtimeAlive: runtime?.runtimeAlive === true,
bootstrapConfirmed: runtime?.bootstrapConfirmed === true,
hardFailure: runtime?.hardFailure === true || runtime?.launchState === 'failed_to_start',
hardFailureReason: runtime?.hardFailureReason ?? runtime?.error,
firstSpawnAcceptedAt: runtime?.firstSpawnAcceptedAt,
lastHeartbeatAt: runtime?.lastHeartbeatAt,
lastRuntimeAliveAt: runtime?.runtimeAlive ? updatedAt : undefined,
lastEvaluatedAt: runtime?.updatedAt ?? updatedAt,
sources: Object.values(sources).some(Boolean) ? sources : undefined,
diagnostics: undefined,
};
entry.launchState = deriveMemberLaunchState(entry);
entry.diagnostics = buildDiagnostics(entry);
members[name] = entry;
}
return createPersistedLaunchSnapshot({
teamName: params.teamName,
expectedMembers: params.expectedMembers,
leadSessionId: params.leadSessionId,
launchPhase: params.launchPhase,
members,
updatedAt,
});
}
export function snapshotToMemberSpawnStatuses(
snapshot: PersistedTeamLaunchSnapshot | null
): Record<string, MemberSpawnStatusEntry> {
if (!snapshot) return {};
const statuses: Record<string, MemberSpawnStatusEntry> = {};
for (const memberName of snapshot.expectedMembers) {
const entry = snapshot.members[memberName];
if (!entry) continue;
let status: MemberSpawnStatusEntry['status'] = 'offline';
let livenessSource: MemberSpawnLivenessSource | undefined;
if (entry.launchState === 'failed_to_start') {
status = 'error';
} else if (entry.launchState === 'confirmed_alive') {
status = 'online';
livenessSource = 'heartbeat';
} else if (entry.launchState === 'runtime_pending_bootstrap') {
status = entry.runtimeAlive ? 'online' : 'waiting';
livenessSource = entry.runtimeAlive ? 'process' : undefined;
} else {
status = entry.agentToolAccepted ? 'waiting' : 'spawning';
}
statuses[memberName] = {
status,
launchState: entry.launchState,
error: entry.hardFailure ? entry.hardFailureReason : undefined,
hardFailureReason: entry.hardFailureReason,
livenessSource,
agentToolAccepted: entry.agentToolAccepted,
runtimeAlive: entry.runtimeAlive,
bootstrapConfirmed: entry.bootstrapConfirmed,
hardFailure: entry.hardFailure,
firstSpawnAcceptedAt: entry.firstSpawnAcceptedAt,
lastHeartbeatAt: entry.lastHeartbeatAt,
updatedAt: entry.lastEvaluatedAt,
};
}
return statuses;
}
export function normalizePersistedLaunchSnapshot(
teamName: string,
parsed: unknown
): PersistedTeamLaunchSnapshot | null {
if (!parsed || typeof parsed !== 'object') {
return null;
}
const maybeLegacy = parsed as LegacyPartialLaunchStateFile;
if (maybeLegacy.state === 'partial_launch_failure') {
const expectedMembers = Array.isArray(maybeLegacy.expectedMembers)
? maybeLegacy.expectedMembers.filter(
(name): name is string => typeof name === 'string' && normalizeMemberName(name).length > 0
)
: [];
const confirmedMembers = Array.isArray(maybeLegacy.confirmedMembers)
? maybeLegacy.confirmedMembers.filter(
(name): name is string => typeof name === 'string' && normalizeMemberName(name).length > 0
)
: [];
const missingMembers = Array.isArray(maybeLegacy.missingMembers)
? maybeLegacy.missingMembers.filter(
(name): name is string => typeof name === 'string' && normalizeMemberName(name).length > 0
)
: [];
if (expectedMembers.length === 0 || missingMembers.length === 0) {
return null;
}
const updatedAt =
typeof maybeLegacy.updatedAt === 'string' ? maybeLegacy.updatedAt : new Date().toISOString();
const members: Record<string, PersistedTeamLaunchMemberState> = {};
for (const name of expectedMembers) {
const failed = missingMembers.includes(name);
const confirmed = confirmedMembers.includes(name);
const entry: PersistedTeamLaunchMemberState = {
name,
launchState: failed ? 'failed_to_start' : confirmed ? 'confirmed_alive' : 'starting',
agentToolAccepted: true,
runtimeAlive: confirmed,
bootstrapConfirmed: confirmed,
hardFailure: failed,
hardFailureReason: failed
? 'Legacy partial launch marker reported teammate missing.'
: undefined,
lastEvaluatedAt: updatedAt,
diagnostics: undefined,
};
entry.diagnostics = buildDiagnostics(entry);
members[name] = entry;
}
return createPersistedLaunchSnapshot({
teamName,
expectedMembers,
leadSessionId:
typeof maybeLegacy.leadSessionId === 'string' && maybeLegacy.leadSessionId.trim().length > 0
? maybeLegacy.leadSessionId.trim()
: undefined,
launchPhase: 'finished',
members,
updatedAt,
});
}
const record = parsed as Record<string, unknown>;
if (record.version !== 2) {
return null;
}
const expectedMembers = Array.isArray(record.expectedMembers)
? record.expectedMembers.filter(
(name): name is string => typeof name === 'string' && normalizeMemberName(name).length > 0
)
: [];
const updatedAt =
typeof record.updatedAt === 'string' && record.updatedAt.trim().length > 0
? record.updatedAt
: new Date().toISOString();
const normalizedMembers: Record<string, PersistedTeamLaunchMemberState> = {};
const rawMembers =
record.members && typeof record.members === 'object'
? (record.members as Record<string, unknown>)
: {};
for (const [memberName, value] of Object.entries(rawMembers)) {
const normalized = normalizePersistedMemberState(memberName, value, updatedAt);
if (!normalized) continue;
normalizedMembers[normalized.name] = normalized;
}
return createPersistedLaunchSnapshot({
teamName:
typeof record.teamName === 'string' && record.teamName.trim().length > 0
? record.teamName.trim()
: teamName,
expectedMembers,
leadSessionId:
typeof record.leadSessionId === 'string' && record.leadSessionId.trim().length > 0
? record.leadSessionId.trim()
: undefined,
launchPhase:
record.launchPhase === 'active' ||
record.launchPhase === 'finished' ||
record.launchPhase === 'reconciled'
? record.launchPhase
: 'finished',
members: normalizedMembers,
updatedAt,
});
}

View file

@ -0,0 +1,56 @@
import { getTeamsBasePath } from '@main/utils/pathDecoder';
import { createLogger } from '@shared/utils/logger';
import * as fs from 'fs';
import * as path from 'path';
import { normalizePersistedLaunchSnapshot } from './TeamLaunchStateEvaluator';
import { atomicWriteAsync } from './atomicWrite';
import type { PersistedTeamLaunchSnapshot } from '@shared/types';
const logger = createLogger('Service:TeamLaunchStateStore');
const TEAM_LAUNCH_STATE_FILE = 'launch-state.json';
const MAX_LAUNCH_STATE_BYTES = 256 * 1024;
export function getTeamLaunchStatePath(teamName: string): string {
return path.join(getTeamsBasePath(), teamName, TEAM_LAUNCH_STATE_FILE);
}
export class TeamLaunchStateStore {
async read(teamName: string): Promise<PersistedTeamLaunchSnapshot | null> {
const targetPath = getTeamLaunchStatePath(teamName);
try {
const stat = await fs.promises.stat(targetPath);
if (!stat.isFile() || stat.size > MAX_LAUNCH_STATE_BYTES) {
return null;
}
const raw = await fs.promises.readFile(targetPath, 'utf8');
return normalizePersistedLaunchSnapshot(teamName, JSON.parse(raw));
} catch {
return null;
}
}
async write(teamName: string, snapshot: PersistedTeamLaunchSnapshot): Promise<void> {
try {
await atomicWriteAsync(
getTeamLaunchStatePath(teamName),
`${JSON.stringify(snapshot, null, 2)}\n`
);
} catch (error) {
logger.warn(
`[${teamName}] Failed to persist launch-state: ${
error instanceof Error ? error.message : String(error)
}`
);
}
}
async clear(teamName: string): Promise<void> {
try {
await fs.promises.rm(getTeamLaunchStatePath(teamName), { force: true });
} catch {
// best-effort
}
}
}

View file

@ -49,6 +49,7 @@ import { createCliAutoSuffixNameGuard, parseNumericSuffixName } from '@shared/ut
import { normalizeOptionalTeamProviderId } from '@shared/utils/teamProvider'; import { normalizeOptionalTeamProviderId } from '@shared/utils/teamProvider';
import { import {
extractToolPreview, extractToolPreview,
parseAgentToolResultStatus,
extractToolResultPreview, extractToolResultPreview,
formatToolSummaryFromCalls, formatToolSummaryFromCalls,
} from '@shared/utils/toolSummary'; } from '@shared/utils/toolSummary';
@ -71,6 +72,12 @@ import { TeamMembersMetaStore } from './TeamMembersMetaStore';
import { TeamMetaStore } from './TeamMetaStore'; import { TeamMetaStore } from './TeamMetaStore';
import { TeamSentMessagesStore } from './TeamSentMessagesStore'; import { TeamSentMessagesStore } from './TeamSentMessagesStore';
import { TeamTaskReader } from './TeamTaskReader'; import { TeamTaskReader } from './TeamTaskReader';
import { TeamLaunchStateStore } from './TeamLaunchStateStore';
import {
createPersistedLaunchSnapshot,
snapshotFromRuntimeMemberStatuses,
snapshotToMemberSpawnStatuses,
} from './TeamLaunchStateEvaluator';
import { import {
applyConfiguredRuntimeBackendsEnv, applyConfiguredRuntimeBackendsEnv,
applyProviderRuntimeEnv, applyProviderRuntimeEnv,
@ -110,9 +117,12 @@ import type {
MemberSpawnStatus, MemberSpawnStatus,
MemberSpawnLivenessSource, MemberSpawnLivenessSource,
MemberSpawnStatusEntry, MemberSpawnStatusEntry,
PersistedTeamLaunchPhase,
PersistedTeamLaunchSummary,
TeamChangeEvent, TeamChangeEvent,
TeamCreateRequest, TeamCreateRequest,
TeamCreateResponse, TeamCreateResponse,
TeamLaunchAggregateState,
TeamLaunchRequest, TeamLaunchRequest,
TeamLaunchResponse, TeamLaunchResponse,
TeamProvisioningPrepareResult, TeamProvisioningPrepareResult,
@ -154,7 +164,6 @@ const STALL_WARNING_THRESHOLD_MS = 20_000;
const TEAM_JSON_READ_TIMEOUT_MS = 5_000; const TEAM_JSON_READ_TIMEOUT_MS = 5_000;
const TEAM_CONFIG_MAX_BYTES = 10 * 1024 * 1024; const TEAM_CONFIG_MAX_BYTES = 10 * 1024 * 1024;
const TEAM_INBOX_MAX_BYTES = 2 * 1024 * 1024; const TEAM_INBOX_MAX_BYTES = 2 * 1024 * 1024;
const TEAM_LAUNCH_STATE_FILE = 'launch-state.json';
const CROSS_TEAM_TOOL_RECIPIENT_NAMES = new Set([ const CROSS_TEAM_TOOL_RECIPIENT_NAMES = new Set([
'cross_team_send', 'cross_team_send',
'cross_team_list_targets', 'cross_team_list_targets',
@ -383,16 +392,6 @@ type ValidConfigProbeResult =
| { ok: true; location: TeamsBaseLocation; configPath: string } | { ok: true; location: TeamsBaseLocation; configPath: string }
| { ok: false }; | { ok: false };
interface PartialLaunchStateFile {
version: 1;
state: 'partial_launch_failure';
updatedAt: string;
leadSessionId?: string;
expectedMembers: string[];
confirmedMembers: string[];
missingMembers: string[];
}
function getTeamsBasePathsToProbe(): { location: TeamsBaseLocation; basePath: string }[] { function getTeamsBasePathsToProbe(): { location: TeamsBaseLocation; basePath: string }[] {
const configured = getTeamsBasePath(); const configured = getTeamsBasePath();
const defaultBase = path.join(getAutoDetectedClaudeBasePath(), 'teams'); const defaultBase = path.join(getAutoDetectedClaudeBasePath(), 'teams');
@ -436,10 +435,6 @@ function looksLikeClaudeStdoutJsonFragment(text: string): boolean {
); );
} }
function getTeamLaunchStatePath(teamName: string): string {
return path.join(getTeamsBasePath(), teamName, TEAM_LAUNCH_STATE_FILE);
}
interface ProvisioningRun { interface ProvisioningRun {
runId: string; runId: string;
teamName: string; teamName: string;
@ -649,19 +644,18 @@ function createInitialMemberSpawnStatusEntry(): MemberSpawnStatusEntry {
} }
function deriveMemberLaunchState(entry: { function deriveMemberLaunchState(entry: {
status: MemberSpawnStatus;
agentToolAccepted?: boolean; agentToolAccepted?: boolean;
runtimeAlive?: boolean; runtimeAlive?: boolean;
bootstrapConfirmed?: boolean; bootstrapConfirmed?: boolean;
hardFailure?: boolean; hardFailure?: boolean;
}): MemberLaunchState { }): MemberLaunchState {
if (entry.hardFailure || entry.status === 'error') { if (entry.hardFailure) {
return 'failed_to_start'; return 'failed_to_start';
} }
if (entry.bootstrapConfirmed) { if (entry.bootstrapConfirmed) {
return 'confirmed_alive'; return 'confirmed_alive';
} }
if (entry.runtimeAlive || entry.agentToolAccepted || entry.status === 'waiting') { if (entry.runtimeAlive || entry.agentToolAccepted) {
return 'runtime_pending_bootstrap'; return 'runtime_pending_bootstrap';
} }
return 'starting'; return 'starting';
@ -861,6 +855,20 @@ function buildTeammateAgentBlockReminder(): string {
].join('\n'); ].join('\n');
} }
function extractHeartbeatTimestamp(text: string, fallback?: string): string | undefined {
const trimmed = text.trim();
if (!trimmed) return fallback?.trim() || undefined;
try {
const parsed = JSON.parse(trimmed) as { timestamp?: unknown };
if (typeof parsed.timestamp === 'string' && parsed.timestamp.trim().length > 0) {
return parsed.timestamp.trim();
}
} catch {
// Best-effort only. Non-JSON teammate messages still use the inbox timestamp fallback.
}
return fallback?.trim() || undefined;
}
function extractBootstrapFailureReason(text: string): string | null { function extractBootstrapFailureReason(text: string): string | null {
const trimmed = text.trim(); const trimmed = text.trim();
if (!trimmed) return null; if (!trimmed) return null;
@ -2026,6 +2034,7 @@ export class TeamProvisioningService {
string, string,
NativeSameTeamFingerprint[] NativeSameTeamFingerprint[]
>(); >();
private readonly launchStateStore = new TeamLaunchStateStore();
private teamChangeEmitter: ((event: TeamChangeEvent) => void) | null = null; private teamChangeEmitter: ((event: TeamChangeEvent) => void) | null = null;
private helpOutputCache: string | null = null; private helpOutputCache: string | null = null;
private helpOutputCacheTime = 0; private helpOutputCacheTime = 0;
@ -2659,7 +2668,14 @@ export class TeamProvisioningService {
this.setMemberSpawnStatus(run, from, 'error', reason); this.setMemberSpawnStatus(run, from, 'error', reason);
continue; continue;
} }
this.setMemberSpawnStatus(run, from, 'online', undefined, 'heartbeat'); this.setMemberSpawnStatus(
run,
from,
'online',
undefined,
'heartbeat',
extractHeartbeatTimestamp(message.text, message.timestamp)
);
} }
} }
@ -3011,10 +3027,22 @@ export class TeamProvisioningService {
if (isError) { if (isError) {
const resultPreview = extractToolResultPreview(resultContent); const resultPreview = extractToolResultPreview(resultContent);
this.handleMemberSpawnFailure(run, spawnedMemberName, resultPreview); this.handleMemberSpawnFailure(run, spawnedMemberName, resultPreview);
} else { } else if (active.toolName === 'Agent') {
const parsedStatus = parseAgentToolResultStatus(resultContent);
if (parsedStatus?.status === 'duplicate_skipped') {
const detail =
parsedStatus.reason === 'already_running'
? 'duplicate spawn skipped - already running'
: 'duplicate spawn skipped - bootstrap pending';
this.appendMemberBootstrapDiagnostic(run, spawnedMemberName, detail);
return;
}
// Agent tool_result only confirms that the runtime accepted the spawn. // Agent tool_result only confirms that the runtime accepted the spawn.
// The teammate becomes truly "online" only after the first inbox heartbeat. // The teammate becomes truly "online" only after the first inbox heartbeat.
this.setMemberSpawnStatus(run, spawnedMemberName, 'waiting'); this.setMemberSpawnStatus(run, spawnedMemberName, 'waiting');
} else {
this.setMemberSpawnStatus(run, spawnedMemberName, 'waiting');
} }
} }
} }
@ -3089,7 +3117,8 @@ export class TeamProvisioningService {
memberName: string, memberName: string,
status: MemberSpawnStatus, status: MemberSpawnStatus,
error?: string, error?: string,
livenessSource?: MemberSpawnLivenessSource livenessSource?: MemberSpawnLivenessSource,
heartbeatAt?: string
): void { ): void {
const prev = run.memberSpawnStatuses.get(memberName) ?? createInitialMemberSpawnStatusEntry(); const prev = run.memberSpawnStatuses.get(memberName) ?? createInitialMemberSpawnStatusEntry();
const updatedAt = nowIso(); const updatedAt = nowIso();
@ -3115,7 +3144,7 @@ export class TeamProvisioningService {
next.firstSpawnAcceptedAt = prev.firstSpawnAcceptedAt ?? updatedAt; next.firstSpawnAcceptedAt = prev.firstSpawnAcceptedAt ?? updatedAt;
if (livenessSource === 'heartbeat') { if (livenessSource === 'heartbeat') {
next.bootstrapConfirmed = true; next.bootstrapConfirmed = true;
next.lastHeartbeatAt = updatedAt; next.lastHeartbeatAt = heartbeatAt?.trim() || prev.lastHeartbeatAt || updatedAt;
} }
next.hardFailure = false; next.hardFailure = false;
next.error = undefined; next.error = undefined;
@ -3158,7 +3187,7 @@ export class TeamProvisioningService {
memberName, memberName,
'spawn accepted, waiting for bootstrap' 'spawn accepted, waiting for bootstrap'
); );
} else if (status === 'online' && livenessSource === 'heartbeat') { } else if (status === 'online' && livenessSource === 'heartbeat' && !prev.bootstrapConfirmed) {
this.appendMemberBootstrapDiagnostic( this.appendMemberBootstrapDiagnostic(
run, run,
memberName, memberName,
@ -3184,38 +3213,67 @@ export class TeamProvisioningService {
runId: run.runId, runId: run.runId,
detail: memberName, detail: memberName,
}); });
if (run.isLaunch) {
void this.persistLaunchStateSnapshot(run, run.provisioningComplete ? 'finished' : 'active');
}
} }
/** /**
* Get current member spawn statuses for a team. * Get current member spawn statuses for a team.
* Returns a map of memberName MemberSpawnStatusEntry. * Returns a map of memberName MemberSpawnStatusEntry.
*/ */
getMemberSpawnStatuses(teamName: string): { async getMemberSpawnStatuses(teamName: string): Promise<{
statuses: Record<string, MemberSpawnStatusEntry>; statuses: Record<string, MemberSpawnStatusEntry>;
runId: string | null; runId: string | null;
} { teamLaunchState?: TeamLaunchAggregateState;
launchPhase?: PersistedTeamLaunchPhase;
expectedMembers?: string[];
updatedAt?: string;
summary?: PersistedTeamLaunchSummary;
source?: 'live' | 'persisted' | 'merged';
}> {
const runId = this.getTrackedRunId(teamName); const runId = this.getTrackedRunId(teamName);
if (!runId) return { statuses: {}, runId: null }; if (!runId) {
const run = this.runs.get(runId); return this.reconcilePersistedLaunchState(teamName).then(({ snapshot, statuses }) => ({
if (!run) return { statuses: {}, runId: null }; statuses,
const result: Record<string, MemberSpawnStatusEntry> = {}; runId: null,
for (const [name, entry] of run.memberSpawnStatuses) { teamLaunchState: snapshot?.teamLaunchState,
result[name] = { launchPhase: snapshot?.launchPhase,
status: entry.status, expectedMembers: snapshot?.expectedMembers,
launchState: entry.launchState, updatedAt: snapshot?.updatedAt,
error: entry.error, summary: snapshot?.summary,
hardFailureReason: entry.hardFailureReason, source: snapshot ? 'persisted' : 'persisted',
livenessSource: entry.livenessSource, }));
agentToolAccepted: entry.agentToolAccepted,
runtimeAlive: entry.runtimeAlive,
bootstrapConfirmed: entry.bootstrapConfirmed,
hardFailure: entry.hardFailure,
firstSpawnAcceptedAt: entry.firstSpawnAcceptedAt,
lastHeartbeatAt: entry.lastHeartbeatAt,
updatedAt: entry.updatedAt,
};
} }
return { statuses: result, runId }; const run = this.runs.get(runId);
if (!run) {
return { statuses: {}, runId: null, source: 'persisted' };
}
await this.refreshMemberSpawnStatusesFromLeadInbox(run);
await this.auditMemberSpawnStatuses(run);
await this.persistLaunchStateSnapshot(run, run.provisioningComplete ? 'finished' : 'active');
const persisted = await this.launchStateStore.read(teamName);
const liveSnapshot = snapshotFromRuntimeMemberStatuses({
teamName: run.teamName,
expectedMembers: run.expectedMembers,
leadSessionId: run.detectedSessionId ?? undefined,
launchPhase: run.provisioningComplete ? 'finished' : 'active',
statuses: this.buildRuntimeSpawnStatusRecord(run),
});
const snapshot = persisted ?? liveSnapshot;
const statuses = snapshotToMemberSpawnStatuses(snapshot);
return {
statuses,
runId,
teamLaunchState: snapshot.teamLaunchState,
launchPhase: snapshot.launchPhase,
expectedMembers: snapshot.expectedMembers,
updatedAt: snapshot.updatedAt,
summary: snapshot.summary,
source: persisted ? 'merged' : 'live',
};
} }
private getMemberLaunchGraceKey(run: ProvisioningRun, memberName: string): string { private getMemberLaunchGraceKey(run: ProvisioningRun, memberName: string): string {
@ -4290,7 +4348,7 @@ export class TeamProvisioningService {
this.runs.set(runId, run); this.runs.set(runId, run);
this.provisioningRunByTeam.set(request.teamName, runId); this.provisioningRunByTeam.set(request.teamName, runId);
run.onProgress(run.progress); run.onProgress(run.progress);
await this.clearPartialLaunchState(request.teamName); await this.clearPersistedLaunchState(request.teamName);
const prompt = buildProvisioningPrompt(request, effectiveMemberSpecs); const prompt = buildProvisioningPrompt(request, effectiveMemberSpecs);
const promptSize = getPromptSizeSummary(prompt); const promptSize = getPromptSizeSummary(prompt);
@ -5886,6 +5944,19 @@ export class TeamProvisioningService {
} }
// Only track spawns for this team // Only track spawns for this team
if (teamName !== run.teamName) continue; if (teamName !== run.teamName) continue;
const existing = run.memberSpawnStatuses.get(memberName);
if (
existing &&
!existing.hardFailure &&
(existing.bootstrapConfirmed || existing.runtimeAlive || existing.agentToolAccepted)
) {
this.appendMemberBootstrapDiagnostic(
run,
memberName,
'respawn blocked as duplicate — teammate already alive or bootstrap pending'
);
continue;
}
this.setMemberSpawnStatus(run, memberName, 'spawning'); this.setMemberSpawnStatus(run, memberName, 'spawning');
const toolUseId = typeof part.id === 'string' ? part.id.trim() : ''; const toolUseId = typeof part.id === 'string' ? part.id.trim() : '';
if (toolUseId) { if (toolUseId) {
@ -5940,6 +6011,8 @@ export class TeamProvisioningService {
return; return;
} }
const liveAgentNames = this.getLiveTeamAgentNames(run.teamName);
// Flag any expected member not found in config.json (excluding the lead) // Flag any expected member not found in config.json (excluding the lead)
for (const expected of run.expectedMembers) { for (const expected of run.expectedMembers) {
const current = run.memberSpawnStatuses.get(expected); const current = run.memberSpawnStatuses.get(expected);
@ -5957,10 +6030,8 @@ export class TeamProvisioningService {
}); });
const runtimeAlive = const runtimeAlive =
matchedRuntimeNames.length > 0 && liveAgentNames.has(expected) ||
matchedRuntimeNames.some((runtimeName) => matchedRuntimeNames.some((runtimeName) => liveAgentNames.has(runtimeName));
this.hasLiveTeamAgentProcess(run.teamName, runtimeName)
);
// A teammate may intentionally stay silent after bootstrap. If Claude Code // A teammate may intentionally stay silent after bootstrap. If Claude Code
// registered the runtime and the OS process is still alive, treat it as // registered the runtime and the OS process is still alive, treat it as
@ -6004,8 +6075,12 @@ export class TeamProvisioningService {
} }
private hasLiveTeamAgentProcess(teamName: string, memberName: string): boolean { private hasLiveTeamAgentProcess(teamName: string, memberName: string): boolean {
return this.getLiveTeamAgentNames(teamName).has(memberName);
}
private getLiveTeamAgentNames(teamName: string): Set<string> {
if (process.platform === 'win32') { if (process.platform === 'win32') {
return false; return new Set();
} }
let output = ''; let output = '';
@ -6015,24 +6090,26 @@ export class TeamProvisioningService {
stdio: ['ignore', 'pipe', 'ignore'], stdio: ['ignore', 'pipe', 'ignore'],
}); });
} catch { } catch {
return false; return new Set();
} }
const teamMarker = `--team-name ${teamName}`; const teamMarker = `--team-name ${teamName}`;
const memberMarker = `--agent-id ${memberName}@${teamName}`; const names = new Set<string>();
for (const line of output.split('\n')) {
return output.split('\n').some((line) => {
const trimmed = line.trim(); const trimmed = line.trim();
return trimmed.includes(teamMarker) && trimmed.includes(memberMarker); if (!trimmed.includes(teamMarker)) continue;
}); const match = trimmed.match(/--agent-id\s+([^\s@]+)@/);
if (!match) continue;
const agentName = match[1]?.trim();
if (agentName) {
names.add(agentName);
}
}
return names;
} }
private async clearPartialLaunchState(teamName: string): Promise<void> { private async clearPersistedLaunchState(teamName: string): Promise<void> {
try { await this.launchStateStore.clear(teamName);
await fs.promises.rm(getTeamLaunchStatePath(teamName), { force: true });
} catch {
// best-effort
}
} }
private getFailedSpawnMembers( private getFailedSpawnMembers(
@ -6076,95 +6153,190 @@ export class TeamProvisioningService {
return { confirmedCount, pendingCount, failedCount, runtimeAlivePendingCount }; return { confirmedCount, pendingCount, failedCount, runtimeAlivePendingCount };
} }
private async persistPartialLaunchState(run: ProvisioningRun): Promise<void> { private buildRuntimeSpawnStatusRecord(
run: ProvisioningRun
): Record<string, MemberSpawnStatusEntry> {
const statuses: Record<string, MemberSpawnStatusEntry> = {};
for (const expected of run.expectedMembers) {
statuses[expected] =
run.memberSpawnStatuses.get(expected) ?? createInitialMemberSpawnStatusEntry();
}
return statuses;
}
private async persistLaunchStateSnapshot(
run: ProvisioningRun,
launchPhase: 'active' | 'finished' | 'reconciled' = run.provisioningComplete
? 'finished'
: 'active'
): Promise<void> {
if (!run.isLaunch || !run.expectedMembers || run.expectedMembers.length === 0) { if (!run.isLaunch || !run.expectedMembers || run.expectedMembers.length === 0) {
if (run.isLaunch) { if (run.isLaunch) {
await this.clearPartialLaunchState(run.teamName); await this.clearPersistedLaunchState(run.teamName);
} }
return; return;
} }
const expectedMembers = Array.from( const snapshot = snapshotFromRuntimeMemberStatuses({
new Set( teamName: run.teamName,
run.expectedMembers expectedMembers: run.expectedMembers,
.map((name) => name.trim()) leadSessionId: run.detectedSessionId ?? undefined,
.filter((name) => name.length > 0 && name !== 'user' && !isLeadMember({ name })) launchPhase,
) statuses: this.buildRuntimeSpawnStatusRecord(run),
); });
if (expectedMembers.length === 0) {
await this.clearPartialLaunchState(run.teamName); if (snapshot.teamLaunchState === 'clean_success' && launchPhase !== 'active') {
await this.clearPersistedLaunchState(run.teamName);
return; return;
} }
const configPath = path.join(getTeamsBasePath(), run.teamName, 'config.json'); await this.launchStateStore.write(run.teamName, snapshot);
let registeredMembers = new Set<string>(); }
let leadSessionId: string | undefined;
private async reconcilePersistedLaunchState(teamName: string): Promise<{
snapshot: ReturnType<typeof createPersistedLaunchSnapshot> | null;
statuses: Record<string, MemberSpawnStatusEntry>;
}> {
const persisted = await this.launchStateStore.read(teamName);
if (!persisted) {
return { snapshot: null, statuses: {} };
}
const configPath = path.join(getTeamsBasePath(), teamName, 'config.json');
let configMembers = new Set<string>();
let leadName = 'team-lead';
try { try {
const raw = await tryReadRegularFileUtf8(configPath, { const raw = await tryReadRegularFileUtf8(configPath, {
timeoutMs: TEAM_JSON_READ_TIMEOUT_MS, timeoutMs: TEAM_JSON_READ_TIMEOUT_MS,
maxBytes: TEAM_CONFIG_MAX_BYTES, maxBytes: TEAM_CONFIG_MAX_BYTES,
}); });
if (!raw) { if (raw) {
await this.clearPartialLaunchState(run.teamName); const config = JSON.parse(raw) as {
return; members?: { name?: string; agentType?: string }[];
};
leadName = config.members?.find((member) => isLeadMember(member))?.name?.trim() || leadName;
configMembers = new Set(
(config.members ?? [])
.map((member) => (typeof member?.name === 'string' ? member.name.trim() : ''))
.filter((name) => name.length > 0 && !isLeadMember({ name }))
);
} }
const config = JSON.parse(raw) as {
leadSessionId?: unknown;
members?: { name?: unknown }[];
};
leadSessionId =
typeof config.leadSessionId === 'string' && config.leadSessionId.trim().length > 0
? config.leadSessionId.trim()
: undefined;
registeredMembers = new Set(
(config.members ?? [])
.map((member) => (typeof member?.name === 'string' ? member.name.trim() : ''))
.filter((name) => name.length > 0 && !isLeadMember({ name }))
);
} catch { } catch {
await this.clearPartialLaunchState(run.teamName); // best-effort
return;
} }
const inboxNames = await this.inboxReader let leadInboxMessages: Awaited<ReturnType<TeamInboxReader['getMessagesFor']>> = [];
.listInboxNames(run.teamName)
.catch(() => [] as string[]);
const confirmedMembers = Array.from(
new Set(
[...registeredMembers, ...inboxNames]
.map((name) => name.trim())
.filter((name) => expectedMembers.includes(name))
)
);
const missingMembers = expectedMembers.filter((name) => !confirmedMembers.includes(name));
if (missingMembers.length === 0 || confirmedMembers.length === 0) {
await this.clearPartialLaunchState(run.teamName);
return;
}
const payload: PartialLaunchStateFile = {
version: 1,
state: 'partial_launch_failure',
updatedAt: new Date().toISOString(),
...(leadSessionId ? { leadSessionId } : {}),
expectedMembers,
confirmedMembers,
missingMembers,
};
try { try {
await atomicWriteAsync( leadInboxMessages = await this.inboxReader.getMessagesFor(teamName, leadName);
getTeamLaunchStatePath(run.teamName), } catch {
JSON.stringify(payload, null, 2) // best-effort
);
} catch (error) {
logger.warn(
`[${run.teamName}] Failed to persist partial launch state: ${
error instanceof Error ? error.message : String(error)
}`
);
} }
const liveAgentNames = this.getLiveTeamAgentNames(teamName);
const nextMembers = { ...persisted.members };
const now = nowIso();
for (const expected of persisted.expectedMembers) {
const current = nextMembers[expected] ?? {
name: expected,
launchState: 'starting',
agentToolAccepted: false,
runtimeAlive: false,
bootstrapConfirmed: false,
hardFailure: false,
lastEvaluatedAt: now,
};
const matchedRuntimeNames = [...configMembers].filter((name) => {
if (name === expected) return true;
const parsed = parseNumericSuffixName(name);
return parsed !== null && parsed.suffix >= 2 && parsed.base === expected;
});
const runtimeAlive =
liveAgentNames.has(expected) ||
matchedRuntimeNames.some((runtimeName) => liveAgentNames.has(runtimeName));
const heartbeatMessage = leadInboxMessages.find((message) => {
if (typeof message.from !== 'string' || message.from.trim() !== expected) return false;
if (typeof message.text !== 'string' || message.text.trim().length === 0) return false;
const firstAcceptedAt = current.firstSpawnAcceptedAt
? Date.parse(current.firstSpawnAcceptedAt)
: NaN;
const messageTs = Date.parse(message.timestamp);
if (
Number.isFinite(firstAcceptedAt) &&
Number.isFinite(messageTs) &&
messageTs < firstAcceptedAt
) {
return false;
}
return true;
});
const heartbeatReason = heartbeatMessage
? extractBootstrapFailureReason(heartbeatMessage.text)
: null;
current.runtimeAlive = runtimeAlive;
current.lastRuntimeAliveAt = runtimeAlive ? now : current.lastRuntimeAliveAt;
current.sources = {
...(current.sources ?? {}),
processAlive: runtimeAlive || undefined,
configRegistered: matchedRuntimeNames.length > 0 || undefined,
configDrift:
heartbeatMessage != null && matchedRuntimeNames.length === 0
? true
: current.sources?.configDrift,
inboxHeartbeat: heartbeatMessage != null ? true : current.sources?.inboxHeartbeat,
};
if (heartbeatReason) {
current.hardFailure = true;
current.hardFailureReason = heartbeatReason;
current.sources.hardFailureSignal = true;
} else if (heartbeatMessage) {
current.bootstrapConfirmed = true;
current.lastHeartbeatAt = heartbeatMessage.timestamp;
current.hardFailure = false;
current.hardFailureReason = undefined;
}
const acceptedAtMs =
current.firstSpawnAcceptedAt != null ? Date.parse(current.firstSpawnAcceptedAt) : NaN;
const graceExpired =
current.agentToolAccepted === true &&
Number.isFinite(acceptedAtMs) &&
Date.now() - acceptedAtMs >= MEMBER_LAUNCH_GRACE_MS;
if (
!current.bootstrapConfirmed &&
!current.runtimeAlive &&
!current.hardFailure &&
graceExpired
) {
current.hardFailure = true;
current.hardFailureReason =
current.hardFailureReason ?? 'Teammate did not join within the launch grace window.';
}
current.launchState = deriveMemberLaunchState(current);
current.lastEvaluatedAt = now;
nextMembers[expected] = {
...current,
diagnostics: undefined,
};
}
const reconciled = createPersistedLaunchSnapshot({
teamName,
expectedMembers: persisted.expectedMembers,
leadSessionId: persisted.leadSessionId,
launchPhase: persisted.launchPhase === 'active' ? 'active' : 'reconciled',
members: nextMembers,
updatedAt: now,
});
if (reconciled.teamLaunchState === 'clean_success') {
await this.clearPersistedLaunchState(teamName);
return { snapshot: null, statuses: {} };
}
await this.launchStateStore.write(teamName, reconciled);
return {
snapshot: reconciled,
statuses: snapshotToMemberSpawnStatuses(reconciled),
};
} }
private captureSendMessages(run: ProvisioningRun, content: Record<string, unknown>[]): void { private captureSendMessages(run: ProvisioningRun, content: Record<string, unknown>[]): void {
@ -8411,7 +8583,7 @@ export class TeamProvisioningService {
// Audit: flag any expected member not registered in config.json after launch. // Audit: flag any expected member not registered in config.json after launch.
await this.refreshMemberSpawnStatusesFromLeadInbox(run); await this.refreshMemberSpawnStatusesFromLeadInbox(run);
await this.auditMemberSpawnStatuses(run); await this.auditMemberSpawnStatuses(run);
await this.persistPartialLaunchState(run); await this.persistLaunchStateSnapshot(run, 'finished');
const failedSpawnMembers = this.getFailedSpawnMembers(run); const failedSpawnMembers = this.getFailedSpawnMembers(run);
const launchSummary = this.getMemberLaunchSummary(run); const launchSummary = this.getMemberLaunchSummary(run);
const hasSpawnFailures = failedSpawnMembers.length > 0; const hasSpawnFailures = failedSpawnMembers.length > 0;
@ -8570,7 +8742,7 @@ export class TeamProvisioningService {
// Audit: flag any expected member not registered in config.json after provisioning. // Audit: flag any expected member not registered in config.json after provisioning.
await this.refreshMemberSpawnStatusesFromLeadInbox(run); await this.refreshMemberSpawnStatusesFromLeadInbox(run);
await this.auditMemberSpawnStatuses(run); await this.auditMemberSpawnStatuses(run);
await this.clearPartialLaunchState(run.teamName); await this.persistLaunchStateSnapshot(run, 'finished');
const failedSpawnMembers = this.getFailedSpawnMembers(run); const failedSpawnMembers = this.getFailedSpawnMembers(run);
const launchSummary = this.getMemberLaunchSummary(run); const launchSummary = this.getMemberLaunchSummary(run);
const hasSpawnFailures = failedSpawnMembers.length > 0; const hasSpawnFailures = failedSpawnMembers.length > 0;
@ -8941,7 +9113,7 @@ export class TeamProvisioningService {
*/ */
private cleanupRun(run: ProvisioningRun): void { private cleanupRun(run: ProvisioningRun): void {
if (run.isLaunch && !run.provisioningComplete) { if (run.isLaunch && !run.provisioningComplete) {
void this.persistPartialLaunchState(run); void this.persistLaunchStateSnapshot(run, 'finished');
} }
this.resetRuntimeToolActivity(run); this.resetRuntimeToolActivity(run);
this.setLeadActivity(run, 'offline'); this.setLeadActivity(run, 'offline');

View file

@ -3,6 +3,7 @@ import * as path from 'node:path';
import { parentPort } from 'node:worker_threads'; import { parentPort } from 'node:worker_threads';
import { isLeadMember } from '@shared/utils/leadDetection'; import { isLeadMember } from '@shared/utils/leadDetection';
import { normalizePersistedLaunchSnapshot } from '@main/services/team/TeamLaunchStateEvaluator';
interface ListTeamsPayload { interface ListTeamsPayload {
teamsDir: string; teamsDir: string;
@ -319,48 +320,49 @@ function dropCliProvisionerMembers(
} }
} }
async function readPartialLaunchState( async function readLaunchState(
teamsDir: string, teamsDir: string,
teamName: string teamName: string
): Promise<{ ): Promise<{
partialLaunchFailure: true; partialLaunchFailure?: true;
expectedMemberCount: number; expectedMemberCount?: number;
confirmedMemberCount: number; confirmedMemberCount?: number;
missingMembers: string[]; missingMembers?: string[];
teamLaunchState?: string;
launchUpdatedAt?: string;
confirmedCount?: number;
pendingCount?: number;
failedCount?: number;
runtimeAlivePendingCount?: number;
} | null> { } | null> {
const launchStatePath = path.join(teamsDir, teamName, TEAM_LAUNCH_STATE_FILE); const launchStatePath = path.join(teamsDir, teamName, TEAM_LAUNCH_STATE_FILE);
try { try {
const stat = await fs.promises.stat(launchStatePath); const stat = await fs.promises.stat(launchStatePath);
if (!stat.isFile() || stat.size > MAX_LAUNCH_STATE_BYTES) return null; if (!stat.isFile() || stat.size > MAX_LAUNCH_STATE_BYTES) return null;
const raw = await fs.promises.readFile(launchStatePath, 'utf8'); const raw = await fs.promises.readFile(launchStatePath, 'utf8');
const parsed = JSON.parse(raw) as { const snapshot = normalizePersistedLaunchSnapshot(teamName, JSON.parse(raw));
state?: unknown; if (!snapshot) return null;
expectedMembers?: unknown; const missingMembers = snapshot.expectedMembers.filter((name) => {
confirmedMembers?: unknown; const member = snapshot.members[name];
missingMembers?: unknown; return member?.launchState === 'failed_to_start';
}; });
if (parsed.state !== 'partial_launch_failure') return null;
const expectedMembers = Array.isArray(parsed.expectedMembers)
? parsed.expectedMembers.filter(
(name): name is string => typeof name === 'string' && name.trim().length > 0
)
: [];
const confirmedMembers = Array.isArray(parsed.confirmedMembers)
? parsed.confirmedMembers.filter(
(name): name is string => typeof name === 'string' && name.trim().length > 0
)
: [];
const missingMembers = Array.isArray(parsed.missingMembers)
? parsed.missingMembers.filter(
(name): name is string => typeof name === 'string' && name.trim().length > 0
)
: [];
if (expectedMembers.length === 0 || missingMembers.length === 0) return null;
return { return {
partialLaunchFailure: true, ...(snapshot.teamLaunchState === 'partial_failure'
expectedMemberCount: expectedMembers.length, ? { partialLaunchFailure: true as const }
confirmedMemberCount: confirmedMembers.length, : {}),
missingMembers, ...(snapshot.expectedMembers.length > 0
? { expectedMemberCount: snapshot.expectedMembers.length }
: {}),
...(snapshot.summary.confirmedCount > 0
? { confirmedMemberCount: snapshot.summary.confirmedCount }
: {}),
...(missingMembers.length > 0 ? { missingMembers } : {}),
teamLaunchState: snapshot.teamLaunchState,
launchUpdatedAt: snapshot.updatedAt,
confirmedCount: snapshot.summary.confirmedCount,
pendingCount: snapshot.summary.pendingCount,
failedCount: snapshot.summary.failedCount,
runtimeAlivePendingCount: snapshot.summary.runtimeAlivePendingCount,
}; };
} catch { } catch {
return null; return null;
@ -591,8 +593,8 @@ async function listTeams(
dropCliProvisionerMembers(memberMap); dropCliProvisionerMembers(memberMap);
const members = Array.from(memberMap.values()); const members = Array.from(memberMap.values());
const partialLaunchState = const launchStateSummary =
(await readPartialLaunchState(payload.teamsDir, teamName)) ?? (await readLaunchState(payload.teamsDir, teamName)) ??
(() => { (() => {
if ( if (
!leadSessionId || !leadSessionId ||
@ -628,7 +630,7 @@ async function listTeams(
...(projectPathHistory ? { projectPathHistory } : {}), ...(projectPathHistory ? { projectPathHistory } : {}),
...(sessionHistory ? { sessionHistory } : {}), ...(sessionHistory ? { sessionHistory } : {}),
...(deletedAt ? { deletedAt } : {}), ...(deletedAt ? { deletedAt } : {}),
...(partialLaunchState ?? {}), ...(launchStateSummary ?? {}),
}; };
const ms = nowMs() - t0; const ms = nowMs() - t0;

View file

@ -1557,11 +1557,16 @@ export const TeamDetailView = ({
> >
<span className="flex items-center gap-1.5 text-xs"> <span className="flex items-center gap-1.5 text-xs">
<AlertTriangle size={14} className="shrink-0" /> <AlertTriangle size={14} className="shrink-0" />
{currentTeamSummary?.partialLaunchFailure {currentTeamSummary?.teamLaunchState === 'partial_pending'
? currentTeamSummary.missingMembers?.length ? currentTeamSummary.runtimeAlivePendingCount &&
? `Last launch failed partway — ${currentTeamSummary.missingMembers.length}/${currentTeamSummary.expectedMemberCount ?? currentTeamSummary.missingMembers.length} teammates did not join` currentTeamSummary.runtimeAlivePendingCount > 0
: 'Last launch failed partway' ? `Last launch is still reconciling — ${currentTeamSummary.confirmedCount ?? 0}/${currentTeamSummary.expectedMemberCount ?? currentTeamSummary.memberCount} teammates confirmed alive, ${currentTeamSummary.runtimeAlivePendingCount} runtime${currentTeamSummary.runtimeAlivePendingCount === 1 ? '' : 's'} pending bootstrap`
: 'Team is offline'} : 'Last launch is still reconciling'
: currentTeamSummary?.partialLaunchFailure
? currentTeamSummary.missingMembers?.length
? `Last launch failed partway — ${currentTeamSummary.missingMembers.length}/${currentTeamSummary.expectedMemberCount ?? currentTeamSummary.missingMembers.length} teammates did not join`
: 'Last launch failed partway'
: 'Team is offline'}
</span> </span>
<Button <Button
variant="ghost" variant="ghost"

View file

@ -64,7 +64,13 @@ function generateUniqueName(sourceName: string, existingNames: string[]): string
} }
} }
type TeamStatus = 'active' | 'idle' | 'provisioning' | 'offline' | 'partial_failure'; type TeamStatus =
| 'active'
| 'idle'
| 'provisioning'
| 'offline'
| 'partial_failure'
| 'partial_pending';
function getRecentProjects(team: TeamSummary): string[] { function getRecentProjects(team: TeamSummary): string[] {
const history = team.projectPathHistory; const history = team.projectPathHistory;
@ -174,7 +180,10 @@ function resolveTeamStatus(
) { ) {
return 'provisioning'; return 'provisioning';
} }
if (team.partialLaunchFailure) { if (team.teamLaunchState === 'partial_pending') {
return 'partial_pending';
}
if (team.partialLaunchFailure || team.teamLaunchState === 'partial_failure') {
return 'partial_failure'; return 'partial_failure';
} }
return 'offline'; return 'offline';
@ -217,6 +226,13 @@ const StatusBadge = ({ status }: { status: TeamStatus }): React.JSX.Element => {
Launch failed partway Launch failed partway
</span> </span>
); );
case 'partial_pending':
return (
<span className="inline-flex items-center gap-1 rounded-full bg-amber-500/15 px-2 py-0.5 text-[10px] font-medium text-amber-300">
<span className="size-1.5 rounded-full bg-amber-300" />
Bootstrap pending
</span>
);
} }
}; };
@ -381,7 +397,8 @@ export const TeamListView = (): React.JSX.Element => {
getCurrentProvisioningProgressForTeam(provisioningState, t.teamName), getCurrentProvisioningProgressForTeam(provisioningState, t.teamName),
leadActivityByTeam leadActivityByTeam
); );
const isRunning = status !== 'offline' && status !== 'partial_failure'; const isRunning =
status !== 'offline' && status !== 'partial_failure' && status !== 'partial_pending';
if (filter.selectedStatuses.has('running') && isRunning) return true; if (filter.selectedStatuses.has('running') && isRunning) return true;
if (filter.selectedStatuses.has('offline') && !isRunning) return true; if (filter.selectedStatuses.has('offline') && !isRunning) return true;
return false; return false;
@ -850,7 +867,9 @@ export const TeamListView = (): React.JSX.Element => {
})()} })()}
</div> </div>
<div className="flex shrink-0 gap-1"> <div className="flex shrink-0 gap-1">
{(status === 'offline' || status === 'partial_failure') && {(status === 'offline' ||
status === 'partial_failure' ||
status === 'partial_pending') &&
team.projectPath && ( team.projectPath && (
<Tooltip> <Tooltip>
<TooltipTrigger asChild> <TooltipTrigger asChild>
@ -924,7 +943,13 @@ export const TeamListView = (): React.JSX.Element => {
{team.description || 'No description'} {team.description || 'No description'}
</p> </p>
</div> </div>
{team.partialLaunchFailure ? ( {team.teamLaunchState === 'partial_pending' ? (
<p className="mt-2 text-[11px] text-amber-300">
{team.runtimeAlivePendingCount && team.runtimeAlivePendingCount > 0
? `Last launch is still reconciling — ${team.confirmedCount ?? 0}/${team.expectedMemberCount ?? team.memberCount} teammates confirmed alive, ${team.runtimeAlivePendingCount} runtime${team.runtimeAlivePendingCount === 1 ? '' : 's'} pending bootstrap.`
: 'Last launch is still reconciling.'}
</p>
) : team.partialLaunchFailure || team.teamLaunchState === 'partial_failure' ? (
<p className="mt-2 text-[11px] text-amber-400"> <p className="mt-2 text-[11px] text-amber-400">
{team.missingMembers?.length {team.missingMembers?.length
? `Last launch stopped before ${team.missingMembers.length}/${team.expectedMemberCount ?? team.missingMembers.length} teammate${team.missingMembers.length === 1 ? '' : 's'} joined.` ? `Last launch stopped before ${team.missingMembers.length}/${team.expectedMemberCount ?? team.missingMembers.length} teammate${team.missingMembers.length === 1 ? '' : 's'} joined.`

View file

@ -17,14 +17,16 @@ interface TeamProvisioningBannerProps {
export const TeamProvisioningBanner = ({ export const TeamProvisioningBanner = ({
teamName, teamName,
}: TeamProvisioningBannerProps): React.JSX.Element | null => { }: TeamProvisioningBannerProps): React.JSX.Element | null => {
const { progress, cancelProvisioning, teamMembers, memberSpawnStatuses } = useStore( const { progress, cancelProvisioning, teamMembers, memberSpawnStatuses, memberSpawnSnapshot } =
useShallow((s) => ({ useStore(
progress: getCurrentProvisioningProgressForTeam(s, teamName), useShallow((s) => ({
cancelProvisioning: s.cancelProvisioning, progress: getCurrentProvisioningProgressForTeam(s, teamName),
teamMembers: s.selectedTeamData?.members, cancelProvisioning: s.cancelProvisioning,
memberSpawnStatuses: s.memberSpawnStatusesByTeam[teamName], teamMembers: s.selectedTeamData?.members,
})) memberSpawnStatuses: s.memberSpawnStatusesByTeam[teamName],
); memberSpawnSnapshot: s.memberSpawnSnapshotsByTeam[teamName],
}))
);
const [dismissed, setDismissed] = useState(false); const [dismissed, setDismissed] = useState(false);
const lastActiveStepRef = useRef(-1); const lastActiveStepRef = useRef(-1);
const bannerInstanceKey = useMemo(() => { const bannerInstanceKey = useMemo(() => {
@ -105,38 +107,49 @@ export const TeamProvisioningBanner = ({
} }
const teammates = (teamMembers ?? []).filter((member) => !isLeadMember(member)); const teammates = (teamMembers ?? []).filter((member) => !isLeadMember(member));
const expectedTeammateCount = memberSpawnSnapshot?.expectedMembers?.length;
const fallbackTeammateCount = expectedTeammateCount ?? teammates.length;
const snapshotSummary = memberSpawnSnapshot?.summary;
const failedSpawnEntries = Object.entries(memberSpawnStatuses ?? {}).filter( const failedSpawnEntries = Object.entries(memberSpawnStatuses ?? {}).filter(
([, entry]) => entry.launchState === 'failed_to_start' ([, entry]) => entry.launchState === 'failed_to_start'
); );
const failedSpawnCount = failedSpawnEntries.length; const failedSpawnCount = snapshotSummary?.failedCount ?? failedSpawnEntries.length;
const heartbeatConfirmedCount = teammates.filter((member) => { const heartbeatConfirmedCount =
const entry = memberSpawnStatuses?.[member.name]; snapshotSummary?.confirmedCount ??
return entry?.launchState === 'confirmed_alive'; teammates.filter((member) => {
}).length; const entry = memberSpawnStatuses?.[member.name];
const processOnlyAliveCount = teammates.filter((member) => { return entry?.launchState === 'confirmed_alive';
const entry = memberSpawnStatuses?.[member.name]; }).length;
return entry?.launchState === 'runtime_pending_bootstrap' && entry.runtimeAlive === true; const processOnlyAliveCount =
}).length; snapshotSummary?.runtimeAlivePendingCount ??
const pendingSpawnCount = teammates.filter((member) => { teammates.filter((member) => {
const entry = memberSpawnStatuses?.[member.name]; const entry = memberSpawnStatuses?.[member.name];
return ( return entry?.launchState === 'runtime_pending_bootstrap' && entry.runtimeAlive === true;
entry?.launchState === 'starting' || }).length;
(entry?.launchState === 'runtime_pending_bootstrap' && entry.runtimeAlive !== true) const pendingSpawnCount =
); snapshotSummary?.pendingCount ??
}).length; teammates.filter((member) => {
const entry = memberSpawnStatuses?.[member.name];
return (
entry?.launchState === 'starting' ||
(entry?.launchState === 'runtime_pending_bootstrap' && entry.runtimeAlive !== true)
);
}).length;
const allTeammatesConfirmedAlive = const allTeammatesConfirmedAlive =
teammates.length > 0 && failedSpawnCount === 0 && heartbeatConfirmedCount === teammates.length; fallbackTeammateCount > 0 &&
failedSpawnCount === 0 &&
heartbeatConfirmedCount === fallbackTeammateCount;
if (isReady) { if (isReady) {
const readyMessage = const readyMessage =
failedSpawnCount > 0 failedSpawnCount > 0
? `Launch finished with errors — ${failedSpawnCount}/${Math.max(teammates.length, failedSpawnCount)} teammates failed to start` ? `Launch finished with errors — ${failedSpawnCount}/${Math.max(fallbackTeammateCount, failedSpawnCount)} teammates failed to start`
: teammates.length === 0 : fallbackTeammateCount === 0
? 'Team launched — lead online' ? 'Team launched — lead online'
: allTeammatesConfirmedAlive : allTeammatesConfirmedAlive
? `Team launched — all ${teammates.length} teammates confirmed alive` ? `Team launched — all ${fallbackTeammateCount} teammates confirmed alive`
: processOnlyAliveCount > 0 || pendingSpawnCount > 0 : processOnlyAliveCount > 0 || pendingSpawnCount > 0
? `Team launched — ${heartbeatConfirmedCount}/${teammates.length} teammates confirmed alive${processOnlyAliveCount > 0 ? `, ${processOnlyAliveCount} runtime${processOnlyAliveCount === 1 ? '' : 's'} alive but bootstrap still pending` : ''}${pendingSpawnCount > 0 ? `${processOnlyAliveCount > 0 ? ', ' : ', '}${pendingSpawnCount} still starting` : ''}` ? `Team launched — ${heartbeatConfirmedCount}/${fallbackTeammateCount} teammates confirmed alive${processOnlyAliveCount > 0 ? `, ${processOnlyAliveCount} runtime${processOnlyAliveCount === 1 ? '' : 's'} alive but bootstrap still pending` : ''}${pendingSpawnCount > 0 ? `${processOnlyAliveCount > 0 ? ', ' : ', '}${pendingSpawnCount} still starting` : ''}`
: 'Team launched — teammate liveness is still being confirmed'; : 'Team launched — teammate liveness is still being confirmed';
return ( return (

View file

@ -168,6 +168,7 @@ export function initializeNotificationListeners(): () => void {
const pendingProjectRefreshTimers = new Map<string, ReturnType<typeof setTimeout>>(); const pendingProjectRefreshTimers = new Map<string, ReturnType<typeof setTimeout>>();
let teamRefreshTimers = new Map<string, ReturnType<typeof setTimeout>>(); let teamRefreshTimers = new Map<string, ReturnType<typeof setTimeout>>();
let teamPresenceRefreshTimers = new Map<string, ReturnType<typeof setTimeout>>(); let teamPresenceRefreshTimers = new Map<string, ReturnType<typeof setTimeout>>();
let memberSpawnRefreshTimers = new Map<string, ReturnType<typeof setTimeout>>();
let toolActivityTimers = new Map<string, ReturnType<typeof setTimeout>>(); let toolActivityTimers = new Map<string, ReturnType<typeof setTimeout>>();
let inProgressChangePresencePollInFlight = false; let inProgressChangePresencePollInFlight = false;
const inProgressChangePresenceCursorByTeam = new Map<string, number>(); const inProgressChangePresenceCursorByTeam = new Map<string, number>();
@ -178,6 +179,7 @@ export function initializeNotificationListeners(): () => void {
const PROJECT_REFRESH_DEBOUNCE_MS = 300; const PROJECT_REFRESH_DEBOUNCE_MS = 300;
const TEAM_REFRESH_THROTTLE_MS = 800; const TEAM_REFRESH_THROTTLE_MS = 800;
const TEAM_PRESENCE_REFRESH_THROTTLE_MS = 400; const TEAM_PRESENCE_REFRESH_THROTTLE_MS = 400;
const TEAM_MEMBER_SPAWN_REFRESH_THROTTLE_MS = 500;
const TEAM_LIST_REFRESH_THROTTLE_MS = 2000; const TEAM_LIST_REFRESH_THROTTLE_MS = 2000;
const GLOBAL_TASKS_REFRESH_THROTTLE_MS = 500; const GLOBAL_TASKS_REFRESH_THROTTLE_MS = 500;
const buildToolActivityTimerKey = ( const buildToolActivityTimerKey = (
@ -919,6 +921,20 @@ export function initializeNotificationListeners(): () => void {
return; return;
} }
if (event.type === 'inbox' || event.type === 'config' || event.type === 'process') {
if (!event?.teamName || !isTeamVisibleInAnyPane(event.teamName)) {
return;
}
if (memberSpawnRefreshTimers.has(event.teamName)) {
return;
}
const timer = setTimeout(() => {
memberSpawnRefreshTimers.delete(event.teamName);
void useStore.getState().fetchMemberSpawnStatuses(event.teamName);
}, TEAM_MEMBER_SPAWN_REFRESH_THROTTLE_MS);
memberSpawnRefreshTimers.set(event.teamName, timer);
}
// Live lead-message events: only refresh the visible team detail, not team/task lists. // Live lead-message events: only refresh the visible team detail, not team/task lists.
// This keeps the refresh lightweight and prevents one noisy team from starving another. // This keeps the refresh lightweight and prevents one noisy team from starving another.
if (event.type === 'lead-message') { if (event.type === 'lead-message') {
@ -998,6 +1014,8 @@ export function initializeNotificationListeners(): () => void {
teamRefreshTimers = new Map(); teamRefreshTimers = new Map();
for (const t of teamPresenceRefreshTimers.values()) clearTimeout(t); for (const t of teamPresenceRefreshTimers.values()) clearTimeout(t);
teamPresenceRefreshTimers = new Map(); teamPresenceRefreshTimers = new Map();
for (const t of memberSpawnRefreshTimers.values()) clearTimeout(t);
memberSpawnRefreshTimers = new Map();
for (const t of toolActivityTimers.values()) clearTimeout(t); for (const t of toolActivityTimers.values()) clearTimeout(t);
toolActivityTimers = new Map(); toolActivityTimers = new Map();
if (teamListRefreshTimer) { if (teamListRefreshTimer) {

View file

@ -139,6 +139,7 @@ import type {
KanbanColumnId, KanbanColumnId,
LeadActivityState, LeadActivityState,
LeadContextUsage, LeadContextUsage,
MemberSpawnStatusesSnapshot,
MemberSpawnStatusEntry, MemberSpawnStatusEntry,
SendMessageRequest, SendMessageRequest,
SendMessageResult, SendMessageResult,
@ -625,6 +626,7 @@ export interface TeamSlice {
toolHistoryByTeam: Record<string, Record<string, ActiveToolCall[]>>; toolHistoryByTeam: Record<string, Record<string, ActiveToolCall[]>>;
/** Per-team per-member spawn statuses during team provisioning/launch. */ /** Per-team per-member spawn statuses during team provisioning/launch. */
memberSpawnStatusesByTeam: Record<string, Record<string, MemberSpawnStatusEntry>>; memberSpawnStatusesByTeam: Record<string, Record<string, MemberSpawnStatusEntry>>;
memberSpawnSnapshotsByTeam: Record<string, MemberSpawnStatusesSnapshot>;
fetchMemberSpawnStatuses: (teamName: string) => Promise<void>; fetchMemberSpawnStatuses: (teamName: string) => Promise<void>;
provisioningErrorByTeam: Record<string, string | null>; provisioningErrorByTeam: Record<string, string | null>;
clearProvisioningError: (teamName?: string) => void; clearProvisioningError: (teamName?: string) => void;
@ -910,6 +912,7 @@ export const createTeamSlice: StateCreator<AppState, [], [], TeamSlice> = (set,
finishedVisibleByTeam: {}, finishedVisibleByTeam: {},
toolHistoryByTeam: {}, toolHistoryByTeam: {},
memberSpawnStatusesByTeam: {}, memberSpawnStatusesByTeam: {},
memberSpawnSnapshotsByTeam: {},
provisioningErrorByTeam: {}, provisioningErrorByTeam: {},
clearProvisioningError: (teamName?: string) => clearProvisioningError: (teamName?: string) =>
set((state) => { set((state) => {
@ -971,6 +974,10 @@ export const createTeamSlice: StateCreator<AppState, [], [], TeamSlice> = (set,
...prev.memberSpawnStatusesByTeam, ...prev.memberSpawnStatusesByTeam,
[teamName]: snapshot.statuses, [teamName]: snapshot.statuses,
}, },
memberSpawnSnapshotsByTeam: {
...prev.memberSpawnSnapshotsByTeam,
[teamName]: snapshot,
},
}; };
}); });
} catch { } catch {
@ -1861,6 +1868,8 @@ export const createTeamSlice: StateCreator<AppState, [], [], TeamSlice> = (set,
delete nextErrors[request.teamName]; delete nextErrors[request.teamName];
const nextSpawnStatuses = { ...state.memberSpawnStatusesByTeam }; const nextSpawnStatuses = { ...state.memberSpawnStatusesByTeam };
delete nextSpawnStatuses[request.teamName]; delete nextSpawnStatuses[request.teamName];
const nextSpawnSnapshots = { ...state.memberSpawnSnapshotsByTeam };
delete nextSpawnSnapshots[request.teamName];
const nextActiveTools = { ...state.activeToolsByTeam }; const nextActiveTools = { ...state.activeToolsByTeam };
delete nextActiveTools[request.teamName]; delete nextActiveTools[request.teamName];
const nextFinishedVisible = { ...state.finishedVisibleByTeam }; const nextFinishedVisible = { ...state.finishedVisibleByTeam };
@ -1893,6 +1902,7 @@ export const createTeamSlice: StateCreator<AppState, [], [], TeamSlice> = (set,
provisioningRuns: cleaned, provisioningRuns: cleaned,
provisioningErrorByTeam: nextErrors, provisioningErrorByTeam: nextErrors,
memberSpawnStatusesByTeam: nextSpawnStatuses, memberSpawnStatusesByTeam: nextSpawnStatuses,
memberSpawnSnapshotsByTeam: nextSpawnSnapshots,
activeToolsByTeam: nextActiveTools, activeToolsByTeam: nextActiveTools,
finishedVisibleByTeam: nextFinishedVisible, finishedVisibleByTeam: nextFinishedVisible,
toolHistoryByTeam: nextToolHistory, toolHistoryByTeam: nextToolHistory,
@ -2056,6 +2066,8 @@ export const createTeamSlice: StateCreator<AppState, [], [], TeamSlice> = (set,
delete nextErrors[request.teamName]; delete nextErrors[request.teamName];
const nextSpawnStatuses = { ...state.memberSpawnStatusesByTeam }; const nextSpawnStatuses = { ...state.memberSpawnStatusesByTeam };
delete nextSpawnStatuses[request.teamName]; delete nextSpawnStatuses[request.teamName];
const nextSpawnSnapshots = { ...state.memberSpawnSnapshotsByTeam };
delete nextSpawnSnapshots[request.teamName];
const nextActiveTools = { ...state.activeToolsByTeam }; const nextActiveTools = { ...state.activeToolsByTeam };
delete nextActiveTools[request.teamName]; delete nextActiveTools[request.teamName];
const nextFinishedVisible = { ...state.finishedVisibleByTeam }; const nextFinishedVisible = { ...state.finishedVisibleByTeam };
@ -2088,6 +2100,7 @@ export const createTeamSlice: StateCreator<AppState, [], [], TeamSlice> = (set,
provisioningRuns: cleaned, provisioningRuns: cleaned,
provisioningErrorByTeam: nextErrors, provisioningErrorByTeam: nextErrors,
memberSpawnStatusesByTeam: nextSpawnStatuses, memberSpawnStatusesByTeam: nextSpawnStatuses,
memberSpawnSnapshotsByTeam: nextSpawnSnapshots,
activeToolsByTeam: nextActiveTools, activeToolsByTeam: nextActiveTools,
finishedVisibleByTeam: nextFinishedVisible, finishedVisibleByTeam: nextFinishedVisible,
toolHistoryByTeam: nextToolHistory, toolHistoryByTeam: nextToolHistory,
@ -2248,8 +2261,10 @@ export const createTeamSlice: StateCreator<AppState, [], [], TeamSlice> = (set,
: state.ignoredRuntimeRunIds; : state.ignoredRuntimeRunIds;
const nextSpawnStatuses = { ...state.memberSpawnStatusesByTeam }; const nextSpawnStatuses = { ...state.memberSpawnStatusesByTeam };
const nextSpawnSnapshots = { ...state.memberSpawnSnapshotsByTeam };
if (isCanonicalRun) { if (isCanonicalRun) {
delete nextSpawnStatuses[existing.teamName]; delete nextSpawnStatuses[existing.teamName];
delete nextSpawnSnapshots[existing.teamName];
} }
const nextActiveTools = { ...state.activeToolsByTeam }; const nextActiveTools = { ...state.activeToolsByTeam };
const nextFinishedVisible = { ...state.finishedVisibleByTeam }; const nextFinishedVisible = { ...state.finishedVisibleByTeam };
@ -2265,6 +2280,7 @@ export const createTeamSlice: StateCreator<AppState, [], [], TeamSlice> = (set,
currentProvisioningRunIdByTeam: nextCurrentRunIdByTeam, currentProvisioningRunIdByTeam: nextCurrentRunIdByTeam,
currentRuntimeRunIdByTeam: nextRuntimeRunIdByTeam, currentRuntimeRunIdByTeam: nextRuntimeRunIdByTeam,
memberSpawnStatusesByTeam: nextSpawnStatuses, memberSpawnStatusesByTeam: nextSpawnStatuses,
memberSpawnSnapshotsByTeam: nextSpawnSnapshots,
activeToolsByTeam: nextActiveTools, activeToolsByTeam: nextActiveTools,
finishedVisibleByTeam: nextFinishedVisible, finishedVisibleByTeam: nextFinishedVisible,
toolHistoryByTeam: nextToolHistory, toolHistoryByTeam: nextToolHistory,
@ -2387,13 +2403,20 @@ export const createTeamSlice: StateCreator<AppState, [], [], TeamSlice> = (set,
if (isCanonicalRun && TERMINAL_PROVISIONING_STATES.has(progress.state)) { if (isCanonicalRun && TERMINAL_PROVISIONING_STATES.has(progress.state)) {
set((prev) => { set((prev) => {
const next = { ...prev.memberSpawnStatusesByTeam }; const next = { ...prev.memberSpawnStatusesByTeam };
const nextSnapshots = { ...prev.memberSpawnSnapshotsByTeam };
const currentStatuses = next[progress.teamName]; const currentStatuses = next[progress.teamName];
if (!currentStatuses) { if (!currentStatuses) {
return { memberSpawnStatusesByTeam: next }; return {
memberSpawnStatusesByTeam: next,
memberSpawnSnapshotsByTeam: nextSnapshots,
};
} }
if (progress.state === 'ready') { if (progress.state === 'ready') {
next[progress.teamName] = currentStatuses; next[progress.teamName] = currentStatuses;
return { memberSpawnStatusesByTeam: next }; return {
memberSpawnStatusesByTeam: next,
memberSpawnSnapshotsByTeam: nextSnapshots,
};
} }
const retainedStatuses = Object.fromEntries( const retainedStatuses = Object.fromEntries(
Object.entries(currentStatuses).filter(([, entry]) => entry.status === 'error') Object.entries(currentStatuses).filter(([, entry]) => entry.status === 'error')
@ -2402,8 +2425,12 @@ export const createTeamSlice: StateCreator<AppState, [], [], TeamSlice> = (set,
next[progress.teamName] = retainedStatuses; next[progress.teamName] = retainedStatuses;
} else { } else {
delete next[progress.teamName]; delete next[progress.teamName];
delete nextSnapshots[progress.teamName];
} }
return { memberSpawnStatusesByTeam: next }; return {
memberSpawnStatusesByTeam: next,
memberSpawnSnapshotsByTeam: nextSnapshots,
};
}); });
} }

View file

@ -66,6 +66,15 @@ export interface TeamSummary {
confirmedMemberCount?: number; confirmedMemberCount?: number;
/** Missing teammate names from the last partial launch marker. */ /** Missing teammate names from the last partial launch marker. */
missingMembers?: string[]; missingMembers?: string[];
/** Durable aggregate launch state derived from persisted launch-state evidence. */
teamLaunchState?: TeamLaunchAggregateState;
/** ISO timestamp of the last durable launch-state evaluation. */
launchUpdatedAt?: string;
/** Durable aggregate teammate counts from launch-state evidence. */
confirmedCount?: number;
pendingCount?: number;
failedCount?: number;
runtimeAlivePendingCount?: number;
} }
export type TeamTaskStatus = 'pending' | 'in_progress' | 'completed' | 'deleted'; export type TeamTaskStatus = 'pending' | 'in_progress' | 'completed' | 'deleted';
@ -446,6 +455,8 @@ export type MemberLaunchState =
| 'runtime_pending_bootstrap' | 'runtime_pending_bootstrap'
| 'confirmed_alive' | 'confirmed_alive'
| 'failed_to_start'; | 'failed_to_start';
export type TeamLaunchAggregateState = 'clean_success' | 'partial_pending' | 'partial_failure';
export type PersistedTeamLaunchPhase = 'active' | 'finished' | 'reconciled';
export type KanbanColumnId = 'todo' | 'in_progress' | 'done' | 'review' | 'approved'; export type KanbanColumnId = 'todo' | 'in_progress' | 'done' | 'review' | 'approved';
@ -575,9 +586,60 @@ export interface LeadContextUsageSnapshot {
runId: string | null; runId: string | null;
} }
export interface PersistedTeamLaunchMemberSources {
inboxHeartbeat?: boolean;
nativeHeartbeat?: boolean;
processAlive?: boolean;
configRegistered?: boolean;
configDrift?: boolean;
hardFailureSignal?: boolean;
duplicateRespawnBlocked?: boolean;
}
export interface PersistedTeamLaunchMemberState {
name: string;
launchState: MemberLaunchState;
agentToolAccepted: boolean;
runtimeAlive: boolean;
bootstrapConfirmed: boolean;
hardFailure: boolean;
hardFailureReason?: string;
firstSpawnAcceptedAt?: string;
lastHeartbeatAt?: string;
lastRuntimeAliveAt?: string;
lastEvaluatedAt: string;
sources?: PersistedTeamLaunchMemberSources;
diagnostics?: string[];
}
export interface PersistedTeamLaunchSummary {
confirmedCount: number;
pendingCount: number;
failedCount: number;
runtimeAlivePendingCount: number;
}
export interface PersistedTeamLaunchSnapshot {
version: 2;
teamName: string;
updatedAt: string;
leadSessionId?: string;
launchPhase: PersistedTeamLaunchPhase;
expectedMembers: string[];
members: Record<string, PersistedTeamLaunchMemberState>;
summary: PersistedTeamLaunchSummary;
teamLaunchState: TeamLaunchAggregateState;
}
export interface MemberSpawnStatusesSnapshot { export interface MemberSpawnStatusesSnapshot {
statuses: Record<string, MemberSpawnStatusEntry>; statuses: Record<string, MemberSpawnStatusEntry>;
runId: string | null; runId: string | null;
teamLaunchState?: TeamLaunchAggregateState;
launchPhase?: PersistedTeamLaunchPhase;
expectedMembers?: string[];
updatedAt?: string;
summary?: PersistedTeamLaunchSummary;
source?: 'live' | 'persisted' | 'merged';
} }
export type MemberSpawnLivenessSource = 'heartbeat' | 'process'; export type MemberSpawnLivenessSource = 'heartbeat' | 'process';

View file

@ -5,6 +5,15 @@ export interface ToolSummaryData {
byName: Record<string, number>; byName: Record<string, number>;
} }
export type AgentToolDuplicateSkipReason = 'already_running' | 'bootstrap_pending';
export interface ParsedAgentToolResultStatus {
status: 'duplicate_skipped';
reason: AgentToolDuplicateSkipReason;
name?: string;
teamName?: string;
}
export function buildToolSummary(content: Record<string, unknown>[]): string | undefined { export function buildToolSummary(content: Record<string, unknown>[]): string | undefined {
const counts = new Map<string, number>(); const counts = new Map<string, number>();
for (const block of content) { for (const block of content) {
@ -228,3 +237,40 @@ export function extractToolResultPreview(content: unknown, max = 80): string | u
if (!joined) return undefined; if (!joined) return undefined;
return truncateStr(joined, max); return truncateStr(joined, max);
} }
/**
* Parse machine-readable Agent tool_result status lines from the raw tool_result content.
* Returns null for any non-Agent or non-duplicate result.
*/
export function parseAgentToolResultStatus(content: unknown): ParsedAgentToolResultStatus | null {
const joined = flattenToolResultContent(content).join('\n').trim();
if (!joined) return null;
const fields = new Map<string, string>();
for (const rawLine of joined.split(/\r?\n/)) {
const line = rawLine.trim();
if (!line) continue;
const separatorIndex = line.indexOf(':');
if (separatorIndex <= 0) continue;
const key = line.slice(0, separatorIndex).trim().toLowerCase();
const value = line.slice(separatorIndex + 1).trim();
if (!value) continue;
fields.set(key, value);
}
if (fields.get('status') !== 'duplicate_skipped') {
return null;
}
const reason = fields.get('reason');
if (reason !== 'already_running' && reason !== 'bootstrap_pending') {
return null;
}
return {
status: 'duplicate_skipped',
reason,
name: fields.get('name'),
teamName: fields.get('team_name'),
};
}