Persist launch-state and duplicate no-op handling
This commit is contained in:
iliya
parent
2959a3d074
commit
30fb2501d3
12 changed files with 1102 additions and 252 deletions
|
|
@ -12,6 +12,7 @@ import * as path from 'path';
|
|||
import { getTeamFsWorkerClient } from './TeamFsWorkerClient';
|
||||
import { TeamMembersMetaStore } from './TeamMembersMetaStore';
|
||||
import { TeamMetaStore } from './TeamMetaStore';
|
||||
import { normalizePersistedLaunchSnapshot } from './TeamLaunchStateEvaluator';
|
||||
|
||||
import type { TeamConfig, TeamMember, TeamSummary, TeamSummaryMember } from '@shared/types';
|
||||
|
||||
|
|
@ -27,16 +28,20 @@ const MAX_PROJECT_PATH_HISTORY_IN_SUMMARY = 200;
|
|||
const MAX_LAUNCH_STATE_BYTES = 32 * 1024;
|
||||
const TEAM_LAUNCH_STATE_FILE = 'launch-state.json';
|
||||
|
||||
interface PartialLaunchStateSummary {
|
||||
partialLaunchFailure: true;
|
||||
expectedMemberCount: number;
|
||||
confirmedMemberCount: number;
|
||||
missingMembers: string[];
|
||||
interface LaunchStateSummary {
|
||||
partialLaunchFailure?: true;
|
||||
expectedMemberCount?: number;
|
||||
confirmedMemberCount?: number;
|
||||
missingMembers?: string[];
|
||||
teamLaunchState?: TeamSummary['teamLaunchState'];
|
||||
launchUpdatedAt?: string;
|
||||
confirmedCount?: number;
|
||||
pendingCount?: number;
|
||||
failedCount?: number;
|
||||
runtimeAlivePendingCount?: number;
|
||||
}
|
||||
|
||||
async function readPartialLaunchStateSummary(
|
||||
teamDir: string
|
||||
): Promise<PartialLaunchStateSummary | null> {
|
||||
async function readLaunchStateSummary(teamDir: string): Promise<LaunchStateSummary | null> {
|
||||
const launchStatePath = path.join(teamDir, TEAM_LAUNCH_STATE_FILE);
|
||||
try {
|
||||
const stat = await fs.promises.stat(launchStatePath);
|
||||
|
|
@ -44,38 +49,31 @@ async function readPartialLaunchStateSummary(
|
|||
return null;
|
||||
}
|
||||
const raw = await readFileUtf8WithTimeout(launchStatePath, PER_TEAM_READ_TIMEOUT_MS);
|
||||
const parsed = JSON.parse(raw) as {
|
||||
state?: unknown;
|
||||
expectedMembers?: unknown;
|
||||
confirmedMembers?: unknown;
|
||||
missingMembers?: unknown;
|
||||
};
|
||||
if (parsed.state !== 'partial_launch_failure') {
|
||||
return null;
|
||||
}
|
||||
const expectedMembers = Array.isArray(parsed.expectedMembers)
|
||||
? parsed.expectedMembers.filter(
|
||||
(name): name is string => typeof name === 'string' && name.trim().length > 0
|
||||
)
|
||||
: [];
|
||||
const confirmedMembers = Array.isArray(parsed.confirmedMembers)
|
||||
? parsed.confirmedMembers.filter(
|
||||
(name): name is string => typeof name === 'string' && name.trim().length > 0
|
||||
)
|
||||
: [];
|
||||
const missingMembers = Array.isArray(parsed.missingMembers)
|
||||
? parsed.missingMembers.filter(
|
||||
(name): name is string => typeof name === 'string' && name.trim().length > 0
|
||||
)
|
||||
: [];
|
||||
if (expectedMembers.length === 0 || missingMembers.length === 0) {
|
||||
const snapshot = normalizePersistedLaunchSnapshot(path.basename(teamDir), JSON.parse(raw));
|
||||
if (!snapshot) {
|
||||
return null;
|
||||
}
|
||||
const missingMembers = snapshot.expectedMembers.filter((name) => {
|
||||
const member = snapshot.members[name];
|
||||
return member?.launchState === 'failed_to_start';
|
||||
});
|
||||
return {
|
||||
partialLaunchFailure: true,
|
||||
expectedMemberCount: expectedMembers.length,
|
||||
confirmedMemberCount: confirmedMembers.length,
|
||||
missingMembers,
|
||||
...(snapshot.teamLaunchState === 'partial_failure'
|
||||
? { partialLaunchFailure: true as const }
|
||||
: {}),
|
||||
...(snapshot.expectedMembers.length > 0
|
||||
? { expectedMemberCount: snapshot.expectedMembers.length }
|
||||
: {}),
|
||||
...(snapshot.summary.confirmedCount > 0
|
||||
? { confirmedMemberCount: snapshot.summary.confirmedCount }
|
||||
: {}),
|
||||
...(missingMembers.length > 0 ? { missingMembers } : {}),
|
||||
teamLaunchState: snapshot.teamLaunchState,
|
||||
launchUpdatedAt: snapshot.updatedAt,
|
||||
confirmedCount: snapshot.summary.confirmedCount,
|
||||
pendingCount: snapshot.summary.pendingCount,
|
||||
failedCount: snapshot.summary.failedCount,
|
||||
runtimeAlivePendingCount: snapshot.summary.runtimeAlivePendingCount,
|
||||
};
|
||||
} catch {
|
||||
return null;
|
||||
|
|
@ -340,8 +338,8 @@ export class TeamConfigReader {
|
|||
}
|
||||
|
||||
const members = Array.from(memberMap.values());
|
||||
const partialLaunchState =
|
||||
(await readPartialLaunchStateSummary(teamDir)) ??
|
||||
const launchStateSummary =
|
||||
(await readLaunchStateSummary(teamDir)) ??
|
||||
(() => {
|
||||
if (
|
||||
!leadSessionId ||
|
||||
|
|
@ -377,7 +375,7 @@ export class TeamConfigReader {
|
|||
...(projectPathHistory ? { projectPathHistory } : {}),
|
||||
...(sessionHistory ? { sessionHistory } : {}),
|
||||
...(deletedAt ? { deletedAt } : {}),
|
||||
...(partialLaunchState ?? {}),
|
||||
...(launchStateSummary ?? {}),
|
||||
};
|
||||
return summary;
|
||||
} catch {
|
||||
|
|
|
|||
426
src/main/services/team/TeamLaunchStateEvaluator.ts
Normal file
426
src/main/services/team/TeamLaunchStateEvaluator.ts
Normal file
|
|
@ -0,0 +1,426 @@
|
|||
import { isLeadMember } from '@shared/utils/leadDetection';
|
||||
|
||||
import type {
|
||||
MemberLaunchState,
|
||||
MemberSpawnLivenessSource,
|
||||
MemberSpawnStatusEntry,
|
||||
PersistedTeamLaunchMemberSources,
|
||||
PersistedTeamLaunchMemberState,
|
||||
PersistedTeamLaunchPhase,
|
||||
PersistedTeamLaunchSnapshot,
|
||||
PersistedTeamLaunchSummary,
|
||||
TeamLaunchAggregateState,
|
||||
} from '@shared/types';
|
||||
|
||||
interface LegacyPartialLaunchStateFile {
|
||||
version?: unknown;
|
||||
state?: unknown;
|
||||
updatedAt?: unknown;
|
||||
leadSessionId?: unknown;
|
||||
expectedMembers?: unknown;
|
||||
confirmedMembers?: unknown;
|
||||
missingMembers?: unknown;
|
||||
}
|
||||
|
||||
type RuntimeMemberSpawnState = Pick<
|
||||
MemberSpawnStatusEntry,
|
||||
| 'launchState'
|
||||
| 'status'
|
||||
| 'error'
|
||||
| 'hardFailureReason'
|
||||
| 'livenessSource'
|
||||
| 'agentToolAccepted'
|
||||
| 'runtimeAlive'
|
||||
| 'bootstrapConfirmed'
|
||||
| 'hardFailure'
|
||||
| 'firstSpawnAcceptedAt'
|
||||
| 'lastHeartbeatAt'
|
||||
| 'updatedAt'
|
||||
>;
|
||||
|
||||
function normalizeMemberName(name: string): string {
|
||||
return name.trim();
|
||||
}
|
||||
|
||||
function buildDiagnostics(
|
||||
member: Pick<
|
||||
PersistedTeamLaunchMemberState,
|
||||
'agentToolAccepted' | 'runtimeAlive' | 'bootstrapConfirmed' | 'hardFailureReason' | 'sources'
|
||||
>
|
||||
): string[] {
|
||||
const diagnostics: string[] = [];
|
||||
if (member.agentToolAccepted) diagnostics.push('spawn accepted');
|
||||
if (member.runtimeAlive) diagnostics.push('runtime alive');
|
||||
if (member.bootstrapConfirmed) diagnostics.push('late heartbeat received');
|
||||
if (member.runtimeAlive && !member.bootstrapConfirmed) diagnostics.push('waiting for bootstrap');
|
||||
if (member.hardFailureReason)
|
||||
diagnostics.push(`hard failure reason: ${member.hardFailureReason}`);
|
||||
if (member.sources?.duplicateRespawnBlocked) diagnostics.push('respawn blocked as duplicate');
|
||||
if (member.sources?.configDrift) diagnostics.push('config drift detected');
|
||||
return diagnostics;
|
||||
}
|
||||
|
||||
export function deriveTeamLaunchAggregateState(
|
||||
summary: PersistedTeamLaunchSummary
|
||||
): TeamLaunchAggregateState {
|
||||
if (summary.failedCount > 0) {
|
||||
return 'partial_failure';
|
||||
}
|
||||
if (summary.pendingCount > 0) {
|
||||
return 'partial_pending';
|
||||
}
|
||||
return 'clean_success';
|
||||
}
|
||||
|
||||
export function summarizePersistedLaunchMembers(
|
||||
expectedMembers: readonly string[],
|
||||
members: Record<string, PersistedTeamLaunchMemberState>
|
||||
): PersistedTeamLaunchSummary {
|
||||
let confirmedCount = 0;
|
||||
let pendingCount = 0;
|
||||
let failedCount = 0;
|
||||
let runtimeAlivePendingCount = 0;
|
||||
const normalizedExpected = expectedMembers.map(normalizeMemberName).filter(Boolean);
|
||||
|
||||
for (const memberName of normalizedExpected) {
|
||||
const entry = members[memberName];
|
||||
if (!entry) {
|
||||
pendingCount += 1;
|
||||
continue;
|
||||
}
|
||||
if (entry.launchState === 'confirmed_alive') {
|
||||
confirmedCount += 1;
|
||||
continue;
|
||||
}
|
||||
if (entry.launchState === 'failed_to_start') {
|
||||
failedCount += 1;
|
||||
continue;
|
||||
}
|
||||
pendingCount += 1;
|
||||
if (entry.runtimeAlive) {
|
||||
runtimeAlivePendingCount += 1;
|
||||
}
|
||||
}
|
||||
|
||||
return { confirmedCount, pendingCount, failedCount, runtimeAlivePendingCount };
|
||||
}
|
||||
|
||||
function deriveMemberLaunchState(
|
||||
member: Pick<
|
||||
PersistedTeamLaunchMemberState,
|
||||
'hardFailure' | 'bootstrapConfirmed' | 'runtimeAlive' | 'agentToolAccepted'
|
||||
>
|
||||
): MemberLaunchState {
|
||||
if (member.hardFailure) {
|
||||
return 'failed_to_start';
|
||||
}
|
||||
if (member.bootstrapConfirmed) {
|
||||
return 'confirmed_alive';
|
||||
}
|
||||
if (member.runtimeAlive || member.agentToolAccepted) {
|
||||
return 'runtime_pending_bootstrap';
|
||||
}
|
||||
return 'starting';
|
||||
}
|
||||
|
||||
function toBoolean(value: unknown): boolean {
|
||||
return value === true;
|
||||
}
|
||||
|
||||
function normalizeSources(value: unknown): PersistedTeamLaunchMemberSources | undefined {
|
||||
if (!value || typeof value !== 'object') {
|
||||
return undefined;
|
||||
}
|
||||
const source = value as Record<string, unknown>;
|
||||
const normalized: PersistedTeamLaunchMemberSources = {
|
||||
inboxHeartbeat: toBoolean(source.inboxHeartbeat),
|
||||
nativeHeartbeat: toBoolean(source.nativeHeartbeat),
|
||||
processAlive: toBoolean(source.processAlive),
|
||||
configRegistered: toBoolean(source.configRegistered),
|
||||
configDrift: toBoolean(source.configDrift),
|
||||
hardFailureSignal: toBoolean(source.hardFailureSignal),
|
||||
duplicateRespawnBlocked: toBoolean(source.duplicateRespawnBlocked),
|
||||
};
|
||||
return Object.values(normalized).some(Boolean) ? normalized : undefined;
|
||||
}
|
||||
|
||||
function normalizePersistedMemberState(
|
||||
memberName: string,
|
||||
value: unknown,
|
||||
updatedAtFallback: string
|
||||
): PersistedTeamLaunchMemberState | null {
|
||||
if (!value || typeof value !== 'object') {
|
||||
return null;
|
||||
}
|
||||
const parsed = value as Record<string, unknown>;
|
||||
const normalizedName = normalizeMemberName(memberName);
|
||||
if (!normalizedName || normalizedName === 'user' || isLeadMember({ name: normalizedName })) {
|
||||
return null;
|
||||
}
|
||||
const next: PersistedTeamLaunchMemberState = {
|
||||
name: normalizedName,
|
||||
launchState: 'starting',
|
||||
agentToolAccepted: toBoolean(parsed.agentToolAccepted),
|
||||
runtimeAlive: toBoolean(parsed.runtimeAlive),
|
||||
bootstrapConfirmed: toBoolean(parsed.bootstrapConfirmed),
|
||||
hardFailure: toBoolean(parsed.hardFailure),
|
||||
hardFailureReason:
|
||||
typeof parsed.hardFailureReason === 'string' && parsed.hardFailureReason.trim().length > 0
|
||||
? parsed.hardFailureReason.trim()
|
||||
: undefined,
|
||||
firstSpawnAcceptedAt:
|
||||
typeof parsed.firstSpawnAcceptedAt === 'string' ? parsed.firstSpawnAcceptedAt : undefined,
|
||||
lastHeartbeatAt:
|
||||
typeof parsed.lastHeartbeatAt === 'string' ? parsed.lastHeartbeatAt : undefined,
|
||||
lastRuntimeAliveAt:
|
||||
typeof parsed.lastRuntimeAliveAt === 'string' ? parsed.lastRuntimeAliveAt : undefined,
|
||||
lastEvaluatedAt:
|
||||
typeof parsed.lastEvaluatedAt === 'string' ? parsed.lastEvaluatedAt : updatedAtFallback,
|
||||
sources: normalizeSources(parsed.sources),
|
||||
diagnostics: Array.isArray(parsed.diagnostics)
|
||||
? parsed.diagnostics.filter(
|
||||
(item): item is string => typeof item === 'string' && item.trim().length > 0
|
||||
)
|
||||
: undefined,
|
||||
};
|
||||
const launchState =
|
||||
parsed.launchState === 'starting' ||
|
||||
parsed.launchState === 'runtime_pending_bootstrap' ||
|
||||
parsed.launchState === 'confirmed_alive' ||
|
||||
parsed.launchState === 'failed_to_start'
|
||||
? parsed.launchState
|
||||
: deriveMemberLaunchState(next);
|
||||
next.launchState = launchState;
|
||||
next.diagnostics = next.diagnostics?.length ? next.diagnostics : buildDiagnostics(next);
|
||||
return next;
|
||||
}
|
||||
|
||||
export function createPersistedLaunchSnapshot(params: {
|
||||
teamName: string;
|
||||
expectedMembers: readonly string[];
|
||||
leadSessionId?: string;
|
||||
launchPhase?: PersistedTeamLaunchPhase;
|
||||
members?: Record<string, PersistedTeamLaunchMemberState>;
|
||||
updatedAt?: string;
|
||||
}): PersistedTeamLaunchSnapshot {
|
||||
const updatedAt = params.updatedAt ?? new Date().toISOString();
|
||||
const expectedMembers = Array.from(
|
||||
new Set(
|
||||
params.expectedMembers
|
||||
.map(normalizeMemberName)
|
||||
.filter((name) => name.length > 0 && name !== 'user' && !isLeadMember({ name }))
|
||||
)
|
||||
);
|
||||
const members = params.members ?? {};
|
||||
const summary = summarizePersistedLaunchMembers(expectedMembers, members);
|
||||
return {
|
||||
version: 2,
|
||||
teamName: params.teamName,
|
||||
updatedAt,
|
||||
...(params.leadSessionId ? { leadSessionId: params.leadSessionId } : {}),
|
||||
launchPhase: params.launchPhase ?? 'active',
|
||||
expectedMembers,
|
||||
members,
|
||||
summary,
|
||||
teamLaunchState: deriveTeamLaunchAggregateState(summary),
|
||||
};
|
||||
}
|
||||
|
||||
export function snapshotFromRuntimeMemberStatuses(params: {
|
||||
teamName: string;
|
||||
expectedMembers: readonly string[];
|
||||
leadSessionId?: string;
|
||||
launchPhase?: PersistedTeamLaunchPhase;
|
||||
statuses: Record<string, RuntimeMemberSpawnState>;
|
||||
updatedAt?: string;
|
||||
}): PersistedTeamLaunchSnapshot {
|
||||
const updatedAt = params.updatedAt ?? new Date().toISOString();
|
||||
const members: Record<string, PersistedTeamLaunchMemberState> = {};
|
||||
|
||||
for (const expected of params.expectedMembers) {
|
||||
const name = normalizeMemberName(expected);
|
||||
if (!name || name === 'user' || isLeadMember({ name })) continue;
|
||||
const runtime = params.statuses[name];
|
||||
const sources: PersistedTeamLaunchMemberSources = {};
|
||||
if (runtime?.livenessSource === 'heartbeat') {
|
||||
sources.nativeHeartbeat = true;
|
||||
sources.inboxHeartbeat = true;
|
||||
}
|
||||
if (runtime?.livenessSource === 'process' || runtime?.runtimeAlive) {
|
||||
sources.processAlive = true;
|
||||
}
|
||||
const entry: PersistedTeamLaunchMemberState = {
|
||||
name,
|
||||
launchState: runtime?.launchState ?? 'starting',
|
||||
agentToolAccepted: runtime?.agentToolAccepted === true,
|
||||
runtimeAlive: runtime?.runtimeAlive === true,
|
||||
bootstrapConfirmed: runtime?.bootstrapConfirmed === true,
|
||||
hardFailure: runtime?.hardFailure === true || runtime?.launchState === 'failed_to_start',
|
||||
hardFailureReason: runtime?.hardFailureReason ?? runtime?.error,
|
||||
firstSpawnAcceptedAt: runtime?.firstSpawnAcceptedAt,
|
||||
lastHeartbeatAt: runtime?.lastHeartbeatAt,
|
||||
lastRuntimeAliveAt: runtime?.runtimeAlive ? updatedAt : undefined,
|
||||
lastEvaluatedAt: runtime?.updatedAt ?? updatedAt,
|
||||
sources: Object.values(sources).some(Boolean) ? sources : undefined,
|
||||
diagnostics: undefined,
|
||||
};
|
||||
entry.launchState = deriveMemberLaunchState(entry);
|
||||
entry.diagnostics = buildDiagnostics(entry);
|
||||
members[name] = entry;
|
||||
}
|
||||
|
||||
return createPersistedLaunchSnapshot({
|
||||
teamName: params.teamName,
|
||||
expectedMembers: params.expectedMembers,
|
||||
leadSessionId: params.leadSessionId,
|
||||
launchPhase: params.launchPhase,
|
||||
members,
|
||||
updatedAt,
|
||||
});
|
||||
}
|
||||
|
||||
export function snapshotToMemberSpawnStatuses(
|
||||
snapshot: PersistedTeamLaunchSnapshot | null
|
||||
): Record<string, MemberSpawnStatusEntry> {
|
||||
if (!snapshot) return {};
|
||||
const statuses: Record<string, MemberSpawnStatusEntry> = {};
|
||||
for (const memberName of snapshot.expectedMembers) {
|
||||
const entry = snapshot.members[memberName];
|
||||
if (!entry) continue;
|
||||
let status: MemberSpawnStatusEntry['status'] = 'offline';
|
||||
let livenessSource: MemberSpawnLivenessSource | undefined;
|
||||
if (entry.launchState === 'failed_to_start') {
|
||||
status = 'error';
|
||||
} else if (entry.launchState === 'confirmed_alive') {
|
||||
status = 'online';
|
||||
livenessSource = 'heartbeat';
|
||||
} else if (entry.launchState === 'runtime_pending_bootstrap') {
|
||||
status = entry.runtimeAlive ? 'online' : 'waiting';
|
||||
livenessSource = entry.runtimeAlive ? 'process' : undefined;
|
||||
} else {
|
||||
status = entry.agentToolAccepted ? 'waiting' : 'spawning';
|
||||
}
|
||||
statuses[memberName] = {
|
||||
status,
|
||||
launchState: entry.launchState,
|
||||
error: entry.hardFailure ? entry.hardFailureReason : undefined,
|
||||
hardFailureReason: entry.hardFailureReason,
|
||||
livenessSource,
|
||||
agentToolAccepted: entry.agentToolAccepted,
|
||||
runtimeAlive: entry.runtimeAlive,
|
||||
bootstrapConfirmed: entry.bootstrapConfirmed,
|
||||
hardFailure: entry.hardFailure,
|
||||
firstSpawnAcceptedAt: entry.firstSpawnAcceptedAt,
|
||||
lastHeartbeatAt: entry.lastHeartbeatAt,
|
||||
updatedAt: entry.lastEvaluatedAt,
|
||||
};
|
||||
}
|
||||
return statuses;
|
||||
}
|
||||
|
||||
export function normalizePersistedLaunchSnapshot(
|
||||
teamName: string,
|
||||
parsed: unknown
|
||||
): PersistedTeamLaunchSnapshot | null {
|
||||
if (!parsed || typeof parsed !== 'object') {
|
||||
return null;
|
||||
}
|
||||
|
||||
const maybeLegacy = parsed as LegacyPartialLaunchStateFile;
|
||||
if (maybeLegacy.state === 'partial_launch_failure') {
|
||||
const expectedMembers = Array.isArray(maybeLegacy.expectedMembers)
|
||||
? maybeLegacy.expectedMembers.filter(
|
||||
(name): name is string => typeof name === 'string' && normalizeMemberName(name).length > 0
|
||||
)
|
||||
: [];
|
||||
const confirmedMembers = Array.isArray(maybeLegacy.confirmedMembers)
|
||||
? maybeLegacy.confirmedMembers.filter(
|
||||
(name): name is string => typeof name === 'string' && normalizeMemberName(name).length > 0
|
||||
)
|
||||
: [];
|
||||
const missingMembers = Array.isArray(maybeLegacy.missingMembers)
|
||||
? maybeLegacy.missingMembers.filter(
|
||||
(name): name is string => typeof name === 'string' && normalizeMemberName(name).length > 0
|
||||
)
|
||||
: [];
|
||||
if (expectedMembers.length === 0 || missingMembers.length === 0) {
|
||||
return null;
|
||||
}
|
||||
const updatedAt =
|
||||
typeof maybeLegacy.updatedAt === 'string' ? maybeLegacy.updatedAt : new Date().toISOString();
|
||||
const members: Record<string, PersistedTeamLaunchMemberState> = {};
|
||||
for (const name of expectedMembers) {
|
||||
const failed = missingMembers.includes(name);
|
||||
const confirmed = confirmedMembers.includes(name);
|
||||
const entry: PersistedTeamLaunchMemberState = {
|
||||
name,
|
||||
launchState: failed ? 'failed_to_start' : confirmed ? 'confirmed_alive' : 'starting',
|
||||
agentToolAccepted: true,
|
||||
runtimeAlive: confirmed,
|
||||
bootstrapConfirmed: confirmed,
|
||||
hardFailure: failed,
|
||||
hardFailureReason: failed
|
||||
? 'Legacy partial launch marker reported teammate missing.'
|
||||
: undefined,
|
||||
lastEvaluatedAt: updatedAt,
|
||||
diagnostics: undefined,
|
||||
};
|
||||
entry.diagnostics = buildDiagnostics(entry);
|
||||
members[name] = entry;
|
||||
}
|
||||
return createPersistedLaunchSnapshot({
|
||||
teamName,
|
||||
expectedMembers,
|
||||
leadSessionId:
|
||||
typeof maybeLegacy.leadSessionId === 'string' && maybeLegacy.leadSessionId.trim().length > 0
|
||||
? maybeLegacy.leadSessionId.trim()
|
||||
: undefined,
|
||||
launchPhase: 'finished',
|
||||
members,
|
||||
updatedAt,
|
||||
});
|
||||
}
|
||||
|
||||
const record = parsed as Record<string, unknown>;
|
||||
if (record.version !== 2) {
|
||||
return null;
|
||||
}
|
||||
const expectedMembers = Array.isArray(record.expectedMembers)
|
||||
? record.expectedMembers.filter(
|
||||
(name): name is string => typeof name === 'string' && normalizeMemberName(name).length > 0
|
||||
)
|
||||
: [];
|
||||
const updatedAt =
|
||||
typeof record.updatedAt === 'string' && record.updatedAt.trim().length > 0
|
||||
? record.updatedAt
|
||||
: new Date().toISOString();
|
||||
const normalizedMembers: Record<string, PersistedTeamLaunchMemberState> = {};
|
||||
const rawMembers =
|
||||
record.members && typeof record.members === 'object'
|
||||
? (record.members as Record<string, unknown>)
|
||||
: {};
|
||||
for (const [memberName, value] of Object.entries(rawMembers)) {
|
||||
const normalized = normalizePersistedMemberState(memberName, value, updatedAt);
|
||||
if (!normalized) continue;
|
||||
normalizedMembers[normalized.name] = normalized;
|
||||
}
|
||||
return createPersistedLaunchSnapshot({
|
||||
teamName:
|
||||
typeof record.teamName === 'string' && record.teamName.trim().length > 0
|
||||
? record.teamName.trim()
|
||||
: teamName,
|
||||
expectedMembers,
|
||||
leadSessionId:
|
||||
typeof record.leadSessionId === 'string' && record.leadSessionId.trim().length > 0
|
||||
? record.leadSessionId.trim()
|
||||
: undefined,
|
||||
launchPhase:
|
||||
record.launchPhase === 'active' ||
|
||||
record.launchPhase === 'finished' ||
|
||||
record.launchPhase === 'reconciled'
|
||||
? record.launchPhase
|
||||
: 'finished',
|
||||
members: normalizedMembers,
|
||||
updatedAt,
|
||||
});
|
||||
}
|
||||
56
src/main/services/team/TeamLaunchStateStore.ts
Normal file
56
src/main/services/team/TeamLaunchStateStore.ts
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
import { getTeamsBasePath } from '@main/utils/pathDecoder';
|
||||
import { createLogger } from '@shared/utils/logger';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
import { normalizePersistedLaunchSnapshot } from './TeamLaunchStateEvaluator';
|
||||
import { atomicWriteAsync } from './atomicWrite';
|
||||
|
||||
import type { PersistedTeamLaunchSnapshot } from '@shared/types';
|
||||
|
||||
const logger = createLogger('Service:TeamLaunchStateStore');
|
||||
const TEAM_LAUNCH_STATE_FILE = 'launch-state.json';
|
||||
const MAX_LAUNCH_STATE_BYTES = 256 * 1024;
|
||||
|
||||
export function getTeamLaunchStatePath(teamName: string): string {
|
||||
return path.join(getTeamsBasePath(), teamName, TEAM_LAUNCH_STATE_FILE);
|
||||
}
|
||||
|
||||
export class TeamLaunchStateStore {
|
||||
async read(teamName: string): Promise<PersistedTeamLaunchSnapshot | null> {
|
||||
const targetPath = getTeamLaunchStatePath(teamName);
|
||||
try {
|
||||
const stat = await fs.promises.stat(targetPath);
|
||||
if (!stat.isFile() || stat.size > MAX_LAUNCH_STATE_BYTES) {
|
||||
return null;
|
||||
}
|
||||
const raw = await fs.promises.readFile(targetPath, 'utf8');
|
||||
return normalizePersistedLaunchSnapshot(teamName, JSON.parse(raw));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async write(teamName: string, snapshot: PersistedTeamLaunchSnapshot): Promise<void> {
|
||||
try {
|
||||
await atomicWriteAsync(
|
||||
getTeamLaunchStatePath(teamName),
|
||||
`${JSON.stringify(snapshot, null, 2)}\n`
|
||||
);
|
||||
} catch (error) {
|
||||
logger.warn(
|
||||
`[${teamName}] Failed to persist launch-state: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async clear(teamName: string): Promise<void> {
|
||||
try {
|
||||
await fs.promises.rm(getTeamLaunchStatePath(teamName), { force: true });
|
||||
} catch {
|
||||
// best-effort
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -49,6 +49,7 @@ import { createCliAutoSuffixNameGuard, parseNumericSuffixName } from '@shared/ut
|
|||
import { normalizeOptionalTeamProviderId } from '@shared/utils/teamProvider';
|
||||
import {
|
||||
extractToolPreview,
|
||||
parseAgentToolResultStatus,
|
||||
extractToolResultPreview,
|
||||
formatToolSummaryFromCalls,
|
||||
} from '@shared/utils/toolSummary';
|
||||
|
|
@ -71,6 +72,12 @@ import { TeamMembersMetaStore } from './TeamMembersMetaStore';
|
|||
import { TeamMetaStore } from './TeamMetaStore';
|
||||
import { TeamSentMessagesStore } from './TeamSentMessagesStore';
|
||||
import { TeamTaskReader } from './TeamTaskReader';
|
||||
import { TeamLaunchStateStore } from './TeamLaunchStateStore';
|
||||
import {
|
||||
createPersistedLaunchSnapshot,
|
||||
snapshotFromRuntimeMemberStatuses,
|
||||
snapshotToMemberSpawnStatuses,
|
||||
} from './TeamLaunchStateEvaluator';
|
||||
import {
|
||||
applyConfiguredRuntimeBackendsEnv,
|
||||
applyProviderRuntimeEnv,
|
||||
|
|
@ -110,9 +117,12 @@ import type {
|
|||
MemberSpawnStatus,
|
||||
MemberSpawnLivenessSource,
|
||||
MemberSpawnStatusEntry,
|
||||
PersistedTeamLaunchPhase,
|
||||
PersistedTeamLaunchSummary,
|
||||
TeamChangeEvent,
|
||||
TeamCreateRequest,
|
||||
TeamCreateResponse,
|
||||
TeamLaunchAggregateState,
|
||||
TeamLaunchRequest,
|
||||
TeamLaunchResponse,
|
||||
TeamProvisioningPrepareResult,
|
||||
|
|
@ -154,7 +164,6 @@ const STALL_WARNING_THRESHOLD_MS = 20_000;
|
|||
const TEAM_JSON_READ_TIMEOUT_MS = 5_000;
|
||||
const TEAM_CONFIG_MAX_BYTES = 10 * 1024 * 1024;
|
||||
const TEAM_INBOX_MAX_BYTES = 2 * 1024 * 1024;
|
||||
const TEAM_LAUNCH_STATE_FILE = 'launch-state.json';
|
||||
const CROSS_TEAM_TOOL_RECIPIENT_NAMES = new Set([
|
||||
'cross_team_send',
|
||||
'cross_team_list_targets',
|
||||
|
|
@ -383,16 +392,6 @@ type ValidConfigProbeResult =
|
|||
| { ok: true; location: TeamsBaseLocation; configPath: string }
|
||||
| { ok: false };
|
||||
|
||||
interface PartialLaunchStateFile {
|
||||
version: 1;
|
||||
state: 'partial_launch_failure';
|
||||
updatedAt: string;
|
||||
leadSessionId?: string;
|
||||
expectedMembers: string[];
|
||||
confirmedMembers: string[];
|
||||
missingMembers: string[];
|
||||
}
|
||||
|
||||
function getTeamsBasePathsToProbe(): { location: TeamsBaseLocation; basePath: string }[] {
|
||||
const configured = getTeamsBasePath();
|
||||
const defaultBase = path.join(getAutoDetectedClaudeBasePath(), 'teams');
|
||||
|
|
@ -436,10 +435,6 @@ function looksLikeClaudeStdoutJsonFragment(text: string): boolean {
|
|||
);
|
||||
}
|
||||
|
||||
function getTeamLaunchStatePath(teamName: string): string {
|
||||
return path.join(getTeamsBasePath(), teamName, TEAM_LAUNCH_STATE_FILE);
|
||||
}
|
||||
|
||||
interface ProvisioningRun {
|
||||
runId: string;
|
||||
teamName: string;
|
||||
|
|
@ -649,19 +644,18 @@ function createInitialMemberSpawnStatusEntry(): MemberSpawnStatusEntry {
|
|||
}
|
||||
|
||||
function deriveMemberLaunchState(entry: {
|
||||
status: MemberSpawnStatus;
|
||||
agentToolAccepted?: boolean;
|
||||
runtimeAlive?: boolean;
|
||||
bootstrapConfirmed?: boolean;
|
||||
hardFailure?: boolean;
|
||||
}): MemberLaunchState {
|
||||
if (entry.hardFailure || entry.status === 'error') {
|
||||
if (entry.hardFailure) {
|
||||
return 'failed_to_start';
|
||||
}
|
||||
if (entry.bootstrapConfirmed) {
|
||||
return 'confirmed_alive';
|
||||
}
|
||||
if (entry.runtimeAlive || entry.agentToolAccepted || entry.status === 'waiting') {
|
||||
if (entry.runtimeAlive || entry.agentToolAccepted) {
|
||||
return 'runtime_pending_bootstrap';
|
||||
}
|
||||
return 'starting';
|
||||
|
|
@ -861,6 +855,20 @@ function buildTeammateAgentBlockReminder(): string {
|
|||
].join('\n');
|
||||
}
|
||||
|
||||
function extractHeartbeatTimestamp(text: string, fallback?: string): string | undefined {
|
||||
const trimmed = text.trim();
|
||||
if (!trimmed) return fallback?.trim() || undefined;
|
||||
try {
|
||||
const parsed = JSON.parse(trimmed) as { timestamp?: unknown };
|
||||
if (typeof parsed.timestamp === 'string' && parsed.timestamp.trim().length > 0) {
|
||||
return parsed.timestamp.trim();
|
||||
}
|
||||
} catch {
|
||||
// Best-effort only. Non-JSON teammate messages still use the inbox timestamp fallback.
|
||||
}
|
||||
return fallback?.trim() || undefined;
|
||||
}
|
||||
|
||||
function extractBootstrapFailureReason(text: string): string | null {
|
||||
const trimmed = text.trim();
|
||||
if (!trimmed) return null;
|
||||
|
|
@ -2026,6 +2034,7 @@ export class TeamProvisioningService {
|
|||
string,
|
||||
NativeSameTeamFingerprint[]
|
||||
>();
|
||||
private readonly launchStateStore = new TeamLaunchStateStore();
|
||||
private teamChangeEmitter: ((event: TeamChangeEvent) => void) | null = null;
|
||||
private helpOutputCache: string | null = null;
|
||||
private helpOutputCacheTime = 0;
|
||||
|
|
@ -2659,7 +2668,14 @@ export class TeamProvisioningService {
|
|||
this.setMemberSpawnStatus(run, from, 'error', reason);
|
||||
continue;
|
||||
}
|
||||
this.setMemberSpawnStatus(run, from, 'online', undefined, 'heartbeat');
|
||||
this.setMemberSpawnStatus(
|
||||
run,
|
||||
from,
|
||||
'online',
|
||||
undefined,
|
||||
'heartbeat',
|
||||
extractHeartbeatTimestamp(message.text, message.timestamp)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -3011,10 +3027,22 @@ export class TeamProvisioningService {
|
|||
if (isError) {
|
||||
const resultPreview = extractToolResultPreview(resultContent);
|
||||
this.handleMemberSpawnFailure(run, spawnedMemberName, resultPreview);
|
||||
} else {
|
||||
} else if (active.toolName === 'Agent') {
|
||||
const parsedStatus = parseAgentToolResultStatus(resultContent);
|
||||
if (parsedStatus?.status === 'duplicate_skipped') {
|
||||
const detail =
|
||||
parsedStatus.reason === 'already_running'
|
||||
? 'duplicate spawn skipped - already running'
|
||||
: 'duplicate spawn skipped - bootstrap pending';
|
||||
this.appendMemberBootstrapDiagnostic(run, spawnedMemberName, detail);
|
||||
return;
|
||||
}
|
||||
|
||||
// Agent tool_result only confirms that the runtime accepted the spawn.
|
||||
// The teammate becomes truly "online" only after the first inbox heartbeat.
|
||||
this.setMemberSpawnStatus(run, spawnedMemberName, 'waiting');
|
||||
} else {
|
||||
this.setMemberSpawnStatus(run, spawnedMemberName, 'waiting');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -3089,7 +3117,8 @@ export class TeamProvisioningService {
|
|||
memberName: string,
|
||||
status: MemberSpawnStatus,
|
||||
error?: string,
|
||||
livenessSource?: MemberSpawnLivenessSource
|
||||
livenessSource?: MemberSpawnLivenessSource,
|
||||
heartbeatAt?: string
|
||||
): void {
|
||||
const prev = run.memberSpawnStatuses.get(memberName) ?? createInitialMemberSpawnStatusEntry();
|
||||
const updatedAt = nowIso();
|
||||
|
|
@ -3115,7 +3144,7 @@ export class TeamProvisioningService {
|
|||
next.firstSpawnAcceptedAt = prev.firstSpawnAcceptedAt ?? updatedAt;
|
||||
if (livenessSource === 'heartbeat') {
|
||||
next.bootstrapConfirmed = true;
|
||||
next.lastHeartbeatAt = updatedAt;
|
||||
next.lastHeartbeatAt = heartbeatAt?.trim() || prev.lastHeartbeatAt || updatedAt;
|
||||
}
|
||||
next.hardFailure = false;
|
||||
next.error = undefined;
|
||||
|
|
@ -3158,7 +3187,7 @@ export class TeamProvisioningService {
|
|||
memberName,
|
||||
'spawn accepted, waiting for bootstrap'
|
||||
);
|
||||
} else if (status === 'online' && livenessSource === 'heartbeat') {
|
||||
} else if (status === 'online' && livenessSource === 'heartbeat' && !prev.bootstrapConfirmed) {
|
||||
this.appendMemberBootstrapDiagnostic(
|
||||
run,
|
||||
memberName,
|
||||
|
|
@ -3184,38 +3213,67 @@ export class TeamProvisioningService {
|
|||
runId: run.runId,
|
||||
detail: memberName,
|
||||
});
|
||||
if (run.isLaunch) {
|
||||
void this.persistLaunchStateSnapshot(run, run.provisioningComplete ? 'finished' : 'active');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current member spawn statuses for a team.
|
||||
* Returns a map of memberName → MemberSpawnStatusEntry.
|
||||
*/
|
||||
getMemberSpawnStatuses(teamName: string): {
|
||||
async getMemberSpawnStatuses(teamName: string): Promise<{
|
||||
statuses: Record<string, MemberSpawnStatusEntry>;
|
||||
runId: string | null;
|
||||
} {
|
||||
teamLaunchState?: TeamLaunchAggregateState;
|
||||
launchPhase?: PersistedTeamLaunchPhase;
|
||||
expectedMembers?: string[];
|
||||
updatedAt?: string;
|
||||
summary?: PersistedTeamLaunchSummary;
|
||||
source?: 'live' | 'persisted' | 'merged';
|
||||
}> {
|
||||
const runId = this.getTrackedRunId(teamName);
|
||||
if (!runId) return { statuses: {}, runId: null };
|
||||
const run = this.runs.get(runId);
|
||||
if (!run) return { statuses: {}, runId: null };
|
||||
const result: Record<string, MemberSpawnStatusEntry> = {};
|
||||
for (const [name, entry] of run.memberSpawnStatuses) {
|
||||
result[name] = {
|
||||
status: entry.status,
|
||||
launchState: entry.launchState,
|
||||
error: entry.error,
|
||||
hardFailureReason: entry.hardFailureReason,
|
||||
livenessSource: entry.livenessSource,
|
||||
agentToolAccepted: entry.agentToolAccepted,
|
||||
runtimeAlive: entry.runtimeAlive,
|
||||
bootstrapConfirmed: entry.bootstrapConfirmed,
|
||||
hardFailure: entry.hardFailure,
|
||||
firstSpawnAcceptedAt: entry.firstSpawnAcceptedAt,
|
||||
lastHeartbeatAt: entry.lastHeartbeatAt,
|
||||
updatedAt: entry.updatedAt,
|
||||
};
|
||||
if (!runId) {
|
||||
return this.reconcilePersistedLaunchState(teamName).then(({ snapshot, statuses }) => ({
|
||||
statuses,
|
||||
runId: null,
|
||||
teamLaunchState: snapshot?.teamLaunchState,
|
||||
launchPhase: snapshot?.launchPhase,
|
||||
expectedMembers: snapshot?.expectedMembers,
|
||||
updatedAt: snapshot?.updatedAt,
|
||||
summary: snapshot?.summary,
|
||||
source: snapshot ? 'persisted' : 'persisted',
|
||||
}));
|
||||
}
|
||||
return { statuses: result, runId };
|
||||
const run = this.runs.get(runId);
|
||||
if (!run) {
|
||||
return { statuses: {}, runId: null, source: 'persisted' };
|
||||
}
|
||||
|
||||
await this.refreshMemberSpawnStatusesFromLeadInbox(run);
|
||||
await this.auditMemberSpawnStatuses(run);
|
||||
await this.persistLaunchStateSnapshot(run, run.provisioningComplete ? 'finished' : 'active');
|
||||
|
||||
const persisted = await this.launchStateStore.read(teamName);
|
||||
const liveSnapshot = snapshotFromRuntimeMemberStatuses({
|
||||
teamName: run.teamName,
|
||||
expectedMembers: run.expectedMembers,
|
||||
leadSessionId: run.detectedSessionId ?? undefined,
|
||||
launchPhase: run.provisioningComplete ? 'finished' : 'active',
|
||||
statuses: this.buildRuntimeSpawnStatusRecord(run),
|
||||
});
|
||||
const snapshot = persisted ?? liveSnapshot;
|
||||
const statuses = snapshotToMemberSpawnStatuses(snapshot);
|
||||
return {
|
||||
statuses,
|
||||
runId,
|
||||
teamLaunchState: snapshot.teamLaunchState,
|
||||
launchPhase: snapshot.launchPhase,
|
||||
expectedMembers: snapshot.expectedMembers,
|
||||
updatedAt: snapshot.updatedAt,
|
||||
summary: snapshot.summary,
|
||||
source: persisted ? 'merged' : 'live',
|
||||
};
|
||||
}
|
||||
|
||||
private getMemberLaunchGraceKey(run: ProvisioningRun, memberName: string): string {
|
||||
|
|
@ -4290,7 +4348,7 @@ export class TeamProvisioningService {
|
|||
this.runs.set(runId, run);
|
||||
this.provisioningRunByTeam.set(request.teamName, runId);
|
||||
run.onProgress(run.progress);
|
||||
await this.clearPartialLaunchState(request.teamName);
|
||||
await this.clearPersistedLaunchState(request.teamName);
|
||||
|
||||
const prompt = buildProvisioningPrompt(request, effectiveMemberSpecs);
|
||||
const promptSize = getPromptSizeSummary(prompt);
|
||||
|
|
@ -5886,6 +5944,19 @@ export class TeamProvisioningService {
|
|||
}
|
||||
// Only track spawns for this team
|
||||
if (teamName !== run.teamName) continue;
|
||||
const existing = run.memberSpawnStatuses.get(memberName);
|
||||
if (
|
||||
existing &&
|
||||
!existing.hardFailure &&
|
||||
(existing.bootstrapConfirmed || existing.runtimeAlive || existing.agentToolAccepted)
|
||||
) {
|
||||
this.appendMemberBootstrapDiagnostic(
|
||||
run,
|
||||
memberName,
|
||||
'respawn blocked as duplicate — teammate already alive or bootstrap pending'
|
||||
);
|
||||
continue;
|
||||
}
|
||||
this.setMemberSpawnStatus(run, memberName, 'spawning');
|
||||
const toolUseId = typeof part.id === 'string' ? part.id.trim() : '';
|
||||
if (toolUseId) {
|
||||
|
|
@ -5940,6 +6011,8 @@ export class TeamProvisioningService {
|
|||
return;
|
||||
}
|
||||
|
||||
const liveAgentNames = this.getLiveTeamAgentNames(run.teamName);
|
||||
|
||||
// Flag any expected member not found in config.json (excluding the lead)
|
||||
for (const expected of run.expectedMembers) {
|
||||
const current = run.memberSpawnStatuses.get(expected);
|
||||
|
|
@ -5957,10 +6030,8 @@ export class TeamProvisioningService {
|
|||
});
|
||||
|
||||
const runtimeAlive =
|
||||
matchedRuntimeNames.length > 0 &&
|
||||
matchedRuntimeNames.some((runtimeName) =>
|
||||
this.hasLiveTeamAgentProcess(run.teamName, runtimeName)
|
||||
);
|
||||
liveAgentNames.has(expected) ||
|
||||
matchedRuntimeNames.some((runtimeName) => liveAgentNames.has(runtimeName));
|
||||
|
||||
// A teammate may intentionally stay silent after bootstrap. If Claude Code
|
||||
// registered the runtime and the OS process is still alive, treat it as
|
||||
|
|
@ -6004,8 +6075,12 @@ export class TeamProvisioningService {
|
|||
}
|
||||
|
||||
private hasLiveTeamAgentProcess(teamName: string, memberName: string): boolean {
|
||||
return this.getLiveTeamAgentNames(teamName).has(memberName);
|
||||
}
|
||||
|
||||
private getLiveTeamAgentNames(teamName: string): Set<string> {
|
||||
if (process.platform === 'win32') {
|
||||
return false;
|
||||
return new Set();
|
||||
}
|
||||
|
||||
let output = '';
|
||||
|
|
@ -6015,24 +6090,26 @@ export class TeamProvisioningService {
|
|||
stdio: ['ignore', 'pipe', 'ignore'],
|
||||
});
|
||||
} catch {
|
||||
return false;
|
||||
return new Set();
|
||||
}
|
||||
|
||||
const teamMarker = `--team-name ${teamName}`;
|
||||
const memberMarker = `--agent-id ${memberName}@${teamName}`;
|
||||
|
||||
return output.split('\n').some((line) => {
|
||||
const names = new Set<string>();
|
||||
for (const line of output.split('\n')) {
|
||||
const trimmed = line.trim();
|
||||
return trimmed.includes(teamMarker) && trimmed.includes(memberMarker);
|
||||
});
|
||||
if (!trimmed.includes(teamMarker)) continue;
|
||||
const match = trimmed.match(/--agent-id\s+([^\s@]+)@/);
|
||||
if (!match) continue;
|
||||
const agentName = match[1]?.trim();
|
||||
if (agentName) {
|
||||
names.add(agentName);
|
||||
}
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
private async clearPartialLaunchState(teamName: string): Promise<void> {
|
||||
try {
|
||||
await fs.promises.rm(getTeamLaunchStatePath(teamName), { force: true });
|
||||
} catch {
|
||||
// best-effort
|
||||
}
|
||||
private async clearPersistedLaunchState(teamName: string): Promise<void> {
|
||||
await this.launchStateStore.clear(teamName);
|
||||
}
|
||||
|
||||
private getFailedSpawnMembers(
|
||||
|
|
@ -6076,95 +6153,190 @@ export class TeamProvisioningService {
|
|||
return { confirmedCount, pendingCount, failedCount, runtimeAlivePendingCount };
|
||||
}
|
||||
|
||||
private async persistPartialLaunchState(run: ProvisioningRun): Promise<void> {
|
||||
private buildRuntimeSpawnStatusRecord(
|
||||
run: ProvisioningRun
|
||||
): Record<string, MemberSpawnStatusEntry> {
|
||||
const statuses: Record<string, MemberSpawnStatusEntry> = {};
|
||||
for (const expected of run.expectedMembers) {
|
||||
statuses[expected] =
|
||||
run.memberSpawnStatuses.get(expected) ?? createInitialMemberSpawnStatusEntry();
|
||||
}
|
||||
return statuses;
|
||||
}
|
||||
|
||||
private async persistLaunchStateSnapshot(
|
||||
run: ProvisioningRun,
|
||||
launchPhase: 'active' | 'finished' | 'reconciled' = run.provisioningComplete
|
||||
? 'finished'
|
||||
: 'active'
|
||||
): Promise<void> {
|
||||
if (!run.isLaunch || !run.expectedMembers || run.expectedMembers.length === 0) {
|
||||
if (run.isLaunch) {
|
||||
await this.clearPartialLaunchState(run.teamName);
|
||||
await this.clearPersistedLaunchState(run.teamName);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const expectedMembers = Array.from(
|
||||
new Set(
|
||||
run.expectedMembers
|
||||
.map((name) => name.trim())
|
||||
.filter((name) => name.length > 0 && name !== 'user' && !isLeadMember({ name }))
|
||||
)
|
||||
);
|
||||
if (expectedMembers.length === 0) {
|
||||
await this.clearPartialLaunchState(run.teamName);
|
||||
const snapshot = snapshotFromRuntimeMemberStatuses({
|
||||
teamName: run.teamName,
|
||||
expectedMembers: run.expectedMembers,
|
||||
leadSessionId: run.detectedSessionId ?? undefined,
|
||||
launchPhase,
|
||||
statuses: this.buildRuntimeSpawnStatusRecord(run),
|
||||
});
|
||||
|
||||
if (snapshot.teamLaunchState === 'clean_success' && launchPhase !== 'active') {
|
||||
await this.clearPersistedLaunchState(run.teamName);
|
||||
return;
|
||||
}
|
||||
|
||||
const configPath = path.join(getTeamsBasePath(), run.teamName, 'config.json');
|
||||
let registeredMembers = new Set<string>();
|
||||
let leadSessionId: string | undefined;
|
||||
await this.launchStateStore.write(run.teamName, snapshot);
|
||||
}
|
||||
|
||||
private async reconcilePersistedLaunchState(teamName: string): Promise<{
|
||||
snapshot: ReturnType<typeof createPersistedLaunchSnapshot> | null;
|
||||
statuses: Record<string, MemberSpawnStatusEntry>;
|
||||
}> {
|
||||
const persisted = await this.launchStateStore.read(teamName);
|
||||
if (!persisted) {
|
||||
return { snapshot: null, statuses: {} };
|
||||
}
|
||||
|
||||
const configPath = path.join(getTeamsBasePath(), teamName, 'config.json');
|
||||
let configMembers = new Set<string>();
|
||||
let leadName = 'team-lead';
|
||||
try {
|
||||
const raw = await tryReadRegularFileUtf8(configPath, {
|
||||
timeoutMs: TEAM_JSON_READ_TIMEOUT_MS,
|
||||
maxBytes: TEAM_CONFIG_MAX_BYTES,
|
||||
});
|
||||
if (!raw) {
|
||||
await this.clearPartialLaunchState(run.teamName);
|
||||
return;
|
||||
if (raw) {
|
||||
const config = JSON.parse(raw) as {
|
||||
members?: { name?: string; agentType?: string }[];
|
||||
};
|
||||
leadName = config.members?.find((member) => isLeadMember(member))?.name?.trim() || leadName;
|
||||
configMembers = new Set(
|
||||
(config.members ?? [])
|
||||
.map((member) => (typeof member?.name === 'string' ? member.name.trim() : ''))
|
||||
.filter((name) => name.length > 0 && !isLeadMember({ name }))
|
||||
);
|
||||
}
|
||||
const config = JSON.parse(raw) as {
|
||||
leadSessionId?: unknown;
|
||||
members?: { name?: unknown }[];
|
||||
};
|
||||
leadSessionId =
|
||||
typeof config.leadSessionId === 'string' && config.leadSessionId.trim().length > 0
|
||||
? config.leadSessionId.trim()
|
||||
: undefined;
|
||||
registeredMembers = new Set(
|
||||
(config.members ?? [])
|
||||
.map((member) => (typeof member?.name === 'string' ? member.name.trim() : ''))
|
||||
.filter((name) => name.length > 0 && !isLeadMember({ name }))
|
||||
);
|
||||
} catch {
|
||||
await this.clearPartialLaunchState(run.teamName);
|
||||
return;
|
||||
// best-effort
|
||||
}
|
||||
|
||||
const inboxNames = await this.inboxReader
|
||||
.listInboxNames(run.teamName)
|
||||
.catch(() => [] as string[]);
|
||||
const confirmedMembers = Array.from(
|
||||
new Set(
|
||||
[...registeredMembers, ...inboxNames]
|
||||
.map((name) => name.trim())
|
||||
.filter((name) => expectedMembers.includes(name))
|
||||
)
|
||||
);
|
||||
const missingMembers = expectedMembers.filter((name) => !confirmedMembers.includes(name));
|
||||
|
||||
if (missingMembers.length === 0 || confirmedMembers.length === 0) {
|
||||
await this.clearPartialLaunchState(run.teamName);
|
||||
return;
|
||||
}
|
||||
|
||||
const payload: PartialLaunchStateFile = {
|
||||
version: 1,
|
||||
state: 'partial_launch_failure',
|
||||
updatedAt: new Date().toISOString(),
|
||||
...(leadSessionId ? { leadSessionId } : {}),
|
||||
expectedMembers,
|
||||
confirmedMembers,
|
||||
missingMembers,
|
||||
};
|
||||
|
||||
let leadInboxMessages: Awaited<ReturnType<TeamInboxReader['getMessagesFor']>> = [];
|
||||
try {
|
||||
await atomicWriteAsync(
|
||||
getTeamLaunchStatePath(run.teamName),
|
||||
JSON.stringify(payload, null, 2)
|
||||
);
|
||||
} catch (error) {
|
||||
logger.warn(
|
||||
`[${run.teamName}] Failed to persist partial launch state: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}`
|
||||
);
|
||||
leadInboxMessages = await this.inboxReader.getMessagesFor(teamName, leadName);
|
||||
} catch {
|
||||
// best-effort
|
||||
}
|
||||
|
||||
const liveAgentNames = this.getLiveTeamAgentNames(teamName);
|
||||
const nextMembers = { ...persisted.members };
|
||||
const now = nowIso();
|
||||
for (const expected of persisted.expectedMembers) {
|
||||
const current = nextMembers[expected] ?? {
|
||||
name: expected,
|
||||
launchState: 'starting',
|
||||
agentToolAccepted: false,
|
||||
runtimeAlive: false,
|
||||
bootstrapConfirmed: false,
|
||||
hardFailure: false,
|
||||
lastEvaluatedAt: now,
|
||||
};
|
||||
const matchedRuntimeNames = [...configMembers].filter((name) => {
|
||||
if (name === expected) return true;
|
||||
const parsed = parseNumericSuffixName(name);
|
||||
return parsed !== null && parsed.suffix >= 2 && parsed.base === expected;
|
||||
});
|
||||
const runtimeAlive =
|
||||
liveAgentNames.has(expected) ||
|
||||
matchedRuntimeNames.some((runtimeName) => liveAgentNames.has(runtimeName));
|
||||
const heartbeatMessage = leadInboxMessages.find((message) => {
|
||||
if (typeof message.from !== 'string' || message.from.trim() !== expected) return false;
|
||||
if (typeof message.text !== 'string' || message.text.trim().length === 0) return false;
|
||||
const firstAcceptedAt = current.firstSpawnAcceptedAt
|
||||
? Date.parse(current.firstSpawnAcceptedAt)
|
||||
: NaN;
|
||||
const messageTs = Date.parse(message.timestamp);
|
||||
if (
|
||||
Number.isFinite(firstAcceptedAt) &&
|
||||
Number.isFinite(messageTs) &&
|
||||
messageTs < firstAcceptedAt
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
const heartbeatReason = heartbeatMessage
|
||||
? extractBootstrapFailureReason(heartbeatMessage.text)
|
||||
: null;
|
||||
current.runtimeAlive = runtimeAlive;
|
||||
current.lastRuntimeAliveAt = runtimeAlive ? now : current.lastRuntimeAliveAt;
|
||||
current.sources = {
|
||||
...(current.sources ?? {}),
|
||||
processAlive: runtimeAlive || undefined,
|
||||
configRegistered: matchedRuntimeNames.length > 0 || undefined,
|
||||
configDrift:
|
||||
heartbeatMessage != null && matchedRuntimeNames.length === 0
|
||||
? true
|
||||
: current.sources?.configDrift,
|
||||
inboxHeartbeat: heartbeatMessage != null ? true : current.sources?.inboxHeartbeat,
|
||||
};
|
||||
if (heartbeatReason) {
|
||||
current.hardFailure = true;
|
||||
current.hardFailureReason = heartbeatReason;
|
||||
current.sources.hardFailureSignal = true;
|
||||
} else if (heartbeatMessage) {
|
||||
current.bootstrapConfirmed = true;
|
||||
current.lastHeartbeatAt = heartbeatMessage.timestamp;
|
||||
current.hardFailure = false;
|
||||
current.hardFailureReason = undefined;
|
||||
}
|
||||
const acceptedAtMs =
|
||||
current.firstSpawnAcceptedAt != null ? Date.parse(current.firstSpawnAcceptedAt) : NaN;
|
||||
const graceExpired =
|
||||
current.agentToolAccepted === true &&
|
||||
Number.isFinite(acceptedAtMs) &&
|
||||
Date.now() - acceptedAtMs >= MEMBER_LAUNCH_GRACE_MS;
|
||||
if (
|
||||
!current.bootstrapConfirmed &&
|
||||
!current.runtimeAlive &&
|
||||
!current.hardFailure &&
|
||||
graceExpired
|
||||
) {
|
||||
current.hardFailure = true;
|
||||
current.hardFailureReason =
|
||||
current.hardFailureReason ?? 'Teammate did not join within the launch grace window.';
|
||||
}
|
||||
current.launchState = deriveMemberLaunchState(current);
|
||||
current.lastEvaluatedAt = now;
|
||||
nextMembers[expected] = {
|
||||
...current,
|
||||
diagnostics: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
const reconciled = createPersistedLaunchSnapshot({
|
||||
teamName,
|
||||
expectedMembers: persisted.expectedMembers,
|
||||
leadSessionId: persisted.leadSessionId,
|
||||
launchPhase: persisted.launchPhase === 'active' ? 'active' : 'reconciled',
|
||||
members: nextMembers,
|
||||
updatedAt: now,
|
||||
});
|
||||
|
||||
if (reconciled.teamLaunchState === 'clean_success') {
|
||||
await this.clearPersistedLaunchState(teamName);
|
||||
return { snapshot: null, statuses: {} };
|
||||
}
|
||||
|
||||
await this.launchStateStore.write(teamName, reconciled);
|
||||
return {
|
||||
snapshot: reconciled,
|
||||
statuses: snapshotToMemberSpawnStatuses(reconciled),
|
||||
};
|
||||
}
|
||||
|
||||
private captureSendMessages(run: ProvisioningRun, content: Record<string, unknown>[]): void {
|
||||
|
|
@ -8411,7 +8583,7 @@ export class TeamProvisioningService {
|
|||
// Audit: flag any expected member not registered in config.json after launch.
|
||||
await this.refreshMemberSpawnStatusesFromLeadInbox(run);
|
||||
await this.auditMemberSpawnStatuses(run);
|
||||
await this.persistPartialLaunchState(run);
|
||||
await this.persistLaunchStateSnapshot(run, 'finished');
|
||||
const failedSpawnMembers = this.getFailedSpawnMembers(run);
|
||||
const launchSummary = this.getMemberLaunchSummary(run);
|
||||
const hasSpawnFailures = failedSpawnMembers.length > 0;
|
||||
|
|
@ -8570,7 +8742,7 @@ export class TeamProvisioningService {
|
|||
// Audit: flag any expected member not registered in config.json after provisioning.
|
||||
await this.refreshMemberSpawnStatusesFromLeadInbox(run);
|
||||
await this.auditMemberSpawnStatuses(run);
|
||||
await this.clearPartialLaunchState(run.teamName);
|
||||
await this.persistLaunchStateSnapshot(run, 'finished');
|
||||
const failedSpawnMembers = this.getFailedSpawnMembers(run);
|
||||
const launchSummary = this.getMemberLaunchSummary(run);
|
||||
const hasSpawnFailures = failedSpawnMembers.length > 0;
|
||||
|
|
@ -8941,7 +9113,7 @@ export class TeamProvisioningService {
|
|||
*/
|
||||
private cleanupRun(run: ProvisioningRun): void {
|
||||
if (run.isLaunch && !run.provisioningComplete) {
|
||||
void this.persistPartialLaunchState(run);
|
||||
void this.persistLaunchStateSnapshot(run, 'finished');
|
||||
}
|
||||
this.resetRuntimeToolActivity(run);
|
||||
this.setLeadActivity(run, 'offline');
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import * as path from 'node:path';
|
|||
import { parentPort } from 'node:worker_threads';
|
||||
|
||||
import { isLeadMember } from '@shared/utils/leadDetection';
|
||||
import { normalizePersistedLaunchSnapshot } from '@main/services/team/TeamLaunchStateEvaluator';
|
||||
|
||||
interface ListTeamsPayload {
|
||||
teamsDir: string;
|
||||
|
|
@ -319,48 +320,49 @@ function dropCliProvisionerMembers(
|
|||
}
|
||||
}
|
||||
|
||||
async function readPartialLaunchState(
|
||||
async function readLaunchState(
|
||||
teamsDir: string,
|
||||
teamName: string
|
||||
): Promise<{
|
||||
partialLaunchFailure: true;
|
||||
expectedMemberCount: number;
|
||||
confirmedMemberCount: number;
|
||||
missingMembers: string[];
|
||||
partialLaunchFailure?: true;
|
||||
expectedMemberCount?: number;
|
||||
confirmedMemberCount?: number;
|
||||
missingMembers?: string[];
|
||||
teamLaunchState?: string;
|
||||
launchUpdatedAt?: string;
|
||||
confirmedCount?: number;
|
||||
pendingCount?: number;
|
||||
failedCount?: number;
|
||||
runtimeAlivePendingCount?: number;
|
||||
} | null> {
|
||||
const launchStatePath = path.join(teamsDir, teamName, TEAM_LAUNCH_STATE_FILE);
|
||||
try {
|
||||
const stat = await fs.promises.stat(launchStatePath);
|
||||
if (!stat.isFile() || stat.size > MAX_LAUNCH_STATE_BYTES) return null;
|
||||
const raw = await fs.promises.readFile(launchStatePath, 'utf8');
|
||||
const parsed = JSON.parse(raw) as {
|
||||
state?: unknown;
|
||||
expectedMembers?: unknown;
|
||||
confirmedMembers?: unknown;
|
||||
missingMembers?: unknown;
|
||||
};
|
||||
if (parsed.state !== 'partial_launch_failure') return null;
|
||||
const expectedMembers = Array.isArray(parsed.expectedMembers)
|
||||
? parsed.expectedMembers.filter(
|
||||
(name): name is string => typeof name === 'string' && name.trim().length > 0
|
||||
)
|
||||
: [];
|
||||
const confirmedMembers = Array.isArray(parsed.confirmedMembers)
|
||||
? parsed.confirmedMembers.filter(
|
||||
(name): name is string => typeof name === 'string' && name.trim().length > 0
|
||||
)
|
||||
: [];
|
||||
const missingMembers = Array.isArray(parsed.missingMembers)
|
||||
? parsed.missingMembers.filter(
|
||||
(name): name is string => typeof name === 'string' && name.trim().length > 0
|
||||
)
|
||||
: [];
|
||||
if (expectedMembers.length === 0 || missingMembers.length === 0) return null;
|
||||
const snapshot = normalizePersistedLaunchSnapshot(teamName, JSON.parse(raw));
|
||||
if (!snapshot) return null;
|
||||
const missingMembers = snapshot.expectedMembers.filter((name) => {
|
||||
const member = snapshot.members[name];
|
||||
return member?.launchState === 'failed_to_start';
|
||||
});
|
||||
return {
|
||||
partialLaunchFailure: true,
|
||||
expectedMemberCount: expectedMembers.length,
|
||||
confirmedMemberCount: confirmedMembers.length,
|
||||
missingMembers,
|
||||
...(snapshot.teamLaunchState === 'partial_failure'
|
||||
? { partialLaunchFailure: true as const }
|
||||
: {}),
|
||||
...(snapshot.expectedMembers.length > 0
|
||||
? { expectedMemberCount: snapshot.expectedMembers.length }
|
||||
: {}),
|
||||
...(snapshot.summary.confirmedCount > 0
|
||||
? { confirmedMemberCount: snapshot.summary.confirmedCount }
|
||||
: {}),
|
||||
...(missingMembers.length > 0 ? { missingMembers } : {}),
|
||||
teamLaunchState: snapshot.teamLaunchState,
|
||||
launchUpdatedAt: snapshot.updatedAt,
|
||||
confirmedCount: snapshot.summary.confirmedCount,
|
||||
pendingCount: snapshot.summary.pendingCount,
|
||||
failedCount: snapshot.summary.failedCount,
|
||||
runtimeAlivePendingCount: snapshot.summary.runtimeAlivePendingCount,
|
||||
};
|
||||
} catch {
|
||||
return null;
|
||||
|
|
@ -591,8 +593,8 @@ async function listTeams(
|
|||
dropCliProvisionerMembers(memberMap);
|
||||
|
||||
const members = Array.from(memberMap.values());
|
||||
const partialLaunchState =
|
||||
(await readPartialLaunchState(payload.teamsDir, teamName)) ??
|
||||
const launchStateSummary =
|
||||
(await readLaunchState(payload.teamsDir, teamName)) ??
|
||||
(() => {
|
||||
if (
|
||||
!leadSessionId ||
|
||||
|
|
@ -628,7 +630,7 @@ async function listTeams(
|
|||
...(projectPathHistory ? { projectPathHistory } : {}),
|
||||
...(sessionHistory ? { sessionHistory } : {}),
|
||||
...(deletedAt ? { deletedAt } : {}),
|
||||
...(partialLaunchState ?? {}),
|
||||
...(launchStateSummary ?? {}),
|
||||
};
|
||||
|
||||
const ms = nowMs() - t0;
|
||||
|
|
|
|||
|
|
@ -1557,11 +1557,16 @@ export const TeamDetailView = ({
|
|||
>
|
||||
<span className="flex items-center gap-1.5 text-xs">
|
||||
<AlertTriangle size={14} className="shrink-0" />
|
||||
{currentTeamSummary?.partialLaunchFailure
|
||||
? currentTeamSummary.missingMembers?.length
|
||||
? `Last launch failed partway — ${currentTeamSummary.missingMembers.length}/${currentTeamSummary.expectedMemberCount ?? currentTeamSummary.missingMembers.length} teammates did not join`
|
||||
: 'Last launch failed partway'
|
||||
: 'Team is offline'}
|
||||
{currentTeamSummary?.teamLaunchState === 'partial_pending'
|
||||
? currentTeamSummary.runtimeAlivePendingCount &&
|
||||
currentTeamSummary.runtimeAlivePendingCount > 0
|
||||
? `Last launch is still reconciling — ${currentTeamSummary.confirmedCount ?? 0}/${currentTeamSummary.expectedMemberCount ?? currentTeamSummary.memberCount} teammates confirmed alive, ${currentTeamSummary.runtimeAlivePendingCount} runtime${currentTeamSummary.runtimeAlivePendingCount === 1 ? '' : 's'} pending bootstrap`
|
||||
: 'Last launch is still reconciling'
|
||||
: currentTeamSummary?.partialLaunchFailure
|
||||
? currentTeamSummary.missingMembers?.length
|
||||
? `Last launch failed partway — ${currentTeamSummary.missingMembers.length}/${currentTeamSummary.expectedMemberCount ?? currentTeamSummary.missingMembers.length} teammates did not join`
|
||||
: 'Last launch failed partway'
|
||||
: 'Team is offline'}
|
||||
</span>
|
||||
<Button
|
||||
variant="ghost"
|
||||
|
|
|
|||
|
|
@ -64,7 +64,13 @@ function generateUniqueName(sourceName: string, existingNames: string[]): string
|
|||
}
|
||||
}
|
||||
|
||||
type TeamStatus = 'active' | 'idle' | 'provisioning' | 'offline' | 'partial_failure';
|
||||
type TeamStatus =
|
||||
| 'active'
|
||||
| 'idle'
|
||||
| 'provisioning'
|
||||
| 'offline'
|
||||
| 'partial_failure'
|
||||
| 'partial_pending';
|
||||
|
||||
function getRecentProjects(team: TeamSummary): string[] {
|
||||
const history = team.projectPathHistory;
|
||||
|
|
@ -174,7 +180,10 @@ function resolveTeamStatus(
|
|||
) {
|
||||
return 'provisioning';
|
||||
}
|
||||
if (team.partialLaunchFailure) {
|
||||
if (team.teamLaunchState === 'partial_pending') {
|
||||
return 'partial_pending';
|
||||
}
|
||||
if (team.partialLaunchFailure || team.teamLaunchState === 'partial_failure') {
|
||||
return 'partial_failure';
|
||||
}
|
||||
return 'offline';
|
||||
|
|
@ -217,6 +226,13 @@ const StatusBadge = ({ status }: { status: TeamStatus }): React.JSX.Element => {
|
|||
Launch failed partway
|
||||
</span>
|
||||
);
|
||||
case 'partial_pending':
|
||||
return (
|
||||
<span className="inline-flex items-center gap-1 rounded-full bg-amber-500/15 px-2 py-0.5 text-[10px] font-medium text-amber-300">
|
||||
<span className="size-1.5 rounded-full bg-amber-300" />
|
||||
Bootstrap pending
|
||||
</span>
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
|
|
@ -381,7 +397,8 @@ export const TeamListView = (): React.JSX.Element => {
|
|||
getCurrentProvisioningProgressForTeam(provisioningState, t.teamName),
|
||||
leadActivityByTeam
|
||||
);
|
||||
const isRunning = status !== 'offline' && status !== 'partial_failure';
|
||||
const isRunning =
|
||||
status !== 'offline' && status !== 'partial_failure' && status !== 'partial_pending';
|
||||
if (filter.selectedStatuses.has('running') && isRunning) return true;
|
||||
if (filter.selectedStatuses.has('offline') && !isRunning) return true;
|
||||
return false;
|
||||
|
|
@ -850,7 +867,9 @@ export const TeamListView = (): React.JSX.Element => {
|
|||
})()}
|
||||
</div>
|
||||
<div className="flex shrink-0 gap-1">
|
||||
{(status === 'offline' || status === 'partial_failure') &&
|
||||
{(status === 'offline' ||
|
||||
status === 'partial_failure' ||
|
||||
status === 'partial_pending') &&
|
||||
team.projectPath && (
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
|
|
@ -924,7 +943,13 @@ export const TeamListView = (): React.JSX.Element => {
|
|||
{team.description || 'No description'}
|
||||
</p>
|
||||
</div>
|
||||
{team.partialLaunchFailure ? (
|
||||
{team.teamLaunchState === 'partial_pending' ? (
|
||||
<p className="mt-2 text-[11px] text-amber-300">
|
||||
{team.runtimeAlivePendingCount && team.runtimeAlivePendingCount > 0
|
||||
? `Last launch is still reconciling — ${team.confirmedCount ?? 0}/${team.expectedMemberCount ?? team.memberCount} teammates confirmed alive, ${team.runtimeAlivePendingCount} runtime${team.runtimeAlivePendingCount === 1 ? '' : 's'} pending bootstrap.`
|
||||
: 'Last launch is still reconciling.'}
|
||||
</p>
|
||||
) : team.partialLaunchFailure || team.teamLaunchState === 'partial_failure' ? (
|
||||
<p className="mt-2 text-[11px] text-amber-400">
|
||||
{team.missingMembers?.length
|
||||
? `Last launch stopped before ${team.missingMembers.length}/${team.expectedMemberCount ?? team.missingMembers.length} teammate${team.missingMembers.length === 1 ? '' : 's'} joined.`
|
||||
|
|
|
|||
|
|
@ -17,14 +17,16 @@ interface TeamProvisioningBannerProps {
|
|||
export const TeamProvisioningBanner = ({
|
||||
teamName,
|
||||
}: TeamProvisioningBannerProps): React.JSX.Element | null => {
|
||||
const { progress, cancelProvisioning, teamMembers, memberSpawnStatuses } = useStore(
|
||||
useShallow((s) => ({
|
||||
progress: getCurrentProvisioningProgressForTeam(s, teamName),
|
||||
cancelProvisioning: s.cancelProvisioning,
|
||||
teamMembers: s.selectedTeamData?.members,
|
||||
memberSpawnStatuses: s.memberSpawnStatusesByTeam[teamName],
|
||||
}))
|
||||
);
|
||||
const { progress, cancelProvisioning, teamMembers, memberSpawnStatuses, memberSpawnSnapshot } =
|
||||
useStore(
|
||||
useShallow((s) => ({
|
||||
progress: getCurrentProvisioningProgressForTeam(s, teamName),
|
||||
cancelProvisioning: s.cancelProvisioning,
|
||||
teamMembers: s.selectedTeamData?.members,
|
||||
memberSpawnStatuses: s.memberSpawnStatusesByTeam[teamName],
|
||||
memberSpawnSnapshot: s.memberSpawnSnapshotsByTeam[teamName],
|
||||
}))
|
||||
);
|
||||
const [dismissed, setDismissed] = useState(false);
|
||||
const lastActiveStepRef = useRef(-1);
|
||||
const bannerInstanceKey = useMemo(() => {
|
||||
|
|
@ -105,38 +107,49 @@ export const TeamProvisioningBanner = ({
|
|||
}
|
||||
|
||||
const teammates = (teamMembers ?? []).filter((member) => !isLeadMember(member));
|
||||
const expectedTeammateCount = memberSpawnSnapshot?.expectedMembers?.length;
|
||||
const fallbackTeammateCount = expectedTeammateCount ?? teammates.length;
|
||||
const snapshotSummary = memberSpawnSnapshot?.summary;
|
||||
const failedSpawnEntries = Object.entries(memberSpawnStatuses ?? {}).filter(
|
||||
([, entry]) => entry.launchState === 'failed_to_start'
|
||||
);
|
||||
const failedSpawnCount = failedSpawnEntries.length;
|
||||
const heartbeatConfirmedCount = teammates.filter((member) => {
|
||||
const entry = memberSpawnStatuses?.[member.name];
|
||||
return entry?.launchState === 'confirmed_alive';
|
||||
}).length;
|
||||
const processOnlyAliveCount = teammates.filter((member) => {
|
||||
const entry = memberSpawnStatuses?.[member.name];
|
||||
return entry?.launchState === 'runtime_pending_bootstrap' && entry.runtimeAlive === true;
|
||||
}).length;
|
||||
const pendingSpawnCount = teammates.filter((member) => {
|
||||
const entry = memberSpawnStatuses?.[member.name];
|
||||
return (
|
||||
entry?.launchState === 'starting' ||
|
||||
(entry?.launchState === 'runtime_pending_bootstrap' && entry.runtimeAlive !== true)
|
||||
);
|
||||
}).length;
|
||||
const failedSpawnCount = snapshotSummary?.failedCount ?? failedSpawnEntries.length;
|
||||
const heartbeatConfirmedCount =
|
||||
snapshotSummary?.confirmedCount ??
|
||||
teammates.filter((member) => {
|
||||
const entry = memberSpawnStatuses?.[member.name];
|
||||
return entry?.launchState === 'confirmed_alive';
|
||||
}).length;
|
||||
const processOnlyAliveCount =
|
||||
snapshotSummary?.runtimeAlivePendingCount ??
|
||||
teammates.filter((member) => {
|
||||
const entry = memberSpawnStatuses?.[member.name];
|
||||
return entry?.launchState === 'runtime_pending_bootstrap' && entry.runtimeAlive === true;
|
||||
}).length;
|
||||
const pendingSpawnCount =
|
||||
snapshotSummary?.pendingCount ??
|
||||
teammates.filter((member) => {
|
||||
const entry = memberSpawnStatuses?.[member.name];
|
||||
return (
|
||||
entry?.launchState === 'starting' ||
|
||||
(entry?.launchState === 'runtime_pending_bootstrap' && entry.runtimeAlive !== true)
|
||||
);
|
||||
}).length;
|
||||
const allTeammatesConfirmedAlive =
|
||||
teammates.length > 0 && failedSpawnCount === 0 && heartbeatConfirmedCount === teammates.length;
|
||||
fallbackTeammateCount > 0 &&
|
||||
failedSpawnCount === 0 &&
|
||||
heartbeatConfirmedCount === fallbackTeammateCount;
|
||||
|
||||
if (isReady) {
|
||||
const readyMessage =
|
||||
failedSpawnCount > 0
|
||||
? `Launch finished with errors — ${failedSpawnCount}/${Math.max(teammates.length, failedSpawnCount)} teammates failed to start`
|
||||
: teammates.length === 0
|
||||
? `Launch finished with errors — ${failedSpawnCount}/${Math.max(fallbackTeammateCount, failedSpawnCount)} teammates failed to start`
|
||||
: fallbackTeammateCount === 0
|
||||
? 'Team launched — lead online'
|
||||
: allTeammatesConfirmedAlive
|
||||
? `Team launched — all ${teammates.length} teammates confirmed alive`
|
||||
? `Team launched — all ${fallbackTeammateCount} teammates confirmed alive`
|
||||
: processOnlyAliveCount > 0 || pendingSpawnCount > 0
|
||||
? `Team launched — ${heartbeatConfirmedCount}/${teammates.length} teammates confirmed alive${processOnlyAliveCount > 0 ? `, ${processOnlyAliveCount} runtime${processOnlyAliveCount === 1 ? '' : 's'} alive but bootstrap still pending` : ''}${pendingSpawnCount > 0 ? `${processOnlyAliveCount > 0 ? ', ' : ', '}${pendingSpawnCount} still starting` : ''}`
|
||||
? `Team launched — ${heartbeatConfirmedCount}/${fallbackTeammateCount} teammates confirmed alive${processOnlyAliveCount > 0 ? `, ${processOnlyAliveCount} runtime${processOnlyAliveCount === 1 ? '' : 's'} alive but bootstrap still pending` : ''}${pendingSpawnCount > 0 ? `${processOnlyAliveCount > 0 ? ', ' : ', '}${pendingSpawnCount} still starting` : ''}`
|
||||
: 'Team launched — teammate liveness is still being confirmed';
|
||||
|
||||
return (
|
||||
|
|
|
|||
|
|
@ -168,6 +168,7 @@ export function initializeNotificationListeners(): () => void {
|
|||
const pendingProjectRefreshTimers = new Map<string, ReturnType<typeof setTimeout>>();
|
||||
let teamRefreshTimers = new Map<string, ReturnType<typeof setTimeout>>();
|
||||
let teamPresenceRefreshTimers = new Map<string, ReturnType<typeof setTimeout>>();
|
||||
let memberSpawnRefreshTimers = new Map<string, ReturnType<typeof setTimeout>>();
|
||||
let toolActivityTimers = new Map<string, ReturnType<typeof setTimeout>>();
|
||||
let inProgressChangePresencePollInFlight = false;
|
||||
const inProgressChangePresenceCursorByTeam = new Map<string, number>();
|
||||
|
|
@ -178,6 +179,7 @@ export function initializeNotificationListeners(): () => void {
|
|||
const PROJECT_REFRESH_DEBOUNCE_MS = 300;
|
||||
const TEAM_REFRESH_THROTTLE_MS = 800;
|
||||
const TEAM_PRESENCE_REFRESH_THROTTLE_MS = 400;
|
||||
const TEAM_MEMBER_SPAWN_REFRESH_THROTTLE_MS = 500;
|
||||
const TEAM_LIST_REFRESH_THROTTLE_MS = 2000;
|
||||
const GLOBAL_TASKS_REFRESH_THROTTLE_MS = 500;
|
||||
const buildToolActivityTimerKey = (
|
||||
|
|
@ -919,6 +921,20 @@ export function initializeNotificationListeners(): () => void {
|
|||
return;
|
||||
}
|
||||
|
||||
if (event.type === 'inbox' || event.type === 'config' || event.type === 'process') {
|
||||
if (!event?.teamName || !isTeamVisibleInAnyPane(event.teamName)) {
|
||||
return;
|
||||
}
|
||||
if (memberSpawnRefreshTimers.has(event.teamName)) {
|
||||
return;
|
||||
}
|
||||
const timer = setTimeout(() => {
|
||||
memberSpawnRefreshTimers.delete(event.teamName);
|
||||
void useStore.getState().fetchMemberSpawnStatuses(event.teamName);
|
||||
}, TEAM_MEMBER_SPAWN_REFRESH_THROTTLE_MS);
|
||||
memberSpawnRefreshTimers.set(event.teamName, timer);
|
||||
}
|
||||
|
||||
// Live lead-message events: only refresh the visible team detail, not team/task lists.
|
||||
// This keeps the refresh lightweight and prevents one noisy team from starving another.
|
||||
if (event.type === 'lead-message') {
|
||||
|
|
@ -998,6 +1014,8 @@ export function initializeNotificationListeners(): () => void {
|
|||
teamRefreshTimers = new Map();
|
||||
for (const t of teamPresenceRefreshTimers.values()) clearTimeout(t);
|
||||
teamPresenceRefreshTimers = new Map();
|
||||
for (const t of memberSpawnRefreshTimers.values()) clearTimeout(t);
|
||||
memberSpawnRefreshTimers = new Map();
|
||||
for (const t of toolActivityTimers.values()) clearTimeout(t);
|
||||
toolActivityTimers = new Map();
|
||||
if (teamListRefreshTimer) {
|
||||
|
|
|
|||
|
|
@ -139,6 +139,7 @@ import type {
|
|||
KanbanColumnId,
|
||||
LeadActivityState,
|
||||
LeadContextUsage,
|
||||
MemberSpawnStatusesSnapshot,
|
||||
MemberSpawnStatusEntry,
|
||||
SendMessageRequest,
|
||||
SendMessageResult,
|
||||
|
|
@ -625,6 +626,7 @@ export interface TeamSlice {
|
|||
toolHistoryByTeam: Record<string, Record<string, ActiveToolCall[]>>;
|
||||
/** Per-team per-member spawn statuses during team provisioning/launch. */
|
||||
memberSpawnStatusesByTeam: Record<string, Record<string, MemberSpawnStatusEntry>>;
|
||||
memberSpawnSnapshotsByTeam: Record<string, MemberSpawnStatusesSnapshot>;
|
||||
fetchMemberSpawnStatuses: (teamName: string) => Promise<void>;
|
||||
provisioningErrorByTeam: Record<string, string | null>;
|
||||
clearProvisioningError: (teamName?: string) => void;
|
||||
|
|
@ -910,6 +912,7 @@ export const createTeamSlice: StateCreator<AppState, [], [], TeamSlice> = (set,
|
|||
finishedVisibleByTeam: {},
|
||||
toolHistoryByTeam: {},
|
||||
memberSpawnStatusesByTeam: {},
|
||||
memberSpawnSnapshotsByTeam: {},
|
||||
provisioningErrorByTeam: {},
|
||||
clearProvisioningError: (teamName?: string) =>
|
||||
set((state) => {
|
||||
|
|
@ -971,6 +974,10 @@ export const createTeamSlice: StateCreator<AppState, [], [], TeamSlice> = (set,
|
|||
...prev.memberSpawnStatusesByTeam,
|
||||
[teamName]: snapshot.statuses,
|
||||
},
|
||||
memberSpawnSnapshotsByTeam: {
|
||||
...prev.memberSpawnSnapshotsByTeam,
|
||||
[teamName]: snapshot,
|
||||
},
|
||||
};
|
||||
});
|
||||
} catch {
|
||||
|
|
@ -1861,6 +1868,8 @@ export const createTeamSlice: StateCreator<AppState, [], [], TeamSlice> = (set,
|
|||
delete nextErrors[request.teamName];
|
||||
const nextSpawnStatuses = { ...state.memberSpawnStatusesByTeam };
|
||||
delete nextSpawnStatuses[request.teamName];
|
||||
const nextSpawnSnapshots = { ...state.memberSpawnSnapshotsByTeam };
|
||||
delete nextSpawnSnapshots[request.teamName];
|
||||
const nextActiveTools = { ...state.activeToolsByTeam };
|
||||
delete nextActiveTools[request.teamName];
|
||||
const nextFinishedVisible = { ...state.finishedVisibleByTeam };
|
||||
|
|
@ -1893,6 +1902,7 @@ export const createTeamSlice: StateCreator<AppState, [], [], TeamSlice> = (set,
|
|||
provisioningRuns: cleaned,
|
||||
provisioningErrorByTeam: nextErrors,
|
||||
memberSpawnStatusesByTeam: nextSpawnStatuses,
|
||||
memberSpawnSnapshotsByTeam: nextSpawnSnapshots,
|
||||
activeToolsByTeam: nextActiveTools,
|
||||
finishedVisibleByTeam: nextFinishedVisible,
|
||||
toolHistoryByTeam: nextToolHistory,
|
||||
|
|
@ -2056,6 +2066,8 @@ export const createTeamSlice: StateCreator<AppState, [], [], TeamSlice> = (set,
|
|||
delete nextErrors[request.teamName];
|
||||
const nextSpawnStatuses = { ...state.memberSpawnStatusesByTeam };
|
||||
delete nextSpawnStatuses[request.teamName];
|
||||
const nextSpawnSnapshots = { ...state.memberSpawnSnapshotsByTeam };
|
||||
delete nextSpawnSnapshots[request.teamName];
|
||||
const nextActiveTools = { ...state.activeToolsByTeam };
|
||||
delete nextActiveTools[request.teamName];
|
||||
const nextFinishedVisible = { ...state.finishedVisibleByTeam };
|
||||
|
|
@ -2088,6 +2100,7 @@ export const createTeamSlice: StateCreator<AppState, [], [], TeamSlice> = (set,
|
|||
provisioningRuns: cleaned,
|
||||
provisioningErrorByTeam: nextErrors,
|
||||
memberSpawnStatusesByTeam: nextSpawnStatuses,
|
||||
memberSpawnSnapshotsByTeam: nextSpawnSnapshots,
|
||||
activeToolsByTeam: nextActiveTools,
|
||||
finishedVisibleByTeam: nextFinishedVisible,
|
||||
toolHistoryByTeam: nextToolHistory,
|
||||
|
|
@ -2248,8 +2261,10 @@ export const createTeamSlice: StateCreator<AppState, [], [], TeamSlice> = (set,
|
|||
: state.ignoredRuntimeRunIds;
|
||||
|
||||
const nextSpawnStatuses = { ...state.memberSpawnStatusesByTeam };
|
||||
const nextSpawnSnapshots = { ...state.memberSpawnSnapshotsByTeam };
|
||||
if (isCanonicalRun) {
|
||||
delete nextSpawnStatuses[existing.teamName];
|
||||
delete nextSpawnSnapshots[existing.teamName];
|
||||
}
|
||||
const nextActiveTools = { ...state.activeToolsByTeam };
|
||||
const nextFinishedVisible = { ...state.finishedVisibleByTeam };
|
||||
|
|
@ -2265,6 +2280,7 @@ export const createTeamSlice: StateCreator<AppState, [], [], TeamSlice> = (set,
|
|||
currentProvisioningRunIdByTeam: nextCurrentRunIdByTeam,
|
||||
currentRuntimeRunIdByTeam: nextRuntimeRunIdByTeam,
|
||||
memberSpawnStatusesByTeam: nextSpawnStatuses,
|
||||
memberSpawnSnapshotsByTeam: nextSpawnSnapshots,
|
||||
activeToolsByTeam: nextActiveTools,
|
||||
finishedVisibleByTeam: nextFinishedVisible,
|
||||
toolHistoryByTeam: nextToolHistory,
|
||||
|
|
@ -2387,13 +2403,20 @@ export const createTeamSlice: StateCreator<AppState, [], [], TeamSlice> = (set,
|
|||
if (isCanonicalRun && TERMINAL_PROVISIONING_STATES.has(progress.state)) {
|
||||
set((prev) => {
|
||||
const next = { ...prev.memberSpawnStatusesByTeam };
|
||||
const nextSnapshots = { ...prev.memberSpawnSnapshotsByTeam };
|
||||
const currentStatuses = next[progress.teamName];
|
||||
if (!currentStatuses) {
|
||||
return { memberSpawnStatusesByTeam: next };
|
||||
return {
|
||||
memberSpawnStatusesByTeam: next,
|
||||
memberSpawnSnapshotsByTeam: nextSnapshots,
|
||||
};
|
||||
}
|
||||
if (progress.state === 'ready') {
|
||||
next[progress.teamName] = currentStatuses;
|
||||
return { memberSpawnStatusesByTeam: next };
|
||||
return {
|
||||
memberSpawnStatusesByTeam: next,
|
||||
memberSpawnSnapshotsByTeam: nextSnapshots,
|
||||
};
|
||||
}
|
||||
const retainedStatuses = Object.fromEntries(
|
||||
Object.entries(currentStatuses).filter(([, entry]) => entry.status === 'error')
|
||||
|
|
@ -2402,8 +2425,12 @@ export const createTeamSlice: StateCreator<AppState, [], [], TeamSlice> = (set,
|
|||
next[progress.teamName] = retainedStatuses;
|
||||
} else {
|
||||
delete next[progress.teamName];
|
||||
delete nextSnapshots[progress.teamName];
|
||||
}
|
||||
return { memberSpawnStatusesByTeam: next };
|
||||
return {
|
||||
memberSpawnStatusesByTeam: next,
|
||||
memberSpawnSnapshotsByTeam: nextSnapshots,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -66,6 +66,15 @@ export interface TeamSummary {
|
|||
confirmedMemberCount?: number;
|
||||
/** Missing teammate names from the last partial launch marker. */
|
||||
missingMembers?: string[];
|
||||
/** Durable aggregate launch state derived from persisted launch-state evidence. */
|
||||
teamLaunchState?: TeamLaunchAggregateState;
|
||||
/** ISO timestamp of the last durable launch-state evaluation. */
|
||||
launchUpdatedAt?: string;
|
||||
/** Durable aggregate teammate counts from launch-state evidence. */
|
||||
confirmedCount?: number;
|
||||
pendingCount?: number;
|
||||
failedCount?: number;
|
||||
runtimeAlivePendingCount?: number;
|
||||
}
|
||||
|
||||
export type TeamTaskStatus = 'pending' | 'in_progress' | 'completed' | 'deleted';
|
||||
|
|
@ -446,6 +455,8 @@ export type MemberLaunchState =
|
|||
| 'runtime_pending_bootstrap'
|
||||
| 'confirmed_alive'
|
||||
| 'failed_to_start';
|
||||
export type TeamLaunchAggregateState = 'clean_success' | 'partial_pending' | 'partial_failure';
|
||||
export type PersistedTeamLaunchPhase = 'active' | 'finished' | 'reconciled';
|
||||
|
||||
export type KanbanColumnId = 'todo' | 'in_progress' | 'done' | 'review' | 'approved';
|
||||
|
||||
|
|
@ -575,9 +586,60 @@ export interface LeadContextUsageSnapshot {
|
|||
runId: string | null;
|
||||
}
|
||||
|
||||
export interface PersistedTeamLaunchMemberSources {
|
||||
inboxHeartbeat?: boolean;
|
||||
nativeHeartbeat?: boolean;
|
||||
processAlive?: boolean;
|
||||
configRegistered?: boolean;
|
||||
configDrift?: boolean;
|
||||
hardFailureSignal?: boolean;
|
||||
duplicateRespawnBlocked?: boolean;
|
||||
}
|
||||
|
||||
export interface PersistedTeamLaunchMemberState {
|
||||
name: string;
|
||||
launchState: MemberLaunchState;
|
||||
agentToolAccepted: boolean;
|
||||
runtimeAlive: boolean;
|
||||
bootstrapConfirmed: boolean;
|
||||
hardFailure: boolean;
|
||||
hardFailureReason?: string;
|
||||
firstSpawnAcceptedAt?: string;
|
||||
lastHeartbeatAt?: string;
|
||||
lastRuntimeAliveAt?: string;
|
||||
lastEvaluatedAt: string;
|
||||
sources?: PersistedTeamLaunchMemberSources;
|
||||
diagnostics?: string[];
|
||||
}
|
||||
|
||||
export interface PersistedTeamLaunchSummary {
|
||||
confirmedCount: number;
|
||||
pendingCount: number;
|
||||
failedCount: number;
|
||||
runtimeAlivePendingCount: number;
|
||||
}
|
||||
|
||||
export interface PersistedTeamLaunchSnapshot {
|
||||
version: 2;
|
||||
teamName: string;
|
||||
updatedAt: string;
|
||||
leadSessionId?: string;
|
||||
launchPhase: PersistedTeamLaunchPhase;
|
||||
expectedMembers: string[];
|
||||
members: Record<string, PersistedTeamLaunchMemberState>;
|
||||
summary: PersistedTeamLaunchSummary;
|
||||
teamLaunchState: TeamLaunchAggregateState;
|
||||
}
|
||||
|
||||
export interface MemberSpawnStatusesSnapshot {
|
||||
statuses: Record<string, MemberSpawnStatusEntry>;
|
||||
runId: string | null;
|
||||
teamLaunchState?: TeamLaunchAggregateState;
|
||||
launchPhase?: PersistedTeamLaunchPhase;
|
||||
expectedMembers?: string[];
|
||||
updatedAt?: string;
|
||||
summary?: PersistedTeamLaunchSummary;
|
||||
source?: 'live' | 'persisted' | 'merged';
|
||||
}
|
||||
|
||||
export type MemberSpawnLivenessSource = 'heartbeat' | 'process';
|
||||
|
|
|
|||
|
|
@ -5,6 +5,15 @@ export interface ToolSummaryData {
|
|||
byName: Record<string, number>;
|
||||
}
|
||||
|
||||
export type AgentToolDuplicateSkipReason = 'already_running' | 'bootstrap_pending';
|
||||
|
||||
export interface ParsedAgentToolResultStatus {
|
||||
status: 'duplicate_skipped';
|
||||
reason: AgentToolDuplicateSkipReason;
|
||||
name?: string;
|
||||
teamName?: string;
|
||||
}
|
||||
|
||||
export function buildToolSummary(content: Record<string, unknown>[]): string | undefined {
|
||||
const counts = new Map<string, number>();
|
||||
for (const block of content) {
|
||||
|
|
@ -228,3 +237,40 @@ export function extractToolResultPreview(content: unknown, max = 80): string | u
|
|||
if (!joined) return undefined;
|
||||
return truncateStr(joined, max);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse machine-readable Agent tool_result status lines from the raw tool_result content.
|
||||
* Returns null for any non-Agent or non-duplicate result.
|
||||
*/
|
||||
export function parseAgentToolResultStatus(content: unknown): ParsedAgentToolResultStatus | null {
|
||||
const joined = flattenToolResultContent(content).join('\n').trim();
|
||||
if (!joined) return null;
|
||||
|
||||
const fields = new Map<string, string>();
|
||||
for (const rawLine of joined.split(/\r?\n/)) {
|
||||
const line = rawLine.trim();
|
||||
if (!line) continue;
|
||||
const separatorIndex = line.indexOf(':');
|
||||
if (separatorIndex <= 0) continue;
|
||||
const key = line.slice(0, separatorIndex).trim().toLowerCase();
|
||||
const value = line.slice(separatorIndex + 1).trim();
|
||||
if (!value) continue;
|
||||
fields.set(key, value);
|
||||
}
|
||||
|
||||
if (fields.get('status') !== 'duplicate_skipped') {
|
||||
return null;
|
||||
}
|
||||
|
||||
const reason = fields.get('reason');
|
||||
if (reason !== 'already_running' && reason !== 'bootstrap_pending') {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
status: 'duplicate_skipped',
|
||||
reason,
|
||||
name: fields.get('name'),
|
||||
teamName: fields.get('team_name'),
|
||||
};
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue