Persist launch-state and duplicate no-op handling
This commit is contained in:
parent
2959a3d074
commit
30fb2501d3
12 changed files with 1102 additions and 252 deletions
|
|
@ -12,6 +12,7 @@ import * as path from 'path';
|
||||||
import { getTeamFsWorkerClient } from './TeamFsWorkerClient';
|
import { getTeamFsWorkerClient } from './TeamFsWorkerClient';
|
||||||
import { TeamMembersMetaStore } from './TeamMembersMetaStore';
|
import { TeamMembersMetaStore } from './TeamMembersMetaStore';
|
||||||
import { TeamMetaStore } from './TeamMetaStore';
|
import { TeamMetaStore } from './TeamMetaStore';
|
||||||
|
import { normalizePersistedLaunchSnapshot } from './TeamLaunchStateEvaluator';
|
||||||
|
|
||||||
import type { TeamConfig, TeamMember, TeamSummary, TeamSummaryMember } from '@shared/types';
|
import type { TeamConfig, TeamMember, TeamSummary, TeamSummaryMember } from '@shared/types';
|
||||||
|
|
||||||
|
|
@ -27,16 +28,20 @@ const MAX_PROJECT_PATH_HISTORY_IN_SUMMARY = 200;
|
||||||
const MAX_LAUNCH_STATE_BYTES = 32 * 1024;
|
const MAX_LAUNCH_STATE_BYTES = 32 * 1024;
|
||||||
const TEAM_LAUNCH_STATE_FILE = 'launch-state.json';
|
const TEAM_LAUNCH_STATE_FILE = 'launch-state.json';
|
||||||
|
|
||||||
interface PartialLaunchStateSummary {
|
interface LaunchStateSummary {
|
||||||
partialLaunchFailure: true;
|
partialLaunchFailure?: true;
|
||||||
expectedMemberCount: number;
|
expectedMemberCount?: number;
|
||||||
confirmedMemberCount: number;
|
confirmedMemberCount?: number;
|
||||||
missingMembers: string[];
|
missingMembers?: string[];
|
||||||
|
teamLaunchState?: TeamSummary['teamLaunchState'];
|
||||||
|
launchUpdatedAt?: string;
|
||||||
|
confirmedCount?: number;
|
||||||
|
pendingCount?: number;
|
||||||
|
failedCount?: number;
|
||||||
|
runtimeAlivePendingCount?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function readPartialLaunchStateSummary(
|
async function readLaunchStateSummary(teamDir: string): Promise<LaunchStateSummary | null> {
|
||||||
teamDir: string
|
|
||||||
): Promise<PartialLaunchStateSummary | null> {
|
|
||||||
const launchStatePath = path.join(teamDir, TEAM_LAUNCH_STATE_FILE);
|
const launchStatePath = path.join(teamDir, TEAM_LAUNCH_STATE_FILE);
|
||||||
try {
|
try {
|
||||||
const stat = await fs.promises.stat(launchStatePath);
|
const stat = await fs.promises.stat(launchStatePath);
|
||||||
|
|
@ -44,38 +49,31 @@ async function readPartialLaunchStateSummary(
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
const raw = await readFileUtf8WithTimeout(launchStatePath, PER_TEAM_READ_TIMEOUT_MS);
|
const raw = await readFileUtf8WithTimeout(launchStatePath, PER_TEAM_READ_TIMEOUT_MS);
|
||||||
const parsed = JSON.parse(raw) as {
|
const snapshot = normalizePersistedLaunchSnapshot(path.basename(teamDir), JSON.parse(raw));
|
||||||
state?: unknown;
|
if (!snapshot) {
|
||||||
expectedMembers?: unknown;
|
|
||||||
confirmedMembers?: unknown;
|
|
||||||
missingMembers?: unknown;
|
|
||||||
};
|
|
||||||
if (parsed.state !== 'partial_launch_failure') {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
const expectedMembers = Array.isArray(parsed.expectedMembers)
|
|
||||||
? parsed.expectedMembers.filter(
|
|
||||||
(name): name is string => typeof name === 'string' && name.trim().length > 0
|
|
||||||
)
|
|
||||||
: [];
|
|
||||||
const confirmedMembers = Array.isArray(parsed.confirmedMembers)
|
|
||||||
? parsed.confirmedMembers.filter(
|
|
||||||
(name): name is string => typeof name === 'string' && name.trim().length > 0
|
|
||||||
)
|
|
||||||
: [];
|
|
||||||
const missingMembers = Array.isArray(parsed.missingMembers)
|
|
||||||
? parsed.missingMembers.filter(
|
|
||||||
(name): name is string => typeof name === 'string' && name.trim().length > 0
|
|
||||||
)
|
|
||||||
: [];
|
|
||||||
if (expectedMembers.length === 0 || missingMembers.length === 0) {
|
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
const missingMembers = snapshot.expectedMembers.filter((name) => {
|
||||||
|
const member = snapshot.members[name];
|
||||||
|
return member?.launchState === 'failed_to_start';
|
||||||
|
});
|
||||||
return {
|
return {
|
||||||
partialLaunchFailure: true,
|
...(snapshot.teamLaunchState === 'partial_failure'
|
||||||
expectedMemberCount: expectedMembers.length,
|
? { partialLaunchFailure: true as const }
|
||||||
confirmedMemberCount: confirmedMembers.length,
|
: {}),
|
||||||
missingMembers,
|
...(snapshot.expectedMembers.length > 0
|
||||||
|
? { expectedMemberCount: snapshot.expectedMembers.length }
|
||||||
|
: {}),
|
||||||
|
...(snapshot.summary.confirmedCount > 0
|
||||||
|
? { confirmedMemberCount: snapshot.summary.confirmedCount }
|
||||||
|
: {}),
|
||||||
|
...(missingMembers.length > 0 ? { missingMembers } : {}),
|
||||||
|
teamLaunchState: snapshot.teamLaunchState,
|
||||||
|
launchUpdatedAt: snapshot.updatedAt,
|
||||||
|
confirmedCount: snapshot.summary.confirmedCount,
|
||||||
|
pendingCount: snapshot.summary.pendingCount,
|
||||||
|
failedCount: snapshot.summary.failedCount,
|
||||||
|
runtimeAlivePendingCount: snapshot.summary.runtimeAlivePendingCount,
|
||||||
};
|
};
|
||||||
} catch {
|
} catch {
|
||||||
return null;
|
return null;
|
||||||
|
|
@ -340,8 +338,8 @@ export class TeamConfigReader {
|
||||||
}
|
}
|
||||||
|
|
||||||
const members = Array.from(memberMap.values());
|
const members = Array.from(memberMap.values());
|
||||||
const partialLaunchState =
|
const launchStateSummary =
|
||||||
(await readPartialLaunchStateSummary(teamDir)) ??
|
(await readLaunchStateSummary(teamDir)) ??
|
||||||
(() => {
|
(() => {
|
||||||
if (
|
if (
|
||||||
!leadSessionId ||
|
!leadSessionId ||
|
||||||
|
|
@ -377,7 +375,7 @@ export class TeamConfigReader {
|
||||||
...(projectPathHistory ? { projectPathHistory } : {}),
|
...(projectPathHistory ? { projectPathHistory } : {}),
|
||||||
...(sessionHistory ? { sessionHistory } : {}),
|
...(sessionHistory ? { sessionHistory } : {}),
|
||||||
...(deletedAt ? { deletedAt } : {}),
|
...(deletedAt ? { deletedAt } : {}),
|
||||||
...(partialLaunchState ?? {}),
|
...(launchStateSummary ?? {}),
|
||||||
};
|
};
|
||||||
return summary;
|
return summary;
|
||||||
} catch {
|
} catch {
|
||||||
|
|
|
||||||
426
src/main/services/team/TeamLaunchStateEvaluator.ts
Normal file
426
src/main/services/team/TeamLaunchStateEvaluator.ts
Normal file
|
|
@ -0,0 +1,426 @@
|
||||||
|
import { isLeadMember } from '@shared/utils/leadDetection';
|
||||||
|
|
||||||
|
import type {
|
||||||
|
MemberLaunchState,
|
||||||
|
MemberSpawnLivenessSource,
|
||||||
|
MemberSpawnStatusEntry,
|
||||||
|
PersistedTeamLaunchMemberSources,
|
||||||
|
PersistedTeamLaunchMemberState,
|
||||||
|
PersistedTeamLaunchPhase,
|
||||||
|
PersistedTeamLaunchSnapshot,
|
||||||
|
PersistedTeamLaunchSummary,
|
||||||
|
TeamLaunchAggregateState,
|
||||||
|
} from '@shared/types';
|
||||||
|
|
||||||
|
interface LegacyPartialLaunchStateFile {
|
||||||
|
version?: unknown;
|
||||||
|
state?: unknown;
|
||||||
|
updatedAt?: unknown;
|
||||||
|
leadSessionId?: unknown;
|
||||||
|
expectedMembers?: unknown;
|
||||||
|
confirmedMembers?: unknown;
|
||||||
|
missingMembers?: unknown;
|
||||||
|
}
|
||||||
|
|
||||||
|
type RuntimeMemberSpawnState = Pick<
|
||||||
|
MemberSpawnStatusEntry,
|
||||||
|
| 'launchState'
|
||||||
|
| 'status'
|
||||||
|
| 'error'
|
||||||
|
| 'hardFailureReason'
|
||||||
|
| 'livenessSource'
|
||||||
|
| 'agentToolAccepted'
|
||||||
|
| 'runtimeAlive'
|
||||||
|
| 'bootstrapConfirmed'
|
||||||
|
| 'hardFailure'
|
||||||
|
| 'firstSpawnAcceptedAt'
|
||||||
|
| 'lastHeartbeatAt'
|
||||||
|
| 'updatedAt'
|
||||||
|
>;
|
||||||
|
|
||||||
|
function normalizeMemberName(name: string): string {
|
||||||
|
return name.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildDiagnostics(
|
||||||
|
member: Pick<
|
||||||
|
PersistedTeamLaunchMemberState,
|
||||||
|
'agentToolAccepted' | 'runtimeAlive' | 'bootstrapConfirmed' | 'hardFailureReason' | 'sources'
|
||||||
|
>
|
||||||
|
): string[] {
|
||||||
|
const diagnostics: string[] = [];
|
||||||
|
if (member.agentToolAccepted) diagnostics.push('spawn accepted');
|
||||||
|
if (member.runtimeAlive) diagnostics.push('runtime alive');
|
||||||
|
if (member.bootstrapConfirmed) diagnostics.push('late heartbeat received');
|
||||||
|
if (member.runtimeAlive && !member.bootstrapConfirmed) diagnostics.push('waiting for bootstrap');
|
||||||
|
if (member.hardFailureReason)
|
||||||
|
diagnostics.push(`hard failure reason: ${member.hardFailureReason}`);
|
||||||
|
if (member.sources?.duplicateRespawnBlocked) diagnostics.push('respawn blocked as duplicate');
|
||||||
|
if (member.sources?.configDrift) diagnostics.push('config drift detected');
|
||||||
|
return diagnostics;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function deriveTeamLaunchAggregateState(
|
||||||
|
summary: PersistedTeamLaunchSummary
|
||||||
|
): TeamLaunchAggregateState {
|
||||||
|
if (summary.failedCount > 0) {
|
||||||
|
return 'partial_failure';
|
||||||
|
}
|
||||||
|
if (summary.pendingCount > 0) {
|
||||||
|
return 'partial_pending';
|
||||||
|
}
|
||||||
|
return 'clean_success';
|
||||||
|
}
|
||||||
|
|
||||||
|
export function summarizePersistedLaunchMembers(
|
||||||
|
expectedMembers: readonly string[],
|
||||||
|
members: Record<string, PersistedTeamLaunchMemberState>
|
||||||
|
): PersistedTeamLaunchSummary {
|
||||||
|
let confirmedCount = 0;
|
||||||
|
let pendingCount = 0;
|
||||||
|
let failedCount = 0;
|
||||||
|
let runtimeAlivePendingCount = 0;
|
||||||
|
const normalizedExpected = expectedMembers.map(normalizeMemberName).filter(Boolean);
|
||||||
|
|
||||||
|
for (const memberName of normalizedExpected) {
|
||||||
|
const entry = members[memberName];
|
||||||
|
if (!entry) {
|
||||||
|
pendingCount += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (entry.launchState === 'confirmed_alive') {
|
||||||
|
confirmedCount += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (entry.launchState === 'failed_to_start') {
|
||||||
|
failedCount += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
pendingCount += 1;
|
||||||
|
if (entry.runtimeAlive) {
|
||||||
|
runtimeAlivePendingCount += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return { confirmedCount, pendingCount, failedCount, runtimeAlivePendingCount };
|
||||||
|
}
|
||||||
|
|
||||||
|
function deriveMemberLaunchState(
|
||||||
|
member: Pick<
|
||||||
|
PersistedTeamLaunchMemberState,
|
||||||
|
'hardFailure' | 'bootstrapConfirmed' | 'runtimeAlive' | 'agentToolAccepted'
|
||||||
|
>
|
||||||
|
): MemberLaunchState {
|
||||||
|
if (member.hardFailure) {
|
||||||
|
return 'failed_to_start';
|
||||||
|
}
|
||||||
|
if (member.bootstrapConfirmed) {
|
||||||
|
return 'confirmed_alive';
|
||||||
|
}
|
||||||
|
if (member.runtimeAlive || member.agentToolAccepted) {
|
||||||
|
return 'runtime_pending_bootstrap';
|
||||||
|
}
|
||||||
|
return 'starting';
|
||||||
|
}
|
||||||
|
|
||||||
|
function toBoolean(value: unknown): boolean {
|
||||||
|
return value === true;
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeSources(value: unknown): PersistedTeamLaunchMemberSources | undefined {
|
||||||
|
if (!value || typeof value !== 'object') {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
const source = value as Record<string, unknown>;
|
||||||
|
const normalized: PersistedTeamLaunchMemberSources = {
|
||||||
|
inboxHeartbeat: toBoolean(source.inboxHeartbeat),
|
||||||
|
nativeHeartbeat: toBoolean(source.nativeHeartbeat),
|
||||||
|
processAlive: toBoolean(source.processAlive),
|
||||||
|
configRegistered: toBoolean(source.configRegistered),
|
||||||
|
configDrift: toBoolean(source.configDrift),
|
||||||
|
hardFailureSignal: toBoolean(source.hardFailureSignal),
|
||||||
|
duplicateRespawnBlocked: toBoolean(source.duplicateRespawnBlocked),
|
||||||
|
};
|
||||||
|
return Object.values(normalized).some(Boolean) ? normalized : undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizePersistedMemberState(
|
||||||
|
memberName: string,
|
||||||
|
value: unknown,
|
||||||
|
updatedAtFallback: string
|
||||||
|
): PersistedTeamLaunchMemberState | null {
|
||||||
|
if (!value || typeof value !== 'object') {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const parsed = value as Record<string, unknown>;
|
||||||
|
const normalizedName = normalizeMemberName(memberName);
|
||||||
|
if (!normalizedName || normalizedName === 'user' || isLeadMember({ name: normalizedName })) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const next: PersistedTeamLaunchMemberState = {
|
||||||
|
name: normalizedName,
|
||||||
|
launchState: 'starting',
|
||||||
|
agentToolAccepted: toBoolean(parsed.agentToolAccepted),
|
||||||
|
runtimeAlive: toBoolean(parsed.runtimeAlive),
|
||||||
|
bootstrapConfirmed: toBoolean(parsed.bootstrapConfirmed),
|
||||||
|
hardFailure: toBoolean(parsed.hardFailure),
|
||||||
|
hardFailureReason:
|
||||||
|
typeof parsed.hardFailureReason === 'string' && parsed.hardFailureReason.trim().length > 0
|
||||||
|
? parsed.hardFailureReason.trim()
|
||||||
|
: undefined,
|
||||||
|
firstSpawnAcceptedAt:
|
||||||
|
typeof parsed.firstSpawnAcceptedAt === 'string' ? parsed.firstSpawnAcceptedAt : undefined,
|
||||||
|
lastHeartbeatAt:
|
||||||
|
typeof parsed.lastHeartbeatAt === 'string' ? parsed.lastHeartbeatAt : undefined,
|
||||||
|
lastRuntimeAliveAt:
|
||||||
|
typeof parsed.lastRuntimeAliveAt === 'string' ? parsed.lastRuntimeAliveAt : undefined,
|
||||||
|
lastEvaluatedAt:
|
||||||
|
typeof parsed.lastEvaluatedAt === 'string' ? parsed.lastEvaluatedAt : updatedAtFallback,
|
||||||
|
sources: normalizeSources(parsed.sources),
|
||||||
|
diagnostics: Array.isArray(parsed.diagnostics)
|
||||||
|
? parsed.diagnostics.filter(
|
||||||
|
(item): item is string => typeof item === 'string' && item.trim().length > 0
|
||||||
|
)
|
||||||
|
: undefined,
|
||||||
|
};
|
||||||
|
const launchState =
|
||||||
|
parsed.launchState === 'starting' ||
|
||||||
|
parsed.launchState === 'runtime_pending_bootstrap' ||
|
||||||
|
parsed.launchState === 'confirmed_alive' ||
|
||||||
|
parsed.launchState === 'failed_to_start'
|
||||||
|
? parsed.launchState
|
||||||
|
: deriveMemberLaunchState(next);
|
||||||
|
next.launchState = launchState;
|
||||||
|
next.diagnostics = next.diagnostics?.length ? next.diagnostics : buildDiagnostics(next);
|
||||||
|
return next;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createPersistedLaunchSnapshot(params: {
|
||||||
|
teamName: string;
|
||||||
|
expectedMembers: readonly string[];
|
||||||
|
leadSessionId?: string;
|
||||||
|
launchPhase?: PersistedTeamLaunchPhase;
|
||||||
|
members?: Record<string, PersistedTeamLaunchMemberState>;
|
||||||
|
updatedAt?: string;
|
||||||
|
}): PersistedTeamLaunchSnapshot {
|
||||||
|
const updatedAt = params.updatedAt ?? new Date().toISOString();
|
||||||
|
const expectedMembers = Array.from(
|
||||||
|
new Set(
|
||||||
|
params.expectedMembers
|
||||||
|
.map(normalizeMemberName)
|
||||||
|
.filter((name) => name.length > 0 && name !== 'user' && !isLeadMember({ name }))
|
||||||
|
)
|
||||||
|
);
|
||||||
|
const members = params.members ?? {};
|
||||||
|
const summary = summarizePersistedLaunchMembers(expectedMembers, members);
|
||||||
|
return {
|
||||||
|
version: 2,
|
||||||
|
teamName: params.teamName,
|
||||||
|
updatedAt,
|
||||||
|
...(params.leadSessionId ? { leadSessionId: params.leadSessionId } : {}),
|
||||||
|
launchPhase: params.launchPhase ?? 'active',
|
||||||
|
expectedMembers,
|
||||||
|
members,
|
||||||
|
summary,
|
||||||
|
teamLaunchState: deriveTeamLaunchAggregateState(summary),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function snapshotFromRuntimeMemberStatuses(params: {
|
||||||
|
teamName: string;
|
||||||
|
expectedMembers: readonly string[];
|
||||||
|
leadSessionId?: string;
|
||||||
|
launchPhase?: PersistedTeamLaunchPhase;
|
||||||
|
statuses: Record<string, RuntimeMemberSpawnState>;
|
||||||
|
updatedAt?: string;
|
||||||
|
}): PersistedTeamLaunchSnapshot {
|
||||||
|
const updatedAt = params.updatedAt ?? new Date().toISOString();
|
||||||
|
const members: Record<string, PersistedTeamLaunchMemberState> = {};
|
||||||
|
|
||||||
|
for (const expected of params.expectedMembers) {
|
||||||
|
const name = normalizeMemberName(expected);
|
||||||
|
if (!name || name === 'user' || isLeadMember({ name })) continue;
|
||||||
|
const runtime = params.statuses[name];
|
||||||
|
const sources: PersistedTeamLaunchMemberSources = {};
|
||||||
|
if (runtime?.livenessSource === 'heartbeat') {
|
||||||
|
sources.nativeHeartbeat = true;
|
||||||
|
sources.inboxHeartbeat = true;
|
||||||
|
}
|
||||||
|
if (runtime?.livenessSource === 'process' || runtime?.runtimeAlive) {
|
||||||
|
sources.processAlive = true;
|
||||||
|
}
|
||||||
|
const entry: PersistedTeamLaunchMemberState = {
|
||||||
|
name,
|
||||||
|
launchState: runtime?.launchState ?? 'starting',
|
||||||
|
agentToolAccepted: runtime?.agentToolAccepted === true,
|
||||||
|
runtimeAlive: runtime?.runtimeAlive === true,
|
||||||
|
bootstrapConfirmed: runtime?.bootstrapConfirmed === true,
|
||||||
|
hardFailure: runtime?.hardFailure === true || runtime?.launchState === 'failed_to_start',
|
||||||
|
hardFailureReason: runtime?.hardFailureReason ?? runtime?.error,
|
||||||
|
firstSpawnAcceptedAt: runtime?.firstSpawnAcceptedAt,
|
||||||
|
lastHeartbeatAt: runtime?.lastHeartbeatAt,
|
||||||
|
lastRuntimeAliveAt: runtime?.runtimeAlive ? updatedAt : undefined,
|
||||||
|
lastEvaluatedAt: runtime?.updatedAt ?? updatedAt,
|
||||||
|
sources: Object.values(sources).some(Boolean) ? sources : undefined,
|
||||||
|
diagnostics: undefined,
|
||||||
|
};
|
||||||
|
entry.launchState = deriveMemberLaunchState(entry);
|
||||||
|
entry.diagnostics = buildDiagnostics(entry);
|
||||||
|
members[name] = entry;
|
||||||
|
}
|
||||||
|
|
||||||
|
return createPersistedLaunchSnapshot({
|
||||||
|
teamName: params.teamName,
|
||||||
|
expectedMembers: params.expectedMembers,
|
||||||
|
leadSessionId: params.leadSessionId,
|
||||||
|
launchPhase: params.launchPhase,
|
||||||
|
members,
|
||||||
|
updatedAt,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function snapshotToMemberSpawnStatuses(
|
||||||
|
snapshot: PersistedTeamLaunchSnapshot | null
|
||||||
|
): Record<string, MemberSpawnStatusEntry> {
|
||||||
|
if (!snapshot) return {};
|
||||||
|
const statuses: Record<string, MemberSpawnStatusEntry> = {};
|
||||||
|
for (const memberName of snapshot.expectedMembers) {
|
||||||
|
const entry = snapshot.members[memberName];
|
||||||
|
if (!entry) continue;
|
||||||
|
let status: MemberSpawnStatusEntry['status'] = 'offline';
|
||||||
|
let livenessSource: MemberSpawnLivenessSource | undefined;
|
||||||
|
if (entry.launchState === 'failed_to_start') {
|
||||||
|
status = 'error';
|
||||||
|
} else if (entry.launchState === 'confirmed_alive') {
|
||||||
|
status = 'online';
|
||||||
|
livenessSource = 'heartbeat';
|
||||||
|
} else if (entry.launchState === 'runtime_pending_bootstrap') {
|
||||||
|
status = entry.runtimeAlive ? 'online' : 'waiting';
|
||||||
|
livenessSource = entry.runtimeAlive ? 'process' : undefined;
|
||||||
|
} else {
|
||||||
|
status = entry.agentToolAccepted ? 'waiting' : 'spawning';
|
||||||
|
}
|
||||||
|
statuses[memberName] = {
|
||||||
|
status,
|
||||||
|
launchState: entry.launchState,
|
||||||
|
error: entry.hardFailure ? entry.hardFailureReason : undefined,
|
||||||
|
hardFailureReason: entry.hardFailureReason,
|
||||||
|
livenessSource,
|
||||||
|
agentToolAccepted: entry.agentToolAccepted,
|
||||||
|
runtimeAlive: entry.runtimeAlive,
|
||||||
|
bootstrapConfirmed: entry.bootstrapConfirmed,
|
||||||
|
hardFailure: entry.hardFailure,
|
||||||
|
firstSpawnAcceptedAt: entry.firstSpawnAcceptedAt,
|
||||||
|
lastHeartbeatAt: entry.lastHeartbeatAt,
|
||||||
|
updatedAt: entry.lastEvaluatedAt,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return statuses;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function normalizePersistedLaunchSnapshot(
|
||||||
|
teamName: string,
|
||||||
|
parsed: unknown
|
||||||
|
): PersistedTeamLaunchSnapshot | null {
|
||||||
|
if (!parsed || typeof parsed !== 'object') {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const maybeLegacy = parsed as LegacyPartialLaunchStateFile;
|
||||||
|
if (maybeLegacy.state === 'partial_launch_failure') {
|
||||||
|
const expectedMembers = Array.isArray(maybeLegacy.expectedMembers)
|
||||||
|
? maybeLegacy.expectedMembers.filter(
|
||||||
|
(name): name is string => typeof name === 'string' && normalizeMemberName(name).length > 0
|
||||||
|
)
|
||||||
|
: [];
|
||||||
|
const confirmedMembers = Array.isArray(maybeLegacy.confirmedMembers)
|
||||||
|
? maybeLegacy.confirmedMembers.filter(
|
||||||
|
(name): name is string => typeof name === 'string' && normalizeMemberName(name).length > 0
|
||||||
|
)
|
||||||
|
: [];
|
||||||
|
const missingMembers = Array.isArray(maybeLegacy.missingMembers)
|
||||||
|
? maybeLegacy.missingMembers.filter(
|
||||||
|
(name): name is string => typeof name === 'string' && normalizeMemberName(name).length > 0
|
||||||
|
)
|
||||||
|
: [];
|
||||||
|
if (expectedMembers.length === 0 || missingMembers.length === 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const updatedAt =
|
||||||
|
typeof maybeLegacy.updatedAt === 'string' ? maybeLegacy.updatedAt : new Date().toISOString();
|
||||||
|
const members: Record<string, PersistedTeamLaunchMemberState> = {};
|
||||||
|
for (const name of expectedMembers) {
|
||||||
|
const failed = missingMembers.includes(name);
|
||||||
|
const confirmed = confirmedMembers.includes(name);
|
||||||
|
const entry: PersistedTeamLaunchMemberState = {
|
||||||
|
name,
|
||||||
|
launchState: failed ? 'failed_to_start' : confirmed ? 'confirmed_alive' : 'starting',
|
||||||
|
agentToolAccepted: true,
|
||||||
|
runtimeAlive: confirmed,
|
||||||
|
bootstrapConfirmed: confirmed,
|
||||||
|
hardFailure: failed,
|
||||||
|
hardFailureReason: failed
|
||||||
|
? 'Legacy partial launch marker reported teammate missing.'
|
||||||
|
: undefined,
|
||||||
|
lastEvaluatedAt: updatedAt,
|
||||||
|
diagnostics: undefined,
|
||||||
|
};
|
||||||
|
entry.diagnostics = buildDiagnostics(entry);
|
||||||
|
members[name] = entry;
|
||||||
|
}
|
||||||
|
return createPersistedLaunchSnapshot({
|
||||||
|
teamName,
|
||||||
|
expectedMembers,
|
||||||
|
leadSessionId:
|
||||||
|
typeof maybeLegacy.leadSessionId === 'string' && maybeLegacy.leadSessionId.trim().length > 0
|
||||||
|
? maybeLegacy.leadSessionId.trim()
|
||||||
|
: undefined,
|
||||||
|
launchPhase: 'finished',
|
||||||
|
members,
|
||||||
|
updatedAt,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const record = parsed as Record<string, unknown>;
|
||||||
|
if (record.version !== 2) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const expectedMembers = Array.isArray(record.expectedMembers)
|
||||||
|
? record.expectedMembers.filter(
|
||||||
|
(name): name is string => typeof name === 'string' && normalizeMemberName(name).length > 0
|
||||||
|
)
|
||||||
|
: [];
|
||||||
|
const updatedAt =
|
||||||
|
typeof record.updatedAt === 'string' && record.updatedAt.trim().length > 0
|
||||||
|
? record.updatedAt
|
||||||
|
: new Date().toISOString();
|
||||||
|
const normalizedMembers: Record<string, PersistedTeamLaunchMemberState> = {};
|
||||||
|
const rawMembers =
|
||||||
|
record.members && typeof record.members === 'object'
|
||||||
|
? (record.members as Record<string, unknown>)
|
||||||
|
: {};
|
||||||
|
for (const [memberName, value] of Object.entries(rawMembers)) {
|
||||||
|
const normalized = normalizePersistedMemberState(memberName, value, updatedAt);
|
||||||
|
if (!normalized) continue;
|
||||||
|
normalizedMembers[normalized.name] = normalized;
|
||||||
|
}
|
||||||
|
return createPersistedLaunchSnapshot({
|
||||||
|
teamName:
|
||||||
|
typeof record.teamName === 'string' && record.teamName.trim().length > 0
|
||||||
|
? record.teamName.trim()
|
||||||
|
: teamName,
|
||||||
|
expectedMembers,
|
||||||
|
leadSessionId:
|
||||||
|
typeof record.leadSessionId === 'string' && record.leadSessionId.trim().length > 0
|
||||||
|
? record.leadSessionId.trim()
|
||||||
|
: undefined,
|
||||||
|
launchPhase:
|
||||||
|
record.launchPhase === 'active' ||
|
||||||
|
record.launchPhase === 'finished' ||
|
||||||
|
record.launchPhase === 'reconciled'
|
||||||
|
? record.launchPhase
|
||||||
|
: 'finished',
|
||||||
|
members: normalizedMembers,
|
||||||
|
updatedAt,
|
||||||
|
});
|
||||||
|
}
|
||||||
56
src/main/services/team/TeamLaunchStateStore.ts
Normal file
56
src/main/services/team/TeamLaunchStateStore.ts
Normal file
|
|
@ -0,0 +1,56 @@
|
||||||
|
import { getTeamsBasePath } from '@main/utils/pathDecoder';
|
||||||
|
import { createLogger } from '@shared/utils/logger';
|
||||||
|
import * as fs from 'fs';
|
||||||
|
import * as path from 'path';
|
||||||
|
|
||||||
|
import { normalizePersistedLaunchSnapshot } from './TeamLaunchStateEvaluator';
|
||||||
|
import { atomicWriteAsync } from './atomicWrite';
|
||||||
|
|
||||||
|
import type { PersistedTeamLaunchSnapshot } from '@shared/types';
|
||||||
|
|
||||||
|
const logger = createLogger('Service:TeamLaunchStateStore');
|
||||||
|
const TEAM_LAUNCH_STATE_FILE = 'launch-state.json';
|
||||||
|
const MAX_LAUNCH_STATE_BYTES = 256 * 1024;
|
||||||
|
|
||||||
|
export function getTeamLaunchStatePath(teamName: string): string {
|
||||||
|
return path.join(getTeamsBasePath(), teamName, TEAM_LAUNCH_STATE_FILE);
|
||||||
|
}
|
||||||
|
|
||||||
|
export class TeamLaunchStateStore {
|
||||||
|
async read(teamName: string): Promise<PersistedTeamLaunchSnapshot | null> {
|
||||||
|
const targetPath = getTeamLaunchStatePath(teamName);
|
||||||
|
try {
|
||||||
|
const stat = await fs.promises.stat(targetPath);
|
||||||
|
if (!stat.isFile() || stat.size > MAX_LAUNCH_STATE_BYTES) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const raw = await fs.promises.readFile(targetPath, 'utf8');
|
||||||
|
return normalizePersistedLaunchSnapshot(teamName, JSON.parse(raw));
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async write(teamName: string, snapshot: PersistedTeamLaunchSnapshot): Promise<void> {
|
||||||
|
try {
|
||||||
|
await atomicWriteAsync(
|
||||||
|
getTeamLaunchStatePath(teamName),
|
||||||
|
`${JSON.stringify(snapshot, null, 2)}\n`
|
||||||
|
);
|
||||||
|
} catch (error) {
|
||||||
|
logger.warn(
|
||||||
|
`[${teamName}] Failed to persist launch-state: ${
|
||||||
|
error instanceof Error ? error.message : String(error)
|
||||||
|
}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async clear(teamName: string): Promise<void> {
|
||||||
|
try {
|
||||||
|
await fs.promises.rm(getTeamLaunchStatePath(teamName), { force: true });
|
||||||
|
} catch {
|
||||||
|
// best-effort
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -49,6 +49,7 @@ import { createCliAutoSuffixNameGuard, parseNumericSuffixName } from '@shared/ut
|
||||||
import { normalizeOptionalTeamProviderId } from '@shared/utils/teamProvider';
|
import { normalizeOptionalTeamProviderId } from '@shared/utils/teamProvider';
|
||||||
import {
|
import {
|
||||||
extractToolPreview,
|
extractToolPreview,
|
||||||
|
parseAgentToolResultStatus,
|
||||||
extractToolResultPreview,
|
extractToolResultPreview,
|
||||||
formatToolSummaryFromCalls,
|
formatToolSummaryFromCalls,
|
||||||
} from '@shared/utils/toolSummary';
|
} from '@shared/utils/toolSummary';
|
||||||
|
|
@ -71,6 +72,12 @@ import { TeamMembersMetaStore } from './TeamMembersMetaStore';
|
||||||
import { TeamMetaStore } from './TeamMetaStore';
|
import { TeamMetaStore } from './TeamMetaStore';
|
||||||
import { TeamSentMessagesStore } from './TeamSentMessagesStore';
|
import { TeamSentMessagesStore } from './TeamSentMessagesStore';
|
||||||
import { TeamTaskReader } from './TeamTaskReader';
|
import { TeamTaskReader } from './TeamTaskReader';
|
||||||
|
import { TeamLaunchStateStore } from './TeamLaunchStateStore';
|
||||||
|
import {
|
||||||
|
createPersistedLaunchSnapshot,
|
||||||
|
snapshotFromRuntimeMemberStatuses,
|
||||||
|
snapshotToMemberSpawnStatuses,
|
||||||
|
} from './TeamLaunchStateEvaluator';
|
||||||
import {
|
import {
|
||||||
applyConfiguredRuntimeBackendsEnv,
|
applyConfiguredRuntimeBackendsEnv,
|
||||||
applyProviderRuntimeEnv,
|
applyProviderRuntimeEnv,
|
||||||
|
|
@ -110,9 +117,12 @@ import type {
|
||||||
MemberSpawnStatus,
|
MemberSpawnStatus,
|
||||||
MemberSpawnLivenessSource,
|
MemberSpawnLivenessSource,
|
||||||
MemberSpawnStatusEntry,
|
MemberSpawnStatusEntry,
|
||||||
|
PersistedTeamLaunchPhase,
|
||||||
|
PersistedTeamLaunchSummary,
|
||||||
TeamChangeEvent,
|
TeamChangeEvent,
|
||||||
TeamCreateRequest,
|
TeamCreateRequest,
|
||||||
TeamCreateResponse,
|
TeamCreateResponse,
|
||||||
|
TeamLaunchAggregateState,
|
||||||
TeamLaunchRequest,
|
TeamLaunchRequest,
|
||||||
TeamLaunchResponse,
|
TeamLaunchResponse,
|
||||||
TeamProvisioningPrepareResult,
|
TeamProvisioningPrepareResult,
|
||||||
|
|
@ -154,7 +164,6 @@ const STALL_WARNING_THRESHOLD_MS = 20_000;
|
||||||
const TEAM_JSON_READ_TIMEOUT_MS = 5_000;
|
const TEAM_JSON_READ_TIMEOUT_MS = 5_000;
|
||||||
const TEAM_CONFIG_MAX_BYTES = 10 * 1024 * 1024;
|
const TEAM_CONFIG_MAX_BYTES = 10 * 1024 * 1024;
|
||||||
const TEAM_INBOX_MAX_BYTES = 2 * 1024 * 1024;
|
const TEAM_INBOX_MAX_BYTES = 2 * 1024 * 1024;
|
||||||
const TEAM_LAUNCH_STATE_FILE = 'launch-state.json';
|
|
||||||
const CROSS_TEAM_TOOL_RECIPIENT_NAMES = new Set([
|
const CROSS_TEAM_TOOL_RECIPIENT_NAMES = new Set([
|
||||||
'cross_team_send',
|
'cross_team_send',
|
||||||
'cross_team_list_targets',
|
'cross_team_list_targets',
|
||||||
|
|
@ -383,16 +392,6 @@ type ValidConfigProbeResult =
|
||||||
| { ok: true; location: TeamsBaseLocation; configPath: string }
|
| { ok: true; location: TeamsBaseLocation; configPath: string }
|
||||||
| { ok: false };
|
| { ok: false };
|
||||||
|
|
||||||
interface PartialLaunchStateFile {
|
|
||||||
version: 1;
|
|
||||||
state: 'partial_launch_failure';
|
|
||||||
updatedAt: string;
|
|
||||||
leadSessionId?: string;
|
|
||||||
expectedMembers: string[];
|
|
||||||
confirmedMembers: string[];
|
|
||||||
missingMembers: string[];
|
|
||||||
}
|
|
||||||
|
|
||||||
function getTeamsBasePathsToProbe(): { location: TeamsBaseLocation; basePath: string }[] {
|
function getTeamsBasePathsToProbe(): { location: TeamsBaseLocation; basePath: string }[] {
|
||||||
const configured = getTeamsBasePath();
|
const configured = getTeamsBasePath();
|
||||||
const defaultBase = path.join(getAutoDetectedClaudeBasePath(), 'teams');
|
const defaultBase = path.join(getAutoDetectedClaudeBasePath(), 'teams');
|
||||||
|
|
@ -436,10 +435,6 @@ function looksLikeClaudeStdoutJsonFragment(text: string): boolean {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
function getTeamLaunchStatePath(teamName: string): string {
|
|
||||||
return path.join(getTeamsBasePath(), teamName, TEAM_LAUNCH_STATE_FILE);
|
|
||||||
}
|
|
||||||
|
|
||||||
interface ProvisioningRun {
|
interface ProvisioningRun {
|
||||||
runId: string;
|
runId: string;
|
||||||
teamName: string;
|
teamName: string;
|
||||||
|
|
@ -649,19 +644,18 @@ function createInitialMemberSpawnStatusEntry(): MemberSpawnStatusEntry {
|
||||||
}
|
}
|
||||||
|
|
||||||
function deriveMemberLaunchState(entry: {
|
function deriveMemberLaunchState(entry: {
|
||||||
status: MemberSpawnStatus;
|
|
||||||
agentToolAccepted?: boolean;
|
agentToolAccepted?: boolean;
|
||||||
runtimeAlive?: boolean;
|
runtimeAlive?: boolean;
|
||||||
bootstrapConfirmed?: boolean;
|
bootstrapConfirmed?: boolean;
|
||||||
hardFailure?: boolean;
|
hardFailure?: boolean;
|
||||||
}): MemberLaunchState {
|
}): MemberLaunchState {
|
||||||
if (entry.hardFailure || entry.status === 'error') {
|
if (entry.hardFailure) {
|
||||||
return 'failed_to_start';
|
return 'failed_to_start';
|
||||||
}
|
}
|
||||||
if (entry.bootstrapConfirmed) {
|
if (entry.bootstrapConfirmed) {
|
||||||
return 'confirmed_alive';
|
return 'confirmed_alive';
|
||||||
}
|
}
|
||||||
if (entry.runtimeAlive || entry.agentToolAccepted || entry.status === 'waiting') {
|
if (entry.runtimeAlive || entry.agentToolAccepted) {
|
||||||
return 'runtime_pending_bootstrap';
|
return 'runtime_pending_bootstrap';
|
||||||
}
|
}
|
||||||
return 'starting';
|
return 'starting';
|
||||||
|
|
@ -861,6 +855,20 @@ function buildTeammateAgentBlockReminder(): string {
|
||||||
].join('\n');
|
].join('\n');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function extractHeartbeatTimestamp(text: string, fallback?: string): string | undefined {
|
||||||
|
const trimmed = text.trim();
|
||||||
|
if (!trimmed) return fallback?.trim() || undefined;
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(trimmed) as { timestamp?: unknown };
|
||||||
|
if (typeof parsed.timestamp === 'string' && parsed.timestamp.trim().length > 0) {
|
||||||
|
return parsed.timestamp.trim();
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Best-effort only. Non-JSON teammate messages still use the inbox timestamp fallback.
|
||||||
|
}
|
||||||
|
return fallback?.trim() || undefined;
|
||||||
|
}
|
||||||
|
|
||||||
function extractBootstrapFailureReason(text: string): string | null {
|
function extractBootstrapFailureReason(text: string): string | null {
|
||||||
const trimmed = text.trim();
|
const trimmed = text.trim();
|
||||||
if (!trimmed) return null;
|
if (!trimmed) return null;
|
||||||
|
|
@ -2026,6 +2034,7 @@ export class TeamProvisioningService {
|
||||||
string,
|
string,
|
||||||
NativeSameTeamFingerprint[]
|
NativeSameTeamFingerprint[]
|
||||||
>();
|
>();
|
||||||
|
private readonly launchStateStore = new TeamLaunchStateStore();
|
||||||
private teamChangeEmitter: ((event: TeamChangeEvent) => void) | null = null;
|
private teamChangeEmitter: ((event: TeamChangeEvent) => void) | null = null;
|
||||||
private helpOutputCache: string | null = null;
|
private helpOutputCache: string | null = null;
|
||||||
private helpOutputCacheTime = 0;
|
private helpOutputCacheTime = 0;
|
||||||
|
|
@ -2659,7 +2668,14 @@ export class TeamProvisioningService {
|
||||||
this.setMemberSpawnStatus(run, from, 'error', reason);
|
this.setMemberSpawnStatus(run, from, 'error', reason);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
this.setMemberSpawnStatus(run, from, 'online', undefined, 'heartbeat');
|
this.setMemberSpawnStatus(
|
||||||
|
run,
|
||||||
|
from,
|
||||||
|
'online',
|
||||||
|
undefined,
|
||||||
|
'heartbeat',
|
||||||
|
extractHeartbeatTimestamp(message.text, message.timestamp)
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -3011,10 +3027,22 @@ export class TeamProvisioningService {
|
||||||
if (isError) {
|
if (isError) {
|
||||||
const resultPreview = extractToolResultPreview(resultContent);
|
const resultPreview = extractToolResultPreview(resultContent);
|
||||||
this.handleMemberSpawnFailure(run, spawnedMemberName, resultPreview);
|
this.handleMemberSpawnFailure(run, spawnedMemberName, resultPreview);
|
||||||
} else {
|
} else if (active.toolName === 'Agent') {
|
||||||
|
const parsedStatus = parseAgentToolResultStatus(resultContent);
|
||||||
|
if (parsedStatus?.status === 'duplicate_skipped') {
|
||||||
|
const detail =
|
||||||
|
parsedStatus.reason === 'already_running'
|
||||||
|
? 'duplicate spawn skipped - already running'
|
||||||
|
: 'duplicate spawn skipped - bootstrap pending';
|
||||||
|
this.appendMemberBootstrapDiagnostic(run, spawnedMemberName, detail);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
// Agent tool_result only confirms that the runtime accepted the spawn.
|
// Agent tool_result only confirms that the runtime accepted the spawn.
|
||||||
// The teammate becomes truly "online" only after the first inbox heartbeat.
|
// The teammate becomes truly "online" only after the first inbox heartbeat.
|
||||||
this.setMemberSpawnStatus(run, spawnedMemberName, 'waiting');
|
this.setMemberSpawnStatus(run, spawnedMemberName, 'waiting');
|
||||||
|
} else {
|
||||||
|
this.setMemberSpawnStatus(run, spawnedMemberName, 'waiting');
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -3089,7 +3117,8 @@ export class TeamProvisioningService {
|
||||||
memberName: string,
|
memberName: string,
|
||||||
status: MemberSpawnStatus,
|
status: MemberSpawnStatus,
|
||||||
error?: string,
|
error?: string,
|
||||||
livenessSource?: MemberSpawnLivenessSource
|
livenessSource?: MemberSpawnLivenessSource,
|
||||||
|
heartbeatAt?: string
|
||||||
): void {
|
): void {
|
||||||
const prev = run.memberSpawnStatuses.get(memberName) ?? createInitialMemberSpawnStatusEntry();
|
const prev = run.memberSpawnStatuses.get(memberName) ?? createInitialMemberSpawnStatusEntry();
|
||||||
const updatedAt = nowIso();
|
const updatedAt = nowIso();
|
||||||
|
|
@ -3115,7 +3144,7 @@ export class TeamProvisioningService {
|
||||||
next.firstSpawnAcceptedAt = prev.firstSpawnAcceptedAt ?? updatedAt;
|
next.firstSpawnAcceptedAt = prev.firstSpawnAcceptedAt ?? updatedAt;
|
||||||
if (livenessSource === 'heartbeat') {
|
if (livenessSource === 'heartbeat') {
|
||||||
next.bootstrapConfirmed = true;
|
next.bootstrapConfirmed = true;
|
||||||
next.lastHeartbeatAt = updatedAt;
|
next.lastHeartbeatAt = heartbeatAt?.trim() || prev.lastHeartbeatAt || updatedAt;
|
||||||
}
|
}
|
||||||
next.hardFailure = false;
|
next.hardFailure = false;
|
||||||
next.error = undefined;
|
next.error = undefined;
|
||||||
|
|
@ -3158,7 +3187,7 @@ export class TeamProvisioningService {
|
||||||
memberName,
|
memberName,
|
||||||
'spawn accepted, waiting for bootstrap'
|
'spawn accepted, waiting for bootstrap'
|
||||||
);
|
);
|
||||||
} else if (status === 'online' && livenessSource === 'heartbeat') {
|
} else if (status === 'online' && livenessSource === 'heartbeat' && !prev.bootstrapConfirmed) {
|
||||||
this.appendMemberBootstrapDiagnostic(
|
this.appendMemberBootstrapDiagnostic(
|
||||||
run,
|
run,
|
||||||
memberName,
|
memberName,
|
||||||
|
|
@ -3184,38 +3213,67 @@ export class TeamProvisioningService {
|
||||||
runId: run.runId,
|
runId: run.runId,
|
||||||
detail: memberName,
|
detail: memberName,
|
||||||
});
|
});
|
||||||
|
if (run.isLaunch) {
|
||||||
|
void this.persistLaunchStateSnapshot(run, run.provisioningComplete ? 'finished' : 'active');
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get current member spawn statuses for a team.
|
* Get current member spawn statuses for a team.
|
||||||
* Returns a map of memberName → MemberSpawnStatusEntry.
|
* Returns a map of memberName → MemberSpawnStatusEntry.
|
||||||
*/
|
*/
|
||||||
getMemberSpawnStatuses(teamName: string): {
|
async getMemberSpawnStatuses(teamName: string): Promise<{
|
||||||
statuses: Record<string, MemberSpawnStatusEntry>;
|
statuses: Record<string, MemberSpawnStatusEntry>;
|
||||||
runId: string | null;
|
runId: string | null;
|
||||||
} {
|
teamLaunchState?: TeamLaunchAggregateState;
|
||||||
|
launchPhase?: PersistedTeamLaunchPhase;
|
||||||
|
expectedMembers?: string[];
|
||||||
|
updatedAt?: string;
|
||||||
|
summary?: PersistedTeamLaunchSummary;
|
||||||
|
source?: 'live' | 'persisted' | 'merged';
|
||||||
|
}> {
|
||||||
const runId = this.getTrackedRunId(teamName);
|
const runId = this.getTrackedRunId(teamName);
|
||||||
if (!runId) return { statuses: {}, runId: null };
|
if (!runId) {
|
||||||
const run = this.runs.get(runId);
|
return this.reconcilePersistedLaunchState(teamName).then(({ snapshot, statuses }) => ({
|
||||||
if (!run) return { statuses: {}, runId: null };
|
statuses,
|
||||||
const result: Record<string, MemberSpawnStatusEntry> = {};
|
runId: null,
|
||||||
for (const [name, entry] of run.memberSpawnStatuses) {
|
teamLaunchState: snapshot?.teamLaunchState,
|
||||||
result[name] = {
|
launchPhase: snapshot?.launchPhase,
|
||||||
status: entry.status,
|
expectedMembers: snapshot?.expectedMembers,
|
||||||
launchState: entry.launchState,
|
updatedAt: snapshot?.updatedAt,
|
||||||
error: entry.error,
|
summary: snapshot?.summary,
|
||||||
hardFailureReason: entry.hardFailureReason,
|
source: snapshot ? 'persisted' : 'persisted',
|
||||||
livenessSource: entry.livenessSource,
|
}));
|
||||||
agentToolAccepted: entry.agentToolAccepted,
|
|
||||||
runtimeAlive: entry.runtimeAlive,
|
|
||||||
bootstrapConfirmed: entry.bootstrapConfirmed,
|
|
||||||
hardFailure: entry.hardFailure,
|
|
||||||
firstSpawnAcceptedAt: entry.firstSpawnAcceptedAt,
|
|
||||||
lastHeartbeatAt: entry.lastHeartbeatAt,
|
|
||||||
updatedAt: entry.updatedAt,
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
return { statuses: result, runId };
|
const run = this.runs.get(runId);
|
||||||
|
if (!run) {
|
||||||
|
return { statuses: {}, runId: null, source: 'persisted' };
|
||||||
|
}
|
||||||
|
|
||||||
|
await this.refreshMemberSpawnStatusesFromLeadInbox(run);
|
||||||
|
await this.auditMemberSpawnStatuses(run);
|
||||||
|
await this.persistLaunchStateSnapshot(run, run.provisioningComplete ? 'finished' : 'active');
|
||||||
|
|
||||||
|
const persisted = await this.launchStateStore.read(teamName);
|
||||||
|
const liveSnapshot = snapshotFromRuntimeMemberStatuses({
|
||||||
|
teamName: run.teamName,
|
||||||
|
expectedMembers: run.expectedMembers,
|
||||||
|
leadSessionId: run.detectedSessionId ?? undefined,
|
||||||
|
launchPhase: run.provisioningComplete ? 'finished' : 'active',
|
||||||
|
statuses: this.buildRuntimeSpawnStatusRecord(run),
|
||||||
|
});
|
||||||
|
const snapshot = persisted ?? liveSnapshot;
|
||||||
|
const statuses = snapshotToMemberSpawnStatuses(snapshot);
|
||||||
|
return {
|
||||||
|
statuses,
|
||||||
|
runId,
|
||||||
|
teamLaunchState: snapshot.teamLaunchState,
|
||||||
|
launchPhase: snapshot.launchPhase,
|
||||||
|
expectedMembers: snapshot.expectedMembers,
|
||||||
|
updatedAt: snapshot.updatedAt,
|
||||||
|
summary: snapshot.summary,
|
||||||
|
source: persisted ? 'merged' : 'live',
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
private getMemberLaunchGraceKey(run: ProvisioningRun, memberName: string): string {
|
private getMemberLaunchGraceKey(run: ProvisioningRun, memberName: string): string {
|
||||||
|
|
@ -4290,7 +4348,7 @@ export class TeamProvisioningService {
|
||||||
this.runs.set(runId, run);
|
this.runs.set(runId, run);
|
||||||
this.provisioningRunByTeam.set(request.teamName, runId);
|
this.provisioningRunByTeam.set(request.teamName, runId);
|
||||||
run.onProgress(run.progress);
|
run.onProgress(run.progress);
|
||||||
await this.clearPartialLaunchState(request.teamName);
|
await this.clearPersistedLaunchState(request.teamName);
|
||||||
|
|
||||||
const prompt = buildProvisioningPrompt(request, effectiveMemberSpecs);
|
const prompt = buildProvisioningPrompt(request, effectiveMemberSpecs);
|
||||||
const promptSize = getPromptSizeSummary(prompt);
|
const promptSize = getPromptSizeSummary(prompt);
|
||||||
|
|
@ -5886,6 +5944,19 @@ export class TeamProvisioningService {
|
||||||
}
|
}
|
||||||
// Only track spawns for this team
|
// Only track spawns for this team
|
||||||
if (teamName !== run.teamName) continue;
|
if (teamName !== run.teamName) continue;
|
||||||
|
const existing = run.memberSpawnStatuses.get(memberName);
|
||||||
|
if (
|
||||||
|
existing &&
|
||||||
|
!existing.hardFailure &&
|
||||||
|
(existing.bootstrapConfirmed || existing.runtimeAlive || existing.agentToolAccepted)
|
||||||
|
) {
|
||||||
|
this.appendMemberBootstrapDiagnostic(
|
||||||
|
run,
|
||||||
|
memberName,
|
||||||
|
'respawn blocked as duplicate — teammate already alive or bootstrap pending'
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
this.setMemberSpawnStatus(run, memberName, 'spawning');
|
this.setMemberSpawnStatus(run, memberName, 'spawning');
|
||||||
const toolUseId = typeof part.id === 'string' ? part.id.trim() : '';
|
const toolUseId = typeof part.id === 'string' ? part.id.trim() : '';
|
||||||
if (toolUseId) {
|
if (toolUseId) {
|
||||||
|
|
@ -5940,6 +6011,8 @@ export class TeamProvisioningService {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const liveAgentNames = this.getLiveTeamAgentNames(run.teamName);
|
||||||
|
|
||||||
// Flag any expected member not found in config.json (excluding the lead)
|
// Flag any expected member not found in config.json (excluding the lead)
|
||||||
for (const expected of run.expectedMembers) {
|
for (const expected of run.expectedMembers) {
|
||||||
const current = run.memberSpawnStatuses.get(expected);
|
const current = run.memberSpawnStatuses.get(expected);
|
||||||
|
|
@ -5957,10 +6030,8 @@ export class TeamProvisioningService {
|
||||||
});
|
});
|
||||||
|
|
||||||
const runtimeAlive =
|
const runtimeAlive =
|
||||||
matchedRuntimeNames.length > 0 &&
|
liveAgentNames.has(expected) ||
|
||||||
matchedRuntimeNames.some((runtimeName) =>
|
matchedRuntimeNames.some((runtimeName) => liveAgentNames.has(runtimeName));
|
||||||
this.hasLiveTeamAgentProcess(run.teamName, runtimeName)
|
|
||||||
);
|
|
||||||
|
|
||||||
// A teammate may intentionally stay silent after bootstrap. If Claude Code
|
// A teammate may intentionally stay silent after bootstrap. If Claude Code
|
||||||
// registered the runtime and the OS process is still alive, treat it as
|
// registered the runtime and the OS process is still alive, treat it as
|
||||||
|
|
@ -6004,8 +6075,12 @@ export class TeamProvisioningService {
|
||||||
}
|
}
|
||||||
|
|
||||||
private hasLiveTeamAgentProcess(teamName: string, memberName: string): boolean {
|
private hasLiveTeamAgentProcess(teamName: string, memberName: string): boolean {
|
||||||
|
return this.getLiveTeamAgentNames(teamName).has(memberName);
|
||||||
|
}
|
||||||
|
|
||||||
|
private getLiveTeamAgentNames(teamName: string): Set<string> {
|
||||||
if (process.platform === 'win32') {
|
if (process.platform === 'win32') {
|
||||||
return false;
|
return new Set();
|
||||||
}
|
}
|
||||||
|
|
||||||
let output = '';
|
let output = '';
|
||||||
|
|
@ -6015,24 +6090,26 @@ export class TeamProvisioningService {
|
||||||
stdio: ['ignore', 'pipe', 'ignore'],
|
stdio: ['ignore', 'pipe', 'ignore'],
|
||||||
});
|
});
|
||||||
} catch {
|
} catch {
|
||||||
return false;
|
return new Set();
|
||||||
}
|
}
|
||||||
|
|
||||||
const teamMarker = `--team-name ${teamName}`;
|
const teamMarker = `--team-name ${teamName}`;
|
||||||
const memberMarker = `--agent-id ${memberName}@${teamName}`;
|
const names = new Set<string>();
|
||||||
|
for (const line of output.split('\n')) {
|
||||||
return output.split('\n').some((line) => {
|
|
||||||
const trimmed = line.trim();
|
const trimmed = line.trim();
|
||||||
return trimmed.includes(teamMarker) && trimmed.includes(memberMarker);
|
if (!trimmed.includes(teamMarker)) continue;
|
||||||
});
|
const match = trimmed.match(/--agent-id\s+([^\s@]+)@/);
|
||||||
|
if (!match) continue;
|
||||||
|
const agentName = match[1]?.trim();
|
||||||
|
if (agentName) {
|
||||||
|
names.add(agentName);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return names;
|
||||||
}
|
}
|
||||||
|
|
||||||
private async clearPartialLaunchState(teamName: string): Promise<void> {
|
private async clearPersistedLaunchState(teamName: string): Promise<void> {
|
||||||
try {
|
await this.launchStateStore.clear(teamName);
|
||||||
await fs.promises.rm(getTeamLaunchStatePath(teamName), { force: true });
|
|
||||||
} catch {
|
|
||||||
// best-effort
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private getFailedSpawnMembers(
|
private getFailedSpawnMembers(
|
||||||
|
|
@ -6076,95 +6153,190 @@ export class TeamProvisioningService {
|
||||||
return { confirmedCount, pendingCount, failedCount, runtimeAlivePendingCount };
|
return { confirmedCount, pendingCount, failedCount, runtimeAlivePendingCount };
|
||||||
}
|
}
|
||||||
|
|
||||||
private async persistPartialLaunchState(run: ProvisioningRun): Promise<void> {
|
private buildRuntimeSpawnStatusRecord(
|
||||||
|
run: ProvisioningRun
|
||||||
|
): Record<string, MemberSpawnStatusEntry> {
|
||||||
|
const statuses: Record<string, MemberSpawnStatusEntry> = {};
|
||||||
|
for (const expected of run.expectedMembers) {
|
||||||
|
statuses[expected] =
|
||||||
|
run.memberSpawnStatuses.get(expected) ?? createInitialMemberSpawnStatusEntry();
|
||||||
|
}
|
||||||
|
return statuses;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async persistLaunchStateSnapshot(
|
||||||
|
run: ProvisioningRun,
|
||||||
|
launchPhase: 'active' | 'finished' | 'reconciled' = run.provisioningComplete
|
||||||
|
? 'finished'
|
||||||
|
: 'active'
|
||||||
|
): Promise<void> {
|
||||||
if (!run.isLaunch || !run.expectedMembers || run.expectedMembers.length === 0) {
|
if (!run.isLaunch || !run.expectedMembers || run.expectedMembers.length === 0) {
|
||||||
if (run.isLaunch) {
|
if (run.isLaunch) {
|
||||||
await this.clearPartialLaunchState(run.teamName);
|
await this.clearPersistedLaunchState(run.teamName);
|
||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const expectedMembers = Array.from(
|
const snapshot = snapshotFromRuntimeMemberStatuses({
|
||||||
new Set(
|
teamName: run.teamName,
|
||||||
run.expectedMembers
|
expectedMembers: run.expectedMembers,
|
||||||
.map((name) => name.trim())
|
leadSessionId: run.detectedSessionId ?? undefined,
|
||||||
.filter((name) => name.length > 0 && name !== 'user' && !isLeadMember({ name }))
|
launchPhase,
|
||||||
)
|
statuses: this.buildRuntimeSpawnStatusRecord(run),
|
||||||
);
|
});
|
||||||
if (expectedMembers.length === 0) {
|
|
||||||
await this.clearPartialLaunchState(run.teamName);
|
if (snapshot.teamLaunchState === 'clean_success' && launchPhase !== 'active') {
|
||||||
|
await this.clearPersistedLaunchState(run.teamName);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const configPath = path.join(getTeamsBasePath(), run.teamName, 'config.json');
|
await this.launchStateStore.write(run.teamName, snapshot);
|
||||||
let registeredMembers = new Set<string>();
|
}
|
||||||
let leadSessionId: string | undefined;
|
|
||||||
|
private async reconcilePersistedLaunchState(teamName: string): Promise<{
|
||||||
|
snapshot: ReturnType<typeof createPersistedLaunchSnapshot> | null;
|
||||||
|
statuses: Record<string, MemberSpawnStatusEntry>;
|
||||||
|
}> {
|
||||||
|
const persisted = await this.launchStateStore.read(teamName);
|
||||||
|
if (!persisted) {
|
||||||
|
return { snapshot: null, statuses: {} };
|
||||||
|
}
|
||||||
|
|
||||||
|
const configPath = path.join(getTeamsBasePath(), teamName, 'config.json');
|
||||||
|
let configMembers = new Set<string>();
|
||||||
|
let leadName = 'team-lead';
|
||||||
try {
|
try {
|
||||||
const raw = await tryReadRegularFileUtf8(configPath, {
|
const raw = await tryReadRegularFileUtf8(configPath, {
|
||||||
timeoutMs: TEAM_JSON_READ_TIMEOUT_MS,
|
timeoutMs: TEAM_JSON_READ_TIMEOUT_MS,
|
||||||
maxBytes: TEAM_CONFIG_MAX_BYTES,
|
maxBytes: TEAM_CONFIG_MAX_BYTES,
|
||||||
});
|
});
|
||||||
if (!raw) {
|
if (raw) {
|
||||||
await this.clearPartialLaunchState(run.teamName);
|
const config = JSON.parse(raw) as {
|
||||||
return;
|
members?: { name?: string; agentType?: string }[];
|
||||||
|
};
|
||||||
|
leadName = config.members?.find((member) => isLeadMember(member))?.name?.trim() || leadName;
|
||||||
|
configMembers = new Set(
|
||||||
|
(config.members ?? [])
|
||||||
|
.map((member) => (typeof member?.name === 'string' ? member.name.trim() : ''))
|
||||||
|
.filter((name) => name.length > 0 && !isLeadMember({ name }))
|
||||||
|
);
|
||||||
}
|
}
|
||||||
const config = JSON.parse(raw) as {
|
|
||||||
leadSessionId?: unknown;
|
|
||||||
members?: { name?: unknown }[];
|
|
||||||
};
|
|
||||||
leadSessionId =
|
|
||||||
typeof config.leadSessionId === 'string' && config.leadSessionId.trim().length > 0
|
|
||||||
? config.leadSessionId.trim()
|
|
||||||
: undefined;
|
|
||||||
registeredMembers = new Set(
|
|
||||||
(config.members ?? [])
|
|
||||||
.map((member) => (typeof member?.name === 'string' ? member.name.trim() : ''))
|
|
||||||
.filter((name) => name.length > 0 && !isLeadMember({ name }))
|
|
||||||
);
|
|
||||||
} catch {
|
} catch {
|
||||||
await this.clearPartialLaunchState(run.teamName);
|
// best-effort
|
||||||
return;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const inboxNames = await this.inboxReader
|
let leadInboxMessages: Awaited<ReturnType<TeamInboxReader['getMessagesFor']>> = [];
|
||||||
.listInboxNames(run.teamName)
|
|
||||||
.catch(() => [] as string[]);
|
|
||||||
const confirmedMembers = Array.from(
|
|
||||||
new Set(
|
|
||||||
[...registeredMembers, ...inboxNames]
|
|
||||||
.map((name) => name.trim())
|
|
||||||
.filter((name) => expectedMembers.includes(name))
|
|
||||||
)
|
|
||||||
);
|
|
||||||
const missingMembers = expectedMembers.filter((name) => !confirmedMembers.includes(name));
|
|
||||||
|
|
||||||
if (missingMembers.length === 0 || confirmedMembers.length === 0) {
|
|
||||||
await this.clearPartialLaunchState(run.teamName);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const payload: PartialLaunchStateFile = {
|
|
||||||
version: 1,
|
|
||||||
state: 'partial_launch_failure',
|
|
||||||
updatedAt: new Date().toISOString(),
|
|
||||||
...(leadSessionId ? { leadSessionId } : {}),
|
|
||||||
expectedMembers,
|
|
||||||
confirmedMembers,
|
|
||||||
missingMembers,
|
|
||||||
};
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await atomicWriteAsync(
|
leadInboxMessages = await this.inboxReader.getMessagesFor(teamName, leadName);
|
||||||
getTeamLaunchStatePath(run.teamName),
|
} catch {
|
||||||
JSON.stringify(payload, null, 2)
|
// best-effort
|
||||||
);
|
|
||||||
} catch (error) {
|
|
||||||
logger.warn(
|
|
||||||
`[${run.teamName}] Failed to persist partial launch state: ${
|
|
||||||
error instanceof Error ? error.message : String(error)
|
|
||||||
}`
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const liveAgentNames = this.getLiveTeamAgentNames(teamName);
|
||||||
|
const nextMembers = { ...persisted.members };
|
||||||
|
const now = nowIso();
|
||||||
|
for (const expected of persisted.expectedMembers) {
|
||||||
|
const current = nextMembers[expected] ?? {
|
||||||
|
name: expected,
|
||||||
|
launchState: 'starting',
|
||||||
|
agentToolAccepted: false,
|
||||||
|
runtimeAlive: false,
|
||||||
|
bootstrapConfirmed: false,
|
||||||
|
hardFailure: false,
|
||||||
|
lastEvaluatedAt: now,
|
||||||
|
};
|
||||||
|
const matchedRuntimeNames = [...configMembers].filter((name) => {
|
||||||
|
if (name === expected) return true;
|
||||||
|
const parsed = parseNumericSuffixName(name);
|
||||||
|
return parsed !== null && parsed.suffix >= 2 && parsed.base === expected;
|
||||||
|
});
|
||||||
|
const runtimeAlive =
|
||||||
|
liveAgentNames.has(expected) ||
|
||||||
|
matchedRuntimeNames.some((runtimeName) => liveAgentNames.has(runtimeName));
|
||||||
|
const heartbeatMessage = leadInboxMessages.find((message) => {
|
||||||
|
if (typeof message.from !== 'string' || message.from.trim() !== expected) return false;
|
||||||
|
if (typeof message.text !== 'string' || message.text.trim().length === 0) return false;
|
||||||
|
const firstAcceptedAt = current.firstSpawnAcceptedAt
|
||||||
|
? Date.parse(current.firstSpawnAcceptedAt)
|
||||||
|
: NaN;
|
||||||
|
const messageTs = Date.parse(message.timestamp);
|
||||||
|
if (
|
||||||
|
Number.isFinite(firstAcceptedAt) &&
|
||||||
|
Number.isFinite(messageTs) &&
|
||||||
|
messageTs < firstAcceptedAt
|
||||||
|
) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
const heartbeatReason = heartbeatMessage
|
||||||
|
? extractBootstrapFailureReason(heartbeatMessage.text)
|
||||||
|
: null;
|
||||||
|
current.runtimeAlive = runtimeAlive;
|
||||||
|
current.lastRuntimeAliveAt = runtimeAlive ? now : current.lastRuntimeAliveAt;
|
||||||
|
current.sources = {
|
||||||
|
...(current.sources ?? {}),
|
||||||
|
processAlive: runtimeAlive || undefined,
|
||||||
|
configRegistered: matchedRuntimeNames.length > 0 || undefined,
|
||||||
|
configDrift:
|
||||||
|
heartbeatMessage != null && matchedRuntimeNames.length === 0
|
||||||
|
? true
|
||||||
|
: current.sources?.configDrift,
|
||||||
|
inboxHeartbeat: heartbeatMessage != null ? true : current.sources?.inboxHeartbeat,
|
||||||
|
};
|
||||||
|
if (heartbeatReason) {
|
||||||
|
current.hardFailure = true;
|
||||||
|
current.hardFailureReason = heartbeatReason;
|
||||||
|
current.sources.hardFailureSignal = true;
|
||||||
|
} else if (heartbeatMessage) {
|
||||||
|
current.bootstrapConfirmed = true;
|
||||||
|
current.lastHeartbeatAt = heartbeatMessage.timestamp;
|
||||||
|
current.hardFailure = false;
|
||||||
|
current.hardFailureReason = undefined;
|
||||||
|
}
|
||||||
|
const acceptedAtMs =
|
||||||
|
current.firstSpawnAcceptedAt != null ? Date.parse(current.firstSpawnAcceptedAt) : NaN;
|
||||||
|
const graceExpired =
|
||||||
|
current.agentToolAccepted === true &&
|
||||||
|
Number.isFinite(acceptedAtMs) &&
|
||||||
|
Date.now() - acceptedAtMs >= MEMBER_LAUNCH_GRACE_MS;
|
||||||
|
if (
|
||||||
|
!current.bootstrapConfirmed &&
|
||||||
|
!current.runtimeAlive &&
|
||||||
|
!current.hardFailure &&
|
||||||
|
graceExpired
|
||||||
|
) {
|
||||||
|
current.hardFailure = true;
|
||||||
|
current.hardFailureReason =
|
||||||
|
current.hardFailureReason ?? 'Teammate did not join within the launch grace window.';
|
||||||
|
}
|
||||||
|
current.launchState = deriveMemberLaunchState(current);
|
||||||
|
current.lastEvaluatedAt = now;
|
||||||
|
nextMembers[expected] = {
|
||||||
|
...current,
|
||||||
|
diagnostics: undefined,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const reconciled = createPersistedLaunchSnapshot({
|
||||||
|
teamName,
|
||||||
|
expectedMembers: persisted.expectedMembers,
|
||||||
|
leadSessionId: persisted.leadSessionId,
|
||||||
|
launchPhase: persisted.launchPhase === 'active' ? 'active' : 'reconciled',
|
||||||
|
members: nextMembers,
|
||||||
|
updatedAt: now,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (reconciled.teamLaunchState === 'clean_success') {
|
||||||
|
await this.clearPersistedLaunchState(teamName);
|
||||||
|
return { snapshot: null, statuses: {} };
|
||||||
|
}
|
||||||
|
|
||||||
|
await this.launchStateStore.write(teamName, reconciled);
|
||||||
|
return {
|
||||||
|
snapshot: reconciled,
|
||||||
|
statuses: snapshotToMemberSpawnStatuses(reconciled),
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
private captureSendMessages(run: ProvisioningRun, content: Record<string, unknown>[]): void {
|
private captureSendMessages(run: ProvisioningRun, content: Record<string, unknown>[]): void {
|
||||||
|
|
@ -8411,7 +8583,7 @@ export class TeamProvisioningService {
|
||||||
// Audit: flag any expected member not registered in config.json after launch.
|
// Audit: flag any expected member not registered in config.json after launch.
|
||||||
await this.refreshMemberSpawnStatusesFromLeadInbox(run);
|
await this.refreshMemberSpawnStatusesFromLeadInbox(run);
|
||||||
await this.auditMemberSpawnStatuses(run);
|
await this.auditMemberSpawnStatuses(run);
|
||||||
await this.persistPartialLaunchState(run);
|
await this.persistLaunchStateSnapshot(run, 'finished');
|
||||||
const failedSpawnMembers = this.getFailedSpawnMembers(run);
|
const failedSpawnMembers = this.getFailedSpawnMembers(run);
|
||||||
const launchSummary = this.getMemberLaunchSummary(run);
|
const launchSummary = this.getMemberLaunchSummary(run);
|
||||||
const hasSpawnFailures = failedSpawnMembers.length > 0;
|
const hasSpawnFailures = failedSpawnMembers.length > 0;
|
||||||
|
|
@ -8570,7 +8742,7 @@ export class TeamProvisioningService {
|
||||||
// Audit: flag any expected member not registered in config.json after provisioning.
|
// Audit: flag any expected member not registered in config.json after provisioning.
|
||||||
await this.refreshMemberSpawnStatusesFromLeadInbox(run);
|
await this.refreshMemberSpawnStatusesFromLeadInbox(run);
|
||||||
await this.auditMemberSpawnStatuses(run);
|
await this.auditMemberSpawnStatuses(run);
|
||||||
await this.clearPartialLaunchState(run.teamName);
|
await this.persistLaunchStateSnapshot(run, 'finished');
|
||||||
const failedSpawnMembers = this.getFailedSpawnMembers(run);
|
const failedSpawnMembers = this.getFailedSpawnMembers(run);
|
||||||
const launchSummary = this.getMemberLaunchSummary(run);
|
const launchSummary = this.getMemberLaunchSummary(run);
|
||||||
const hasSpawnFailures = failedSpawnMembers.length > 0;
|
const hasSpawnFailures = failedSpawnMembers.length > 0;
|
||||||
|
|
@ -8941,7 +9113,7 @@ export class TeamProvisioningService {
|
||||||
*/
|
*/
|
||||||
private cleanupRun(run: ProvisioningRun): void {
|
private cleanupRun(run: ProvisioningRun): void {
|
||||||
if (run.isLaunch && !run.provisioningComplete) {
|
if (run.isLaunch && !run.provisioningComplete) {
|
||||||
void this.persistPartialLaunchState(run);
|
void this.persistLaunchStateSnapshot(run, 'finished');
|
||||||
}
|
}
|
||||||
this.resetRuntimeToolActivity(run);
|
this.resetRuntimeToolActivity(run);
|
||||||
this.setLeadActivity(run, 'offline');
|
this.setLeadActivity(run, 'offline');
|
||||||
|
|
|
||||||
|
|
@ -3,6 +3,7 @@ import * as path from 'node:path';
|
||||||
import { parentPort } from 'node:worker_threads';
|
import { parentPort } from 'node:worker_threads';
|
||||||
|
|
||||||
import { isLeadMember } from '@shared/utils/leadDetection';
|
import { isLeadMember } from '@shared/utils/leadDetection';
|
||||||
|
import { normalizePersistedLaunchSnapshot } from '@main/services/team/TeamLaunchStateEvaluator';
|
||||||
|
|
||||||
interface ListTeamsPayload {
|
interface ListTeamsPayload {
|
||||||
teamsDir: string;
|
teamsDir: string;
|
||||||
|
|
@ -319,48 +320,49 @@ function dropCliProvisionerMembers(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function readPartialLaunchState(
|
async function readLaunchState(
|
||||||
teamsDir: string,
|
teamsDir: string,
|
||||||
teamName: string
|
teamName: string
|
||||||
): Promise<{
|
): Promise<{
|
||||||
partialLaunchFailure: true;
|
partialLaunchFailure?: true;
|
||||||
expectedMemberCount: number;
|
expectedMemberCount?: number;
|
||||||
confirmedMemberCount: number;
|
confirmedMemberCount?: number;
|
||||||
missingMembers: string[];
|
missingMembers?: string[];
|
||||||
|
teamLaunchState?: string;
|
||||||
|
launchUpdatedAt?: string;
|
||||||
|
confirmedCount?: number;
|
||||||
|
pendingCount?: number;
|
||||||
|
failedCount?: number;
|
||||||
|
runtimeAlivePendingCount?: number;
|
||||||
} | null> {
|
} | null> {
|
||||||
const launchStatePath = path.join(teamsDir, teamName, TEAM_LAUNCH_STATE_FILE);
|
const launchStatePath = path.join(teamsDir, teamName, TEAM_LAUNCH_STATE_FILE);
|
||||||
try {
|
try {
|
||||||
const stat = await fs.promises.stat(launchStatePath);
|
const stat = await fs.promises.stat(launchStatePath);
|
||||||
if (!stat.isFile() || stat.size > MAX_LAUNCH_STATE_BYTES) return null;
|
if (!stat.isFile() || stat.size > MAX_LAUNCH_STATE_BYTES) return null;
|
||||||
const raw = await fs.promises.readFile(launchStatePath, 'utf8');
|
const raw = await fs.promises.readFile(launchStatePath, 'utf8');
|
||||||
const parsed = JSON.parse(raw) as {
|
const snapshot = normalizePersistedLaunchSnapshot(teamName, JSON.parse(raw));
|
||||||
state?: unknown;
|
if (!snapshot) return null;
|
||||||
expectedMembers?: unknown;
|
const missingMembers = snapshot.expectedMembers.filter((name) => {
|
||||||
confirmedMembers?: unknown;
|
const member = snapshot.members[name];
|
||||||
missingMembers?: unknown;
|
return member?.launchState === 'failed_to_start';
|
||||||
};
|
});
|
||||||
if (parsed.state !== 'partial_launch_failure') return null;
|
|
||||||
const expectedMembers = Array.isArray(parsed.expectedMembers)
|
|
||||||
? parsed.expectedMembers.filter(
|
|
||||||
(name): name is string => typeof name === 'string' && name.trim().length > 0
|
|
||||||
)
|
|
||||||
: [];
|
|
||||||
const confirmedMembers = Array.isArray(parsed.confirmedMembers)
|
|
||||||
? parsed.confirmedMembers.filter(
|
|
||||||
(name): name is string => typeof name === 'string' && name.trim().length > 0
|
|
||||||
)
|
|
||||||
: [];
|
|
||||||
const missingMembers = Array.isArray(parsed.missingMembers)
|
|
||||||
? parsed.missingMembers.filter(
|
|
||||||
(name): name is string => typeof name === 'string' && name.trim().length > 0
|
|
||||||
)
|
|
||||||
: [];
|
|
||||||
if (expectedMembers.length === 0 || missingMembers.length === 0) return null;
|
|
||||||
return {
|
return {
|
||||||
partialLaunchFailure: true,
|
...(snapshot.teamLaunchState === 'partial_failure'
|
||||||
expectedMemberCount: expectedMembers.length,
|
? { partialLaunchFailure: true as const }
|
||||||
confirmedMemberCount: confirmedMembers.length,
|
: {}),
|
||||||
missingMembers,
|
...(snapshot.expectedMembers.length > 0
|
||||||
|
? { expectedMemberCount: snapshot.expectedMembers.length }
|
||||||
|
: {}),
|
||||||
|
...(snapshot.summary.confirmedCount > 0
|
||||||
|
? { confirmedMemberCount: snapshot.summary.confirmedCount }
|
||||||
|
: {}),
|
||||||
|
...(missingMembers.length > 0 ? { missingMembers } : {}),
|
||||||
|
teamLaunchState: snapshot.teamLaunchState,
|
||||||
|
launchUpdatedAt: snapshot.updatedAt,
|
||||||
|
confirmedCount: snapshot.summary.confirmedCount,
|
||||||
|
pendingCount: snapshot.summary.pendingCount,
|
||||||
|
failedCount: snapshot.summary.failedCount,
|
||||||
|
runtimeAlivePendingCount: snapshot.summary.runtimeAlivePendingCount,
|
||||||
};
|
};
|
||||||
} catch {
|
} catch {
|
||||||
return null;
|
return null;
|
||||||
|
|
@ -591,8 +593,8 @@ async function listTeams(
|
||||||
dropCliProvisionerMembers(memberMap);
|
dropCliProvisionerMembers(memberMap);
|
||||||
|
|
||||||
const members = Array.from(memberMap.values());
|
const members = Array.from(memberMap.values());
|
||||||
const partialLaunchState =
|
const launchStateSummary =
|
||||||
(await readPartialLaunchState(payload.teamsDir, teamName)) ??
|
(await readLaunchState(payload.teamsDir, teamName)) ??
|
||||||
(() => {
|
(() => {
|
||||||
if (
|
if (
|
||||||
!leadSessionId ||
|
!leadSessionId ||
|
||||||
|
|
@ -628,7 +630,7 @@ async function listTeams(
|
||||||
...(projectPathHistory ? { projectPathHistory } : {}),
|
...(projectPathHistory ? { projectPathHistory } : {}),
|
||||||
...(sessionHistory ? { sessionHistory } : {}),
|
...(sessionHistory ? { sessionHistory } : {}),
|
||||||
...(deletedAt ? { deletedAt } : {}),
|
...(deletedAt ? { deletedAt } : {}),
|
||||||
...(partialLaunchState ?? {}),
|
...(launchStateSummary ?? {}),
|
||||||
};
|
};
|
||||||
|
|
||||||
const ms = nowMs() - t0;
|
const ms = nowMs() - t0;
|
||||||
|
|
|
||||||
|
|
@ -1557,11 +1557,16 @@ export const TeamDetailView = ({
|
||||||
>
|
>
|
||||||
<span className="flex items-center gap-1.5 text-xs">
|
<span className="flex items-center gap-1.5 text-xs">
|
||||||
<AlertTriangle size={14} className="shrink-0" />
|
<AlertTriangle size={14} className="shrink-0" />
|
||||||
{currentTeamSummary?.partialLaunchFailure
|
{currentTeamSummary?.teamLaunchState === 'partial_pending'
|
||||||
? currentTeamSummary.missingMembers?.length
|
? currentTeamSummary.runtimeAlivePendingCount &&
|
||||||
? `Last launch failed partway — ${currentTeamSummary.missingMembers.length}/${currentTeamSummary.expectedMemberCount ?? currentTeamSummary.missingMembers.length} teammates did not join`
|
currentTeamSummary.runtimeAlivePendingCount > 0
|
||||||
: 'Last launch failed partway'
|
? `Last launch is still reconciling — ${currentTeamSummary.confirmedCount ?? 0}/${currentTeamSummary.expectedMemberCount ?? currentTeamSummary.memberCount} teammates confirmed alive, ${currentTeamSummary.runtimeAlivePendingCount} runtime${currentTeamSummary.runtimeAlivePendingCount === 1 ? '' : 's'} pending bootstrap`
|
||||||
: 'Team is offline'}
|
: 'Last launch is still reconciling'
|
||||||
|
: currentTeamSummary?.partialLaunchFailure
|
||||||
|
? currentTeamSummary.missingMembers?.length
|
||||||
|
? `Last launch failed partway — ${currentTeamSummary.missingMembers.length}/${currentTeamSummary.expectedMemberCount ?? currentTeamSummary.missingMembers.length} teammates did not join`
|
||||||
|
: 'Last launch failed partway'
|
||||||
|
: 'Team is offline'}
|
||||||
</span>
|
</span>
|
||||||
<Button
|
<Button
|
||||||
variant="ghost"
|
variant="ghost"
|
||||||
|
|
|
||||||
|
|
@ -64,7 +64,13 @@ function generateUniqueName(sourceName: string, existingNames: string[]): string
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type TeamStatus = 'active' | 'idle' | 'provisioning' | 'offline' | 'partial_failure';
|
type TeamStatus =
|
||||||
|
| 'active'
|
||||||
|
| 'idle'
|
||||||
|
| 'provisioning'
|
||||||
|
| 'offline'
|
||||||
|
| 'partial_failure'
|
||||||
|
| 'partial_pending';
|
||||||
|
|
||||||
function getRecentProjects(team: TeamSummary): string[] {
|
function getRecentProjects(team: TeamSummary): string[] {
|
||||||
const history = team.projectPathHistory;
|
const history = team.projectPathHistory;
|
||||||
|
|
@ -174,7 +180,10 @@ function resolveTeamStatus(
|
||||||
) {
|
) {
|
||||||
return 'provisioning';
|
return 'provisioning';
|
||||||
}
|
}
|
||||||
if (team.partialLaunchFailure) {
|
if (team.teamLaunchState === 'partial_pending') {
|
||||||
|
return 'partial_pending';
|
||||||
|
}
|
||||||
|
if (team.partialLaunchFailure || team.teamLaunchState === 'partial_failure') {
|
||||||
return 'partial_failure';
|
return 'partial_failure';
|
||||||
}
|
}
|
||||||
return 'offline';
|
return 'offline';
|
||||||
|
|
@ -217,6 +226,13 @@ const StatusBadge = ({ status }: { status: TeamStatus }): React.JSX.Element => {
|
||||||
Launch failed partway
|
Launch failed partway
|
||||||
</span>
|
</span>
|
||||||
);
|
);
|
||||||
|
case 'partial_pending':
|
||||||
|
return (
|
||||||
|
<span className="inline-flex items-center gap-1 rounded-full bg-amber-500/15 px-2 py-0.5 text-[10px] font-medium text-amber-300">
|
||||||
|
<span className="size-1.5 rounded-full bg-amber-300" />
|
||||||
|
Bootstrap pending
|
||||||
|
</span>
|
||||||
|
);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
@ -381,7 +397,8 @@ export const TeamListView = (): React.JSX.Element => {
|
||||||
getCurrentProvisioningProgressForTeam(provisioningState, t.teamName),
|
getCurrentProvisioningProgressForTeam(provisioningState, t.teamName),
|
||||||
leadActivityByTeam
|
leadActivityByTeam
|
||||||
);
|
);
|
||||||
const isRunning = status !== 'offline' && status !== 'partial_failure';
|
const isRunning =
|
||||||
|
status !== 'offline' && status !== 'partial_failure' && status !== 'partial_pending';
|
||||||
if (filter.selectedStatuses.has('running') && isRunning) return true;
|
if (filter.selectedStatuses.has('running') && isRunning) return true;
|
||||||
if (filter.selectedStatuses.has('offline') && !isRunning) return true;
|
if (filter.selectedStatuses.has('offline') && !isRunning) return true;
|
||||||
return false;
|
return false;
|
||||||
|
|
@ -850,7 +867,9 @@ export const TeamListView = (): React.JSX.Element => {
|
||||||
})()}
|
})()}
|
||||||
</div>
|
</div>
|
||||||
<div className="flex shrink-0 gap-1">
|
<div className="flex shrink-0 gap-1">
|
||||||
{(status === 'offline' || status === 'partial_failure') &&
|
{(status === 'offline' ||
|
||||||
|
status === 'partial_failure' ||
|
||||||
|
status === 'partial_pending') &&
|
||||||
team.projectPath && (
|
team.projectPath && (
|
||||||
<Tooltip>
|
<Tooltip>
|
||||||
<TooltipTrigger asChild>
|
<TooltipTrigger asChild>
|
||||||
|
|
@ -924,7 +943,13 @@ export const TeamListView = (): React.JSX.Element => {
|
||||||
{team.description || 'No description'}
|
{team.description || 'No description'}
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
{team.partialLaunchFailure ? (
|
{team.teamLaunchState === 'partial_pending' ? (
|
||||||
|
<p className="mt-2 text-[11px] text-amber-300">
|
||||||
|
{team.runtimeAlivePendingCount && team.runtimeAlivePendingCount > 0
|
||||||
|
? `Last launch is still reconciling — ${team.confirmedCount ?? 0}/${team.expectedMemberCount ?? team.memberCount} teammates confirmed alive, ${team.runtimeAlivePendingCount} runtime${team.runtimeAlivePendingCount === 1 ? '' : 's'} pending bootstrap.`
|
||||||
|
: 'Last launch is still reconciling.'}
|
||||||
|
</p>
|
||||||
|
) : team.partialLaunchFailure || team.teamLaunchState === 'partial_failure' ? (
|
||||||
<p className="mt-2 text-[11px] text-amber-400">
|
<p className="mt-2 text-[11px] text-amber-400">
|
||||||
{team.missingMembers?.length
|
{team.missingMembers?.length
|
||||||
? `Last launch stopped before ${team.missingMembers.length}/${team.expectedMemberCount ?? team.missingMembers.length} teammate${team.missingMembers.length === 1 ? '' : 's'} joined.`
|
? `Last launch stopped before ${team.missingMembers.length}/${team.expectedMemberCount ?? team.missingMembers.length} teammate${team.missingMembers.length === 1 ? '' : 's'} joined.`
|
||||||
|
|
|
||||||
|
|
@ -17,14 +17,16 @@ interface TeamProvisioningBannerProps {
|
||||||
export const TeamProvisioningBanner = ({
|
export const TeamProvisioningBanner = ({
|
||||||
teamName,
|
teamName,
|
||||||
}: TeamProvisioningBannerProps): React.JSX.Element | null => {
|
}: TeamProvisioningBannerProps): React.JSX.Element | null => {
|
||||||
const { progress, cancelProvisioning, teamMembers, memberSpawnStatuses } = useStore(
|
const { progress, cancelProvisioning, teamMembers, memberSpawnStatuses, memberSpawnSnapshot } =
|
||||||
useShallow((s) => ({
|
useStore(
|
||||||
progress: getCurrentProvisioningProgressForTeam(s, teamName),
|
useShallow((s) => ({
|
||||||
cancelProvisioning: s.cancelProvisioning,
|
progress: getCurrentProvisioningProgressForTeam(s, teamName),
|
||||||
teamMembers: s.selectedTeamData?.members,
|
cancelProvisioning: s.cancelProvisioning,
|
||||||
memberSpawnStatuses: s.memberSpawnStatusesByTeam[teamName],
|
teamMembers: s.selectedTeamData?.members,
|
||||||
}))
|
memberSpawnStatuses: s.memberSpawnStatusesByTeam[teamName],
|
||||||
);
|
memberSpawnSnapshot: s.memberSpawnSnapshotsByTeam[teamName],
|
||||||
|
}))
|
||||||
|
);
|
||||||
const [dismissed, setDismissed] = useState(false);
|
const [dismissed, setDismissed] = useState(false);
|
||||||
const lastActiveStepRef = useRef(-1);
|
const lastActiveStepRef = useRef(-1);
|
||||||
const bannerInstanceKey = useMemo(() => {
|
const bannerInstanceKey = useMemo(() => {
|
||||||
|
|
@ -105,38 +107,49 @@ export const TeamProvisioningBanner = ({
|
||||||
}
|
}
|
||||||
|
|
||||||
const teammates = (teamMembers ?? []).filter((member) => !isLeadMember(member));
|
const teammates = (teamMembers ?? []).filter((member) => !isLeadMember(member));
|
||||||
|
const expectedTeammateCount = memberSpawnSnapshot?.expectedMembers?.length;
|
||||||
|
const fallbackTeammateCount = expectedTeammateCount ?? teammates.length;
|
||||||
|
const snapshotSummary = memberSpawnSnapshot?.summary;
|
||||||
const failedSpawnEntries = Object.entries(memberSpawnStatuses ?? {}).filter(
|
const failedSpawnEntries = Object.entries(memberSpawnStatuses ?? {}).filter(
|
||||||
([, entry]) => entry.launchState === 'failed_to_start'
|
([, entry]) => entry.launchState === 'failed_to_start'
|
||||||
);
|
);
|
||||||
const failedSpawnCount = failedSpawnEntries.length;
|
const failedSpawnCount = snapshotSummary?.failedCount ?? failedSpawnEntries.length;
|
||||||
const heartbeatConfirmedCount = teammates.filter((member) => {
|
const heartbeatConfirmedCount =
|
||||||
const entry = memberSpawnStatuses?.[member.name];
|
snapshotSummary?.confirmedCount ??
|
||||||
return entry?.launchState === 'confirmed_alive';
|
teammates.filter((member) => {
|
||||||
}).length;
|
const entry = memberSpawnStatuses?.[member.name];
|
||||||
const processOnlyAliveCount = teammates.filter((member) => {
|
return entry?.launchState === 'confirmed_alive';
|
||||||
const entry = memberSpawnStatuses?.[member.name];
|
}).length;
|
||||||
return entry?.launchState === 'runtime_pending_bootstrap' && entry.runtimeAlive === true;
|
const processOnlyAliveCount =
|
||||||
}).length;
|
snapshotSummary?.runtimeAlivePendingCount ??
|
||||||
const pendingSpawnCount = teammates.filter((member) => {
|
teammates.filter((member) => {
|
||||||
const entry = memberSpawnStatuses?.[member.name];
|
const entry = memberSpawnStatuses?.[member.name];
|
||||||
return (
|
return entry?.launchState === 'runtime_pending_bootstrap' && entry.runtimeAlive === true;
|
||||||
entry?.launchState === 'starting' ||
|
}).length;
|
||||||
(entry?.launchState === 'runtime_pending_bootstrap' && entry.runtimeAlive !== true)
|
const pendingSpawnCount =
|
||||||
);
|
snapshotSummary?.pendingCount ??
|
||||||
}).length;
|
teammates.filter((member) => {
|
||||||
|
const entry = memberSpawnStatuses?.[member.name];
|
||||||
|
return (
|
||||||
|
entry?.launchState === 'starting' ||
|
||||||
|
(entry?.launchState === 'runtime_pending_bootstrap' && entry.runtimeAlive !== true)
|
||||||
|
);
|
||||||
|
}).length;
|
||||||
const allTeammatesConfirmedAlive =
|
const allTeammatesConfirmedAlive =
|
||||||
teammates.length > 0 && failedSpawnCount === 0 && heartbeatConfirmedCount === teammates.length;
|
fallbackTeammateCount > 0 &&
|
||||||
|
failedSpawnCount === 0 &&
|
||||||
|
heartbeatConfirmedCount === fallbackTeammateCount;
|
||||||
|
|
||||||
if (isReady) {
|
if (isReady) {
|
||||||
const readyMessage =
|
const readyMessage =
|
||||||
failedSpawnCount > 0
|
failedSpawnCount > 0
|
||||||
? `Launch finished with errors — ${failedSpawnCount}/${Math.max(teammates.length, failedSpawnCount)} teammates failed to start`
|
? `Launch finished with errors — ${failedSpawnCount}/${Math.max(fallbackTeammateCount, failedSpawnCount)} teammates failed to start`
|
||||||
: teammates.length === 0
|
: fallbackTeammateCount === 0
|
||||||
? 'Team launched — lead online'
|
? 'Team launched — lead online'
|
||||||
: allTeammatesConfirmedAlive
|
: allTeammatesConfirmedAlive
|
||||||
? `Team launched — all ${teammates.length} teammates confirmed alive`
|
? `Team launched — all ${fallbackTeammateCount} teammates confirmed alive`
|
||||||
: processOnlyAliveCount > 0 || pendingSpawnCount > 0
|
: processOnlyAliveCount > 0 || pendingSpawnCount > 0
|
||||||
? `Team launched — ${heartbeatConfirmedCount}/${teammates.length} teammates confirmed alive${processOnlyAliveCount > 0 ? `, ${processOnlyAliveCount} runtime${processOnlyAliveCount === 1 ? '' : 's'} alive but bootstrap still pending` : ''}${pendingSpawnCount > 0 ? `${processOnlyAliveCount > 0 ? ', ' : ', '}${pendingSpawnCount} still starting` : ''}`
|
? `Team launched — ${heartbeatConfirmedCount}/${fallbackTeammateCount} teammates confirmed alive${processOnlyAliveCount > 0 ? `, ${processOnlyAliveCount} runtime${processOnlyAliveCount === 1 ? '' : 's'} alive but bootstrap still pending` : ''}${pendingSpawnCount > 0 ? `${processOnlyAliveCount > 0 ? ', ' : ', '}${pendingSpawnCount} still starting` : ''}`
|
||||||
: 'Team launched — teammate liveness is still being confirmed';
|
: 'Team launched — teammate liveness is still being confirmed';
|
||||||
|
|
||||||
return (
|
return (
|
||||||
|
|
|
||||||
|
|
@ -168,6 +168,7 @@ export function initializeNotificationListeners(): () => void {
|
||||||
const pendingProjectRefreshTimers = new Map<string, ReturnType<typeof setTimeout>>();
|
const pendingProjectRefreshTimers = new Map<string, ReturnType<typeof setTimeout>>();
|
||||||
let teamRefreshTimers = new Map<string, ReturnType<typeof setTimeout>>();
|
let teamRefreshTimers = new Map<string, ReturnType<typeof setTimeout>>();
|
||||||
let teamPresenceRefreshTimers = new Map<string, ReturnType<typeof setTimeout>>();
|
let teamPresenceRefreshTimers = new Map<string, ReturnType<typeof setTimeout>>();
|
||||||
|
let memberSpawnRefreshTimers = new Map<string, ReturnType<typeof setTimeout>>();
|
||||||
let toolActivityTimers = new Map<string, ReturnType<typeof setTimeout>>();
|
let toolActivityTimers = new Map<string, ReturnType<typeof setTimeout>>();
|
||||||
let inProgressChangePresencePollInFlight = false;
|
let inProgressChangePresencePollInFlight = false;
|
||||||
const inProgressChangePresenceCursorByTeam = new Map<string, number>();
|
const inProgressChangePresenceCursorByTeam = new Map<string, number>();
|
||||||
|
|
@ -178,6 +179,7 @@ export function initializeNotificationListeners(): () => void {
|
||||||
const PROJECT_REFRESH_DEBOUNCE_MS = 300;
|
const PROJECT_REFRESH_DEBOUNCE_MS = 300;
|
||||||
const TEAM_REFRESH_THROTTLE_MS = 800;
|
const TEAM_REFRESH_THROTTLE_MS = 800;
|
||||||
const TEAM_PRESENCE_REFRESH_THROTTLE_MS = 400;
|
const TEAM_PRESENCE_REFRESH_THROTTLE_MS = 400;
|
||||||
|
const TEAM_MEMBER_SPAWN_REFRESH_THROTTLE_MS = 500;
|
||||||
const TEAM_LIST_REFRESH_THROTTLE_MS = 2000;
|
const TEAM_LIST_REFRESH_THROTTLE_MS = 2000;
|
||||||
const GLOBAL_TASKS_REFRESH_THROTTLE_MS = 500;
|
const GLOBAL_TASKS_REFRESH_THROTTLE_MS = 500;
|
||||||
const buildToolActivityTimerKey = (
|
const buildToolActivityTimerKey = (
|
||||||
|
|
@ -919,6 +921,20 @@ export function initializeNotificationListeners(): () => void {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (event.type === 'inbox' || event.type === 'config' || event.type === 'process') {
|
||||||
|
if (!event?.teamName || !isTeamVisibleInAnyPane(event.teamName)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (memberSpawnRefreshTimers.has(event.teamName)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const timer = setTimeout(() => {
|
||||||
|
memberSpawnRefreshTimers.delete(event.teamName);
|
||||||
|
void useStore.getState().fetchMemberSpawnStatuses(event.teamName);
|
||||||
|
}, TEAM_MEMBER_SPAWN_REFRESH_THROTTLE_MS);
|
||||||
|
memberSpawnRefreshTimers.set(event.teamName, timer);
|
||||||
|
}
|
||||||
|
|
||||||
// Live lead-message events: only refresh the visible team detail, not team/task lists.
|
// Live lead-message events: only refresh the visible team detail, not team/task lists.
|
||||||
// This keeps the refresh lightweight and prevents one noisy team from starving another.
|
// This keeps the refresh lightweight and prevents one noisy team from starving another.
|
||||||
if (event.type === 'lead-message') {
|
if (event.type === 'lead-message') {
|
||||||
|
|
@ -998,6 +1014,8 @@ export function initializeNotificationListeners(): () => void {
|
||||||
teamRefreshTimers = new Map();
|
teamRefreshTimers = new Map();
|
||||||
for (const t of teamPresenceRefreshTimers.values()) clearTimeout(t);
|
for (const t of teamPresenceRefreshTimers.values()) clearTimeout(t);
|
||||||
teamPresenceRefreshTimers = new Map();
|
teamPresenceRefreshTimers = new Map();
|
||||||
|
for (const t of memberSpawnRefreshTimers.values()) clearTimeout(t);
|
||||||
|
memberSpawnRefreshTimers = new Map();
|
||||||
for (const t of toolActivityTimers.values()) clearTimeout(t);
|
for (const t of toolActivityTimers.values()) clearTimeout(t);
|
||||||
toolActivityTimers = new Map();
|
toolActivityTimers = new Map();
|
||||||
if (teamListRefreshTimer) {
|
if (teamListRefreshTimer) {
|
||||||
|
|
|
||||||
|
|
@ -139,6 +139,7 @@ import type {
|
||||||
KanbanColumnId,
|
KanbanColumnId,
|
||||||
LeadActivityState,
|
LeadActivityState,
|
||||||
LeadContextUsage,
|
LeadContextUsage,
|
||||||
|
MemberSpawnStatusesSnapshot,
|
||||||
MemberSpawnStatusEntry,
|
MemberSpawnStatusEntry,
|
||||||
SendMessageRequest,
|
SendMessageRequest,
|
||||||
SendMessageResult,
|
SendMessageResult,
|
||||||
|
|
@ -625,6 +626,7 @@ export interface TeamSlice {
|
||||||
toolHistoryByTeam: Record<string, Record<string, ActiveToolCall[]>>;
|
toolHistoryByTeam: Record<string, Record<string, ActiveToolCall[]>>;
|
||||||
/** Per-team per-member spawn statuses during team provisioning/launch. */
|
/** Per-team per-member spawn statuses during team provisioning/launch. */
|
||||||
memberSpawnStatusesByTeam: Record<string, Record<string, MemberSpawnStatusEntry>>;
|
memberSpawnStatusesByTeam: Record<string, Record<string, MemberSpawnStatusEntry>>;
|
||||||
|
memberSpawnSnapshotsByTeam: Record<string, MemberSpawnStatusesSnapshot>;
|
||||||
fetchMemberSpawnStatuses: (teamName: string) => Promise<void>;
|
fetchMemberSpawnStatuses: (teamName: string) => Promise<void>;
|
||||||
provisioningErrorByTeam: Record<string, string | null>;
|
provisioningErrorByTeam: Record<string, string | null>;
|
||||||
clearProvisioningError: (teamName?: string) => void;
|
clearProvisioningError: (teamName?: string) => void;
|
||||||
|
|
@ -910,6 +912,7 @@ export const createTeamSlice: StateCreator<AppState, [], [], TeamSlice> = (set,
|
||||||
finishedVisibleByTeam: {},
|
finishedVisibleByTeam: {},
|
||||||
toolHistoryByTeam: {},
|
toolHistoryByTeam: {},
|
||||||
memberSpawnStatusesByTeam: {},
|
memberSpawnStatusesByTeam: {},
|
||||||
|
memberSpawnSnapshotsByTeam: {},
|
||||||
provisioningErrorByTeam: {},
|
provisioningErrorByTeam: {},
|
||||||
clearProvisioningError: (teamName?: string) =>
|
clearProvisioningError: (teamName?: string) =>
|
||||||
set((state) => {
|
set((state) => {
|
||||||
|
|
@ -971,6 +974,10 @@ export const createTeamSlice: StateCreator<AppState, [], [], TeamSlice> = (set,
|
||||||
...prev.memberSpawnStatusesByTeam,
|
...prev.memberSpawnStatusesByTeam,
|
||||||
[teamName]: snapshot.statuses,
|
[teamName]: snapshot.statuses,
|
||||||
},
|
},
|
||||||
|
memberSpawnSnapshotsByTeam: {
|
||||||
|
...prev.memberSpawnSnapshotsByTeam,
|
||||||
|
[teamName]: snapshot,
|
||||||
|
},
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
} catch {
|
} catch {
|
||||||
|
|
@ -1861,6 +1868,8 @@ export const createTeamSlice: StateCreator<AppState, [], [], TeamSlice> = (set,
|
||||||
delete nextErrors[request.teamName];
|
delete nextErrors[request.teamName];
|
||||||
const nextSpawnStatuses = { ...state.memberSpawnStatusesByTeam };
|
const nextSpawnStatuses = { ...state.memberSpawnStatusesByTeam };
|
||||||
delete nextSpawnStatuses[request.teamName];
|
delete nextSpawnStatuses[request.teamName];
|
||||||
|
const nextSpawnSnapshots = { ...state.memberSpawnSnapshotsByTeam };
|
||||||
|
delete nextSpawnSnapshots[request.teamName];
|
||||||
const nextActiveTools = { ...state.activeToolsByTeam };
|
const nextActiveTools = { ...state.activeToolsByTeam };
|
||||||
delete nextActiveTools[request.teamName];
|
delete nextActiveTools[request.teamName];
|
||||||
const nextFinishedVisible = { ...state.finishedVisibleByTeam };
|
const nextFinishedVisible = { ...state.finishedVisibleByTeam };
|
||||||
|
|
@ -1893,6 +1902,7 @@ export const createTeamSlice: StateCreator<AppState, [], [], TeamSlice> = (set,
|
||||||
provisioningRuns: cleaned,
|
provisioningRuns: cleaned,
|
||||||
provisioningErrorByTeam: nextErrors,
|
provisioningErrorByTeam: nextErrors,
|
||||||
memberSpawnStatusesByTeam: nextSpawnStatuses,
|
memberSpawnStatusesByTeam: nextSpawnStatuses,
|
||||||
|
memberSpawnSnapshotsByTeam: nextSpawnSnapshots,
|
||||||
activeToolsByTeam: nextActiveTools,
|
activeToolsByTeam: nextActiveTools,
|
||||||
finishedVisibleByTeam: nextFinishedVisible,
|
finishedVisibleByTeam: nextFinishedVisible,
|
||||||
toolHistoryByTeam: nextToolHistory,
|
toolHistoryByTeam: nextToolHistory,
|
||||||
|
|
@ -2056,6 +2066,8 @@ export const createTeamSlice: StateCreator<AppState, [], [], TeamSlice> = (set,
|
||||||
delete nextErrors[request.teamName];
|
delete nextErrors[request.teamName];
|
||||||
const nextSpawnStatuses = { ...state.memberSpawnStatusesByTeam };
|
const nextSpawnStatuses = { ...state.memberSpawnStatusesByTeam };
|
||||||
delete nextSpawnStatuses[request.teamName];
|
delete nextSpawnStatuses[request.teamName];
|
||||||
|
const nextSpawnSnapshots = { ...state.memberSpawnSnapshotsByTeam };
|
||||||
|
delete nextSpawnSnapshots[request.teamName];
|
||||||
const nextActiveTools = { ...state.activeToolsByTeam };
|
const nextActiveTools = { ...state.activeToolsByTeam };
|
||||||
delete nextActiveTools[request.teamName];
|
delete nextActiveTools[request.teamName];
|
||||||
const nextFinishedVisible = { ...state.finishedVisibleByTeam };
|
const nextFinishedVisible = { ...state.finishedVisibleByTeam };
|
||||||
|
|
@ -2088,6 +2100,7 @@ export const createTeamSlice: StateCreator<AppState, [], [], TeamSlice> = (set,
|
||||||
provisioningRuns: cleaned,
|
provisioningRuns: cleaned,
|
||||||
provisioningErrorByTeam: nextErrors,
|
provisioningErrorByTeam: nextErrors,
|
||||||
memberSpawnStatusesByTeam: nextSpawnStatuses,
|
memberSpawnStatusesByTeam: nextSpawnStatuses,
|
||||||
|
memberSpawnSnapshotsByTeam: nextSpawnSnapshots,
|
||||||
activeToolsByTeam: nextActiveTools,
|
activeToolsByTeam: nextActiveTools,
|
||||||
finishedVisibleByTeam: nextFinishedVisible,
|
finishedVisibleByTeam: nextFinishedVisible,
|
||||||
toolHistoryByTeam: nextToolHistory,
|
toolHistoryByTeam: nextToolHistory,
|
||||||
|
|
@ -2248,8 +2261,10 @@ export const createTeamSlice: StateCreator<AppState, [], [], TeamSlice> = (set,
|
||||||
: state.ignoredRuntimeRunIds;
|
: state.ignoredRuntimeRunIds;
|
||||||
|
|
||||||
const nextSpawnStatuses = { ...state.memberSpawnStatusesByTeam };
|
const nextSpawnStatuses = { ...state.memberSpawnStatusesByTeam };
|
||||||
|
const nextSpawnSnapshots = { ...state.memberSpawnSnapshotsByTeam };
|
||||||
if (isCanonicalRun) {
|
if (isCanonicalRun) {
|
||||||
delete nextSpawnStatuses[existing.teamName];
|
delete nextSpawnStatuses[existing.teamName];
|
||||||
|
delete nextSpawnSnapshots[existing.teamName];
|
||||||
}
|
}
|
||||||
const nextActiveTools = { ...state.activeToolsByTeam };
|
const nextActiveTools = { ...state.activeToolsByTeam };
|
||||||
const nextFinishedVisible = { ...state.finishedVisibleByTeam };
|
const nextFinishedVisible = { ...state.finishedVisibleByTeam };
|
||||||
|
|
@ -2265,6 +2280,7 @@ export const createTeamSlice: StateCreator<AppState, [], [], TeamSlice> = (set,
|
||||||
currentProvisioningRunIdByTeam: nextCurrentRunIdByTeam,
|
currentProvisioningRunIdByTeam: nextCurrentRunIdByTeam,
|
||||||
currentRuntimeRunIdByTeam: nextRuntimeRunIdByTeam,
|
currentRuntimeRunIdByTeam: nextRuntimeRunIdByTeam,
|
||||||
memberSpawnStatusesByTeam: nextSpawnStatuses,
|
memberSpawnStatusesByTeam: nextSpawnStatuses,
|
||||||
|
memberSpawnSnapshotsByTeam: nextSpawnSnapshots,
|
||||||
activeToolsByTeam: nextActiveTools,
|
activeToolsByTeam: nextActiveTools,
|
||||||
finishedVisibleByTeam: nextFinishedVisible,
|
finishedVisibleByTeam: nextFinishedVisible,
|
||||||
toolHistoryByTeam: nextToolHistory,
|
toolHistoryByTeam: nextToolHistory,
|
||||||
|
|
@ -2387,13 +2403,20 @@ export const createTeamSlice: StateCreator<AppState, [], [], TeamSlice> = (set,
|
||||||
if (isCanonicalRun && TERMINAL_PROVISIONING_STATES.has(progress.state)) {
|
if (isCanonicalRun && TERMINAL_PROVISIONING_STATES.has(progress.state)) {
|
||||||
set((prev) => {
|
set((prev) => {
|
||||||
const next = { ...prev.memberSpawnStatusesByTeam };
|
const next = { ...prev.memberSpawnStatusesByTeam };
|
||||||
|
const nextSnapshots = { ...prev.memberSpawnSnapshotsByTeam };
|
||||||
const currentStatuses = next[progress.teamName];
|
const currentStatuses = next[progress.teamName];
|
||||||
if (!currentStatuses) {
|
if (!currentStatuses) {
|
||||||
return { memberSpawnStatusesByTeam: next };
|
return {
|
||||||
|
memberSpawnStatusesByTeam: next,
|
||||||
|
memberSpawnSnapshotsByTeam: nextSnapshots,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
if (progress.state === 'ready') {
|
if (progress.state === 'ready') {
|
||||||
next[progress.teamName] = currentStatuses;
|
next[progress.teamName] = currentStatuses;
|
||||||
return { memberSpawnStatusesByTeam: next };
|
return {
|
||||||
|
memberSpawnStatusesByTeam: next,
|
||||||
|
memberSpawnSnapshotsByTeam: nextSnapshots,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
const retainedStatuses = Object.fromEntries(
|
const retainedStatuses = Object.fromEntries(
|
||||||
Object.entries(currentStatuses).filter(([, entry]) => entry.status === 'error')
|
Object.entries(currentStatuses).filter(([, entry]) => entry.status === 'error')
|
||||||
|
|
@ -2402,8 +2425,12 @@ export const createTeamSlice: StateCreator<AppState, [], [], TeamSlice> = (set,
|
||||||
next[progress.teamName] = retainedStatuses;
|
next[progress.teamName] = retainedStatuses;
|
||||||
} else {
|
} else {
|
||||||
delete next[progress.teamName];
|
delete next[progress.teamName];
|
||||||
|
delete nextSnapshots[progress.teamName];
|
||||||
}
|
}
|
||||||
return { memberSpawnStatusesByTeam: next };
|
return {
|
||||||
|
memberSpawnStatusesByTeam: next,
|
||||||
|
memberSpawnSnapshotsByTeam: nextSnapshots,
|
||||||
|
};
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -66,6 +66,15 @@ export interface TeamSummary {
|
||||||
confirmedMemberCount?: number;
|
confirmedMemberCount?: number;
|
||||||
/** Missing teammate names from the last partial launch marker. */
|
/** Missing teammate names from the last partial launch marker. */
|
||||||
missingMembers?: string[];
|
missingMembers?: string[];
|
||||||
|
/** Durable aggregate launch state derived from persisted launch-state evidence. */
|
||||||
|
teamLaunchState?: TeamLaunchAggregateState;
|
||||||
|
/** ISO timestamp of the last durable launch-state evaluation. */
|
||||||
|
launchUpdatedAt?: string;
|
||||||
|
/** Durable aggregate teammate counts from launch-state evidence. */
|
||||||
|
confirmedCount?: number;
|
||||||
|
pendingCount?: number;
|
||||||
|
failedCount?: number;
|
||||||
|
runtimeAlivePendingCount?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export type TeamTaskStatus = 'pending' | 'in_progress' | 'completed' | 'deleted';
|
export type TeamTaskStatus = 'pending' | 'in_progress' | 'completed' | 'deleted';
|
||||||
|
|
@ -446,6 +455,8 @@ export type MemberLaunchState =
|
||||||
| 'runtime_pending_bootstrap'
|
| 'runtime_pending_bootstrap'
|
||||||
| 'confirmed_alive'
|
| 'confirmed_alive'
|
||||||
| 'failed_to_start';
|
| 'failed_to_start';
|
||||||
|
export type TeamLaunchAggregateState = 'clean_success' | 'partial_pending' | 'partial_failure';
|
||||||
|
export type PersistedTeamLaunchPhase = 'active' | 'finished' | 'reconciled';
|
||||||
|
|
||||||
export type KanbanColumnId = 'todo' | 'in_progress' | 'done' | 'review' | 'approved';
|
export type KanbanColumnId = 'todo' | 'in_progress' | 'done' | 'review' | 'approved';
|
||||||
|
|
||||||
|
|
@ -575,9 +586,60 @@ export interface LeadContextUsageSnapshot {
|
||||||
runId: string | null;
|
runId: string | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface PersistedTeamLaunchMemberSources {
|
||||||
|
inboxHeartbeat?: boolean;
|
||||||
|
nativeHeartbeat?: boolean;
|
||||||
|
processAlive?: boolean;
|
||||||
|
configRegistered?: boolean;
|
||||||
|
configDrift?: boolean;
|
||||||
|
hardFailureSignal?: boolean;
|
||||||
|
duplicateRespawnBlocked?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PersistedTeamLaunchMemberState {
|
||||||
|
name: string;
|
||||||
|
launchState: MemberLaunchState;
|
||||||
|
agentToolAccepted: boolean;
|
||||||
|
runtimeAlive: boolean;
|
||||||
|
bootstrapConfirmed: boolean;
|
||||||
|
hardFailure: boolean;
|
||||||
|
hardFailureReason?: string;
|
||||||
|
firstSpawnAcceptedAt?: string;
|
||||||
|
lastHeartbeatAt?: string;
|
||||||
|
lastRuntimeAliveAt?: string;
|
||||||
|
lastEvaluatedAt: string;
|
||||||
|
sources?: PersistedTeamLaunchMemberSources;
|
||||||
|
diagnostics?: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PersistedTeamLaunchSummary {
|
||||||
|
confirmedCount: number;
|
||||||
|
pendingCount: number;
|
||||||
|
failedCount: number;
|
||||||
|
runtimeAlivePendingCount: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PersistedTeamLaunchSnapshot {
|
||||||
|
version: 2;
|
||||||
|
teamName: string;
|
||||||
|
updatedAt: string;
|
||||||
|
leadSessionId?: string;
|
||||||
|
launchPhase: PersistedTeamLaunchPhase;
|
||||||
|
expectedMembers: string[];
|
||||||
|
members: Record<string, PersistedTeamLaunchMemberState>;
|
||||||
|
summary: PersistedTeamLaunchSummary;
|
||||||
|
teamLaunchState: TeamLaunchAggregateState;
|
||||||
|
}
|
||||||
|
|
||||||
export interface MemberSpawnStatusesSnapshot {
|
export interface MemberSpawnStatusesSnapshot {
|
||||||
statuses: Record<string, MemberSpawnStatusEntry>;
|
statuses: Record<string, MemberSpawnStatusEntry>;
|
||||||
runId: string | null;
|
runId: string | null;
|
||||||
|
teamLaunchState?: TeamLaunchAggregateState;
|
||||||
|
launchPhase?: PersistedTeamLaunchPhase;
|
||||||
|
expectedMembers?: string[];
|
||||||
|
updatedAt?: string;
|
||||||
|
summary?: PersistedTeamLaunchSummary;
|
||||||
|
source?: 'live' | 'persisted' | 'merged';
|
||||||
}
|
}
|
||||||
|
|
||||||
export type MemberSpawnLivenessSource = 'heartbeat' | 'process';
|
export type MemberSpawnLivenessSource = 'heartbeat' | 'process';
|
||||||
|
|
|
||||||
|
|
@ -5,6 +5,15 @@ export interface ToolSummaryData {
|
||||||
byName: Record<string, number>;
|
byName: Record<string, number>;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type AgentToolDuplicateSkipReason = 'already_running' | 'bootstrap_pending';
|
||||||
|
|
||||||
|
export interface ParsedAgentToolResultStatus {
|
||||||
|
status: 'duplicate_skipped';
|
||||||
|
reason: AgentToolDuplicateSkipReason;
|
||||||
|
name?: string;
|
||||||
|
teamName?: string;
|
||||||
|
}
|
||||||
|
|
||||||
export function buildToolSummary(content: Record<string, unknown>[]): string | undefined {
|
export function buildToolSummary(content: Record<string, unknown>[]): string | undefined {
|
||||||
const counts = new Map<string, number>();
|
const counts = new Map<string, number>();
|
||||||
for (const block of content) {
|
for (const block of content) {
|
||||||
|
|
@ -228,3 +237,40 @@ export function extractToolResultPreview(content: unknown, max = 80): string | u
|
||||||
if (!joined) return undefined;
|
if (!joined) return undefined;
|
||||||
return truncateStr(joined, max);
|
return truncateStr(joined, max);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parse machine-readable Agent tool_result status lines from the raw tool_result content.
|
||||||
|
* Returns null for any non-Agent or non-duplicate result.
|
||||||
|
*/
|
||||||
|
export function parseAgentToolResultStatus(content: unknown): ParsedAgentToolResultStatus | null {
|
||||||
|
const joined = flattenToolResultContent(content).join('\n').trim();
|
||||||
|
if (!joined) return null;
|
||||||
|
|
||||||
|
const fields = new Map<string, string>();
|
||||||
|
for (const rawLine of joined.split(/\r?\n/)) {
|
||||||
|
const line = rawLine.trim();
|
||||||
|
if (!line) continue;
|
||||||
|
const separatorIndex = line.indexOf(':');
|
||||||
|
if (separatorIndex <= 0) continue;
|
||||||
|
const key = line.slice(0, separatorIndex).trim().toLowerCase();
|
||||||
|
const value = line.slice(separatorIndex + 1).trim();
|
||||||
|
if (!value) continue;
|
||||||
|
fields.set(key, value);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (fields.get('status') !== 'duplicate_skipped') {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const reason = fields.get('reason');
|
||||||
|
if (reason !== 'already_running' && reason !== 'bootstrap_pending') {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
status: 'duplicate_skipped',
|
||||||
|
reason,
|
||||||
|
name: fields.get('name'),
|
||||||
|
teamName: fields.get('team_name'),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue