import { getTeamsBasePath } from '@main/utils/pathDecoder'; import { createPersistedLaunchSnapshot } from './TeamLaunchStateEvaluator'; import * as fs from 'fs'; import * as path from 'path'; import type { PersistedTeamLaunchMemberState, PersistedTeamLaunchSnapshot, TeamProvisioningProgress, TeamRuntimeState, } from '@shared/types'; const TEAM_BOOTSTRAP_STATE_FILE = 'bootstrap-state.json'; const TEAM_BOOTSTRAP_JOURNAL_FILE = 'bootstrap-journal.jsonl'; const TEAM_BOOTSTRAP_LOCK_DIR = '.bootstrap.lock'; const TEAM_BOOTSTRAP_LOCK_METADATA_FILE = 'metadata.json'; const MAX_BOOTSTRAP_STATE_BYTES = 256 * 1024; const MAX_BOOTSTRAP_JOURNAL_BYTES = 256 * 1024; const MAX_BOOTSTRAP_LOCK_METADATA_BYTES = 64 * 1024; const ACTIVE_BOOTSTRAP_STUCK_CLASSIFICATION_MS = 3 * 60 * 1000; type RawBootstrapMemberState = { name?: unknown; status?: unknown; lastAttemptAt?: unknown; lastObservedAt?: unknown; failureReason?: unknown; }; type RawBootstrapState = { version?: unknown; runId?: unknown; teamName?: unknown; startedAt?: unknown; ownerPid?: unknown; updatedAt?: unknown; phase?: unknown; realTaskSubmissionState?: unknown; members?: unknown; terminal?: unknown; }; type RawBootstrapJournalRecord = | { ts?: unknown; type?: 'phase'; phase?: unknown } | { ts?: unknown; type?: 'lock'; action?: unknown; ownerPid?: unknown; detail?: unknown } | { ts?: unknown; type?: 'member'; name?: unknown; action?: unknown; detail?: unknown } | { ts?: unknown; type?: 'terminal'; status?: unknown; reason?: unknown } | { ts?: unknown; type?: 'real_task'; state?: unknown; detail?: unknown }; type RawBootstrapLockMetadata = { pid?: unknown; runId?: unknown; requestHash?: unknown; ownerStartedAt?: unknown; createdAt?: unknown; nonce?: unknown; }; type BootstrapStateInspection = { raw: RawBootstrapState | null; issue?: string; }; type BootstrapJournalInspection = { warnings?: string[]; issue?: string; }; type BootstrapLockMetadata = { pid: number; runId: string; ownerStartedAt?: number; }; type BootstrapRuntimePhase = | 'validating_spec' | 'loading_existing_state' | 'acquiring_bootstrap_lock' | 'creating_team' | 'spawning_members' | 'auditing_truth' | 'completed' | 'failed' | 'canceled'; function isBootstrapPhaseTerminal(phase: BootstrapRuntimePhase): boolean { return phase === 'completed' || phase === 'failed' || phase === 'canceled'; } function isProcessAlive(pid: number): boolean { if (!Number.isFinite(pid) || pid <= 0) { return false; } try { process.kill(pid, 0); return true; } catch (error) { return (error as NodeJS.ErrnoException | undefined)?.code === 'EPERM'; } } function classifyBootstrapOwnerState(raw: RawBootstrapState): { ownerDead: boolean; stale: boolean; failureReason?: string; } { const phase = typeof raw.phase === 'string' ? (raw.phase as BootstrapRuntimePhase) : null; if (!phase || isBootstrapPhaseTerminal(phase)) { return { ownerDead: false, stale: false }; } const ownerPid = typeof raw.ownerPid === 'number' ? raw.ownerPid : null; if (ownerPid === null || isProcessAlive(ownerPid)) { return { ownerDead: false, stale: false }; } const updatedAtMs = typeof raw.updatedAt === 'number' ? raw.updatedAt : typeof raw.updatedAt === 'string' ? Date.parse(raw.updatedAt) : NaN; const stale = Number.isFinite(updatedAtMs) && Date.now() - updatedAtMs >= ACTIVE_BOOTSTRAP_STUCK_CLASSIFICATION_MS; return { ownerDead: true, stale, failureReason: stale ? `bootstrap owner pid ${ownerPid} is gone and persisted bootstrap state is stale` : `bootstrap owner pid ${ownerPid} is gone before bootstrap reached a terminal state`, }; } async function inspectBootstrapState(teamName: string): Promise { const targetPath = getTeamBootstrapStatePath(teamName); try { const stat = await fs.promises.lstat(targetPath); if (stat.isSymbolicLink()) { return { raw: null, issue: 'Persisted deterministic bootstrap state is unreadable because bootstrap-state.json is a symlink.', }; } if (!stat.isFile()) { return { raw: null, issue: 'Persisted deterministic bootstrap state is unreadable because bootstrap-state.json is not a regular file.', }; } if (stat.size > MAX_BOOTSTRAP_STATE_BYTES) { return { raw: null, issue: 'Persisted deterministic bootstrap state is unreadable because bootstrap-state.json is oversized.', }; } const raw = JSON.parse(await fs.promises.readFile(targetPath, 'utf8')) as RawBootstrapState; if (raw.version !== 1) { return { raw: null, issue: 'Persisted deterministic bootstrap state is unreadable because bootstrap-state.json has an unsupported schema version.', }; } return { raw }; } catch (error) { if ((error as NodeJS.ErrnoException | undefined)?.code === 'ENOENT') { return { raw: null }; } return { raw: null, issue: 'Persisted deterministic bootstrap state is unreadable because bootstrap-state.json is invalid, truncated, or inaccessible.', }; } } async function readRawBootstrapState(teamName: string): Promise { return (await inspectBootstrapState(teamName)).raw; } function getBootstrapProgressProjection( phase: BootstrapRuntimePhase, memberCount: number ): { state: Exclude; message: string } | null { switch (phase) { case 'validating_spec': return { state: 'validating', message: 'Validating deterministic bootstrap spec', }; case 'loading_existing_state': return { state: 'configuring', message: 'Loading existing team state', }; case 'acquiring_bootstrap_lock': return { state: 'configuring', message: 'Acquiring deterministic bootstrap lock', }; case 'creating_team': return { state: 'assembling', message: 'Creating team config', }; case 'spawning_members': return { state: 'assembling', message: memberCount > 0 ? `Spawning teammate runtimes (${memberCount})` : 'Spawning teammate runtimes', }; case 'auditing_truth': return { state: 'finalizing', message: 'Auditing registered teammates and bootstrap truth', }; case 'completed': return { state: 'ready', message: 'Deterministic bootstrap completed', }; case 'failed': return { state: 'failed', message: 'Deterministic bootstrap failed', }; case 'canceled': return { state: 'cancelled', message: 'Deterministic bootstrap cancelled', }; default: return null; } } function toIso(value: unknown, fallback: string): string { if (typeof value === 'string' && value.trim().length > 0) { return value; } if (typeof value === 'number' && Number.isFinite(value) && value > 0) { return new Date(value).toISOString(); } return fallback; } function normalizeBootstrapMemberState( memberName: string, raw: RawBootstrapMemberState, updatedAt: string ): PersistedTeamLaunchMemberState { const status = typeof raw.status === 'string' ? raw.status : 'pending'; const hardFailure = status === 'failed'; const bootstrapConfirmed = status === 'bootstrap_confirmed'; const runtimeAlive = bootstrapConfirmed || status === 'runtime_alive'; const agentToolAccepted = bootstrapConfirmed || runtimeAlive || status === 'registered' || status === 'spawn_started' || hardFailure; return { name: memberName, launchState: hardFailure ? 'failed_to_start' : bootstrapConfirmed ? 'confirmed_alive' : runtimeAlive || agentToolAccepted ? 'runtime_pending_bootstrap' : 'starting', agentToolAccepted, runtimeAlive, bootstrapConfirmed, hardFailure, hardFailureReason: typeof raw.failureReason === 'string' && raw.failureReason.trim().length > 0 ? raw.failureReason.trim() : undefined, firstSpawnAcceptedAt: agentToolAccepted ? toIso(raw.lastAttemptAt, updatedAt) : undefined, lastHeartbeatAt: bootstrapConfirmed ? toIso(raw.lastObservedAt, updatedAt) : undefined, lastRuntimeAliveAt: runtimeAlive ? toIso(raw.lastObservedAt, updatedAt) : undefined, lastEvaluatedAt: toIso(raw.lastObservedAt, updatedAt), sources: { configRegistered: status === 'registered' || status === 'runtime_alive' || status === 'bootstrap_confirmed' || hardFailure, processAlive: runtimeAlive || undefined, hardFailureSignal: hardFailure || undefined, }, diagnostics: hardFailure ? [ typeof raw.failureReason === 'string' && raw.failureReason.trim().length > 0 ? raw.failureReason.trim() : 'deterministic bootstrap failed', ] : runtimeAlive ? bootstrapConfirmed ? ['late heartbeat received'] : ['runtime alive', 'waiting for bootstrap'] : agentToolAccepted ? ['spawn accepted'] : undefined, }; } export function getTeamBootstrapStatePath(teamName: string): string { return path.join(getTeamsBasePath(), teamName, TEAM_BOOTSTRAP_STATE_FILE); } function getTeamBootstrapJournalPath(teamName: string): string { return path.join(getTeamsBasePath(), teamName, TEAM_BOOTSTRAP_JOURNAL_FILE); } function getTeamBootstrapLockMetadataPath(teamName: string): string { return path.join( getTeamsBasePath(), teamName, TEAM_BOOTSTRAP_LOCK_DIR, TEAM_BOOTSTRAP_LOCK_METADATA_FILE ); } async function readBootstrapLockMetadata(teamName: string): Promise { const targetPath = getTeamBootstrapLockMetadataPath(teamName); try { const stat = await fs.promises.lstat(targetPath); if (stat.isSymbolicLink() || !stat.isFile() || stat.size > MAX_BOOTSTRAP_LOCK_METADATA_BYTES) { return null; } const raw = JSON.parse( await fs.promises.readFile(targetPath, 'utf8') ) as RawBootstrapLockMetadata; if ( typeof raw.pid !== 'number' || !Number.isFinite(raw.pid) || raw.pid <= 0 || typeof raw.runId !== 'string' || raw.runId.trim().length === 0 ) { return null; } return { pid: raw.pid, runId: raw.runId.trim(), ownerStartedAt: typeof raw.ownerStartedAt === 'number' && Number.isFinite(raw.ownerStartedAt) ? raw.ownerStartedAt : undefined, }; } catch { return null; } } async function readBootstrapJournalWarnings(teamName: string): Promise { return (await inspectBootstrapJournal(teamName)).warnings; } async function inspectBootstrapJournal(teamName: string): Promise { const targetPath = getTeamBootstrapJournalPath(teamName); try { const stat = await fs.promises.lstat(targetPath); if (stat.isSymbolicLink()) { return { issue: 'Persisted deterministic bootstrap journal is unreadable because bootstrap-journal.jsonl is a symlink.', }; } if (!stat.isFile()) { return { issue: 'Persisted deterministic bootstrap journal is unreadable because bootstrap-journal.jsonl is not a regular file.', }; } if (stat.size > MAX_BOOTSTRAP_JOURNAL_BYTES) { return { issue: 'Persisted deterministic bootstrap journal is unreadable because bootstrap-journal.jsonl is oversized.', }; } const raw = await fs.promises.readFile(targetPath, 'utf8'); const lines = raw .split('\n') .map((line) => line.trim()) .filter((line) => line.length > 0) .slice(-3); const messages = lines .map((line) => { try { return JSON.parse(line) as RawBootstrapJournalRecord; } catch { return null; } }) .filter((record): record is RawBootstrapJournalRecord => Boolean(record)) .map((record) => { if (record.type === 'phase' && typeof record.phase === 'string') { return `bootstrap phase: ${record.phase}`; } if (record.type === 'lock' && typeof record.action === 'string') { const owner = typeof record.ownerPid === 'number' ? ` (pid ${record.ownerPid})` : ''; return `bootstrap lock ${record.action}${owner}`; } if ( record.type === 'member' && typeof record.name === 'string' && typeof record.action === 'string' ) { return typeof record.detail === 'string' && record.detail.trim().length > 0 ? `${record.name}: ${record.action} (${record.detail.trim()})` : `${record.name}: ${record.action}`; } if (record.type === 'terminal' && typeof record.status === 'string') { return typeof record.reason === 'string' && record.reason.trim().length > 0 ? `bootstrap ${record.status}: ${record.reason.trim()}` : `bootstrap ${record.status}`; } if (record.type === 'real_task' && typeof record.state === 'string') { return typeof record.detail === 'string' && record.detail.trim().length > 0 ? `first task ${record.state}: ${record.detail.trim()}` : `first task ${record.state}`; } return null; }) .filter((item): item is string => Boolean(item)); return { warnings: messages.length > 0 ? [`Recent deterministic bootstrap events: ${messages.join(' | ')}`] : undefined, }; } catch (error) { if ((error as NodeJS.ErrnoException | undefined)?.code === 'ENOENT') { return {}; } return { issue: 'Persisted deterministic bootstrap journal is unreadable because bootstrap-journal.jsonl is invalid, truncated, or inaccessible.', }; } } async function readDegradedBootstrapRuntimeState( teamName: string, stateIssue: string ): Promise { const lockMetadata = await readBootstrapLockMetadata(teamName); if (!lockMetadata) { return null; } const journalInspection = await inspectBootstrapJournal(teamName); const warnings = [ stateIssue, journalInspection.issue, ...(journalInspection.warnings ?? []), ].filter((item): item is string => typeof item === 'string' && item.trim().length > 0); const ownerAlive = isProcessAlive(lockMetadata.pid); const now = new Date().toISOString(); return { teamName, isAlive: false, runId: lockMetadata.runId, progress: { runId: lockMetadata.runId, teamName, state: ownerAlive ? 'configuring' : 'failed', message: ownerAlive ? 'Deterministic bootstrap recovery is degraded because persisted bootstrap state is unreadable' : 'Deterministic bootstrap recovery failed because persisted bootstrap state is unreadable and the bootstrap owner is gone', messageSeverity: 'warning', error: ownerAlive ? stateIssue : `${stateIssue} Bootstrap owner pid ${lockMetadata.pid} is not alive.`, warnings: warnings.length > 0 ? warnings : undefined, startedAt: typeof lockMetadata.ownerStartedAt === 'number' && Number.isFinite(lockMetadata.ownerStartedAt) ? new Date(lockMetadata.ownerStartedAt).toISOString() : now, updatedAt: now, pid: lockMetadata.pid, }, }; } export async function readBootstrapLaunchSnapshot( teamName: string ): Promise { const raw = await readRawBootstrapState(teamName); if (!raw) { return null; } try { const updatedAt = toIso(raw.updatedAt, new Date().toISOString()); const rawMembers = Array.isArray(raw.members) ? raw.members : []; const members: Record = {}; const expectedMembers: string[] = []; for (const item of rawMembers) { if (!item || typeof item !== 'object') continue; const rawMember = item as RawBootstrapMemberState; const memberName = typeof rawMember.name === 'string' ? rawMember.name.trim() : ''; if (!memberName || memberName === 'team-lead' || memberName === 'user') continue; expectedMembers.push(memberName); members[memberName] = normalizeBootstrapMemberState(memberName, rawMember, updatedAt); } const terminal = raw.terminal && typeof raw.terminal === 'object' ? (raw.terminal as Record) : null; const terminalStatus = typeof terminal?.status === 'string' ? terminal.status : undefined; const phase = typeof raw.phase === 'string' ? raw.phase : undefined; const ownerState = classifyBootstrapOwnerState(raw); const launchPhase = terminalStatus === 'completed' || terminalStatus === 'partial_success' || terminalStatus === 'failed' || terminalStatus === 'canceled' || ownerState.ownerDead || phase === 'completed' || phase === 'failed' || phase === 'canceled' ? 'finished' : 'active'; if (ownerState.ownerDead) { const diagnostics = ownerState.failureReason ? [ownerState.failureReason] : undefined; for (const memberName of expectedMembers) { const entry = members[memberName]; if ( !entry || entry.launchState === 'confirmed_alive' || entry.launchState === 'failed_to_start' ) { continue; } members[memberName] = { ...entry, launchState: 'failed_to_start', hardFailure: true, hardFailureReason: ownerState.failureReason, diagnostics: diagnostics ?? entry.diagnostics, sources: { ...entry.sources, hardFailureSignal: true, }, }; } } return createPersistedLaunchSnapshot({ teamName: typeof raw.teamName === 'string' && raw.teamName.trim().length > 0 ? raw.teamName.trim() : teamName, expectedMembers, launchPhase, members, updatedAt, }); } catch { return null; } } export async function readBootstrapRealTaskSubmissionState( teamName: string ): Promise<'not_submitted' | 'submitted' | 'unknown' | null> { const raw = await readRawBootstrapState(teamName); if (!raw) { return null; } const state = raw.realTaskSubmissionState; return state === 'not_submitted' || state === 'submitted' || state === 'unknown' ? state : null; } export async function readBootstrapRuntimeState( teamName: string ): Promise { const inspection = await inspectBootstrapState(teamName); const raw = inspection.raw; if (!raw) { return inspection.issue ? readDegradedBootstrapRuntimeState(teamName, inspection.issue) : null; } try { const journalWarnings = await readBootstrapJournalWarnings(teamName); const phase = typeof raw.phase === 'string' ? (raw.phase as BootstrapRuntimePhase) : null; if (!phase) { return null; } const ownerState = classifyBootstrapOwnerState(raw); if (ownerState.ownerDead) { const startedAt = toIso(raw.startedAt, new Date().toISOString()); const updatedAt = toIso(raw.updatedAt, startedAt); return { teamName: typeof raw.teamName === 'string' && raw.teamName.trim().length > 0 ? raw.teamName.trim() : teamName, isAlive: false, runId: typeof raw.runId === 'string' ? raw.runId : null, progress: { runId: typeof raw.runId === 'string' ? raw.runId : teamName, teamName: typeof raw.teamName === 'string' && raw.teamName.trim().length > 0 ? raw.teamName.trim() : teamName, state: 'failed', message: ownerState.stale ? 'Deterministic bootstrap became stuck after owner process exited' : 'Deterministic bootstrap owner exited before bootstrap completed', error: ownerState.failureReason, warnings: journalWarnings, startedAt, updatedAt, ...(typeof raw.ownerPid === 'number' ? { pid: raw.ownerPid } : {}), }, }; } const activePhases: BootstrapRuntimePhase[] = [ 'validating_spec', 'loading_existing_state', 'acquiring_bootstrap_lock', 'creating_team', 'spawning_members', 'auditing_truth', ]; if (!activePhases.includes(phase)) { return null; } const projection = getBootstrapProgressProjection( phase, Array.isArray(raw.members) ? raw.members.length : 0 ); if (!projection) { return null; } const startedAt = toIso(raw.startedAt, new Date().toISOString()); const updatedAt = toIso(raw.updatedAt, startedAt); const runId = typeof raw.runId === 'string' && raw.runId.trim().length > 0 ? raw.runId : null; const pid = typeof raw.ownerPid === 'number' && Number.isFinite(raw.ownerPid) && raw.ownerPid > 0 ? raw.ownerPid : undefined; const progress: TeamProvisioningProgress = { runId: runId ?? `bootstrap:${teamName}`, teamName: typeof raw.teamName === 'string' && raw.teamName.trim().length > 0 ? raw.teamName.trim() : teamName, state: projection.state, message: projection.message, warnings: journalWarnings, startedAt, updatedAt, ...(pid ? { pid } : {}), }; return { teamName: typeof raw.teamName === 'string' && raw.teamName.trim().length > 0 ? raw.teamName.trim() : teamName, isAlive: false, runId, progress, }; } catch { return null; } } export async function clearBootstrapState(teamName: string): Promise { try { await fs.promises.rm(getTeamBootstrapStatePath(teamName), { force: true }); } catch { // best-effort } } export function choosePreferredLaunchSnapshot( bootstrapSnapshot: T | null, launchSnapshot: T | null ): T | null { if (!bootstrapSnapshot) return launchSnapshot; if (!launchSnapshot) return bootstrapSnapshot; const bootstrapMs = Date.parse(bootstrapSnapshot.updatedAt ?? ''); const launchMs = Date.parse(launchSnapshot.updatedAt ?? ''); if (Number.isFinite(bootstrapMs) && Number.isFinite(launchMs)) { return bootstrapMs >= launchMs ? bootstrapSnapshot : launchSnapshot; } return bootstrapSnapshot; }