diff --git a/src/main/services/team/TeamBootstrapStateReader.ts b/src/main/services/team/TeamBootstrapStateReader.ts index 9ee6876d..3df80feb 100644 --- a/src/main/services/team/TeamBootstrapStateReader.ts +++ b/src/main/services/team/TeamBootstrapStateReader.ts @@ -64,6 +64,7 @@ type BootstrapStateInspection = { type BootstrapJournalInspection = { warnings?: string[]; issue?: string; + lastPhase?: BootstrapRuntimePhase; }; type BootstrapLockMetadata = { @@ -400,7 +401,7 @@ async function inspectBootstrapJournal(teamName: string): Promise line.length > 0) .slice(-3); - const messages = lines + const records = lines .map((line) => { try { return JSON.parse(line) as RawBootstrapJournalRecord; @@ -408,7 +409,9 @@ async function inspectBootstrapJournal(teamName: string): Promise Boolean(record)) + .filter((record): record is RawBootstrapJournalRecord => Boolean(record)); + + const messages = records .map((record) => { if (record.type === 'phase' && typeof record.phase === 'string') { return `bootstrap phase: ${record.phase}`; @@ -447,7 +450,17 @@ async function inspectBootstrapJournal(teamName: string): Promise => + record.type === 'phase' && typeof record.phase === 'string' + ); + return { + ...(lastPhaseRecord?.phase + ? { lastPhase: lastPhaseRecord.phase as BootstrapRuntimePhase } + : {}), warnings: messages.length > 0 ? [`Recent deterministic bootstrap events: ${messages.join(' | ')}`] @@ -481,6 +494,24 @@ async function readDegradedBootstrapRuntimeState( ].filter((item): item is string => typeof item === 'string' && item.trim().length > 0); const ownerAlive = isProcessAlive(lockMetadata.pid); const now = new Date().toISOString(); + const degradedProjection = + ownerAlive && journalInspection.lastPhase + ? getBootstrapProgressProjection(journalInspection.lastPhase, 0) + : null; + const projectedState = + degradedProjection && + degradedProjection.state !== 'ready' && + degradedProjection.state !== 'failed' && + degradedProjection.state !== 'cancelled' + ? degradedProjection.state + : 'configuring'; + const projectedMessage = + degradedProjection && + degradedProjection.state !== 'ready' && + degradedProjection.state !== 'failed' && + degradedProjection.state !== 'cancelled' + ? `${degradedProjection.message} (degraded recovery)` + : 'Deterministic bootstrap recovery is degraded because persisted bootstrap state is unreadable'; return { teamName, @@ -489,9 +520,9 @@ async function readDegradedBootstrapRuntimeState( progress: { runId: lockMetadata.runId, teamName, - state: ownerAlive ? 'configuring' : 'failed', + state: ownerAlive ? projectedState : 'failed', message: ownerAlive - ? 'Deterministic bootstrap recovery is degraded because persisted bootstrap state is unreadable' + ? projectedMessage : 'Deterministic bootstrap recovery failed because persisted bootstrap state is unreadable and the bootstrap owner is gone', messageSeverity: 'warning', error: ownerAlive diff --git a/test/main/services/team/TeamBootstrapStateReader.test.ts b/test/main/services/team/TeamBootstrapStateReader.test.ts index 31e7f928..3977e6e0 100644 --- a/test/main/services/team/TeamBootstrapStateReader.test.ts +++ b/test/main/services/team/TeamBootstrapStateReader.test.ts @@ -309,13 +309,21 @@ describe('TeamBootstrapStateReader', () => { }), }); hoisted.files.set('/mock/teams/demo/bootstrap-journal.jsonl', { - contents: JSON.stringify({ - ts: 3, - type: 'member', - runId: 'run-lock', - name: 'alice', - action: 'spawn_started', - }), + contents: [ + JSON.stringify({ + ts: 2, + type: 'phase', + runId: 'run-lock', + phase: 'spawning_members', + }), + JSON.stringify({ + ts: 3, + type: 'member', + runId: 'run-lock', + name: 'alice', + action: 'spawn_started', + }), + ].join('\n'), }); await expect(readBootstrapRuntimeState('demo')).resolves.toMatchObject({ @@ -323,11 +331,14 @@ describe('TeamBootstrapStateReader', () => { isAlive: false, runId: 'run-lock', progress: { - state: 'configuring', - message: - 'Deterministic bootstrap recovery is degraded because persisted bootstrap state is unreadable', + state: 'assembling', + message: 'Spawning teammate runtimes (degraded recovery)', messageSeverity: 'warning', pid: 4242, + warnings: [ + 'Persisted deterministic bootstrap state is unreadable because bootstrap-state.json is invalid, truncated, or inaccessible.', + 'Recent deterministic bootstrap events: bootstrap phase: spawning_members | alice: spawn_started', + ], }, });