feat(team): improve degraded bootstrap recovery
This commit is contained in:
parent
d484f1d62d
commit
c92bf3a6a5
2 changed files with 56 additions and 14 deletions
|
|
@ -64,6 +64,7 @@ type BootstrapStateInspection = {
|
|||
type BootstrapJournalInspection = {
|
||||
warnings?: string[];
|
||||
issue?: string;
|
||||
lastPhase?: BootstrapRuntimePhase;
|
||||
};
|
||||
|
||||
type BootstrapLockMetadata = {
|
||||
|
|
@ -400,7 +401,7 @@ async function inspectBootstrapJournal(teamName: string): Promise<BootstrapJourn
|
|||
.filter((line) => line.length > 0)
|
||||
.slice(-3);
|
||||
|
||||
const messages = lines
|
||||
const records = lines
|
||||
.map((line) => {
|
||||
try {
|
||||
return JSON.parse(line) as RawBootstrapJournalRecord;
|
||||
|
|
@ -408,7 +409,9 @@ async function inspectBootstrapJournal(teamName: string): Promise<BootstrapJourn
|
|||
return null;
|
||||
}
|
||||
})
|
||||
.filter((record): record is RawBootstrapJournalRecord => Boolean(record))
|
||||
.filter((record): record is RawBootstrapJournalRecord => Boolean(record));
|
||||
|
||||
const messages = records
|
||||
.map((record) => {
|
||||
if (record.type === 'phase' && typeof record.phase === 'string') {
|
||||
return `bootstrap phase: ${record.phase}`;
|
||||
|
|
@ -447,7 +450,17 @@ async function inspectBootstrapJournal(teamName: string): Promise<BootstrapJourn
|
|||
};
|
||||
}
|
||||
|
||||
const lastPhaseRecord = [...records]
|
||||
.reverse()
|
||||
.find(
|
||||
(record): record is Extract<RawBootstrapJournalRecord, { type?: 'phase' }> =>
|
||||
record.type === 'phase' && typeof record.phase === 'string'
|
||||
);
|
||||
|
||||
return {
|
||||
...(lastPhaseRecord?.phase
|
||||
? { lastPhase: lastPhaseRecord.phase as BootstrapRuntimePhase }
|
||||
: {}),
|
||||
warnings:
|
||||
messages.length > 0
|
||||
? [`Recent deterministic bootstrap events: ${messages.join(' | ')}`]
|
||||
|
|
@ -481,6 +494,24 @@ async function readDegradedBootstrapRuntimeState(
|
|||
].filter((item): item is string => typeof item === 'string' && item.trim().length > 0);
|
||||
const ownerAlive = isProcessAlive(lockMetadata.pid);
|
||||
const now = new Date().toISOString();
|
||||
const degradedProjection =
|
||||
ownerAlive && journalInspection.lastPhase
|
||||
? getBootstrapProgressProjection(journalInspection.lastPhase, 0)
|
||||
: null;
|
||||
const projectedState =
|
||||
degradedProjection &&
|
||||
degradedProjection.state !== 'ready' &&
|
||||
degradedProjection.state !== 'failed' &&
|
||||
degradedProjection.state !== 'cancelled'
|
||||
? degradedProjection.state
|
||||
: 'configuring';
|
||||
const projectedMessage =
|
||||
degradedProjection &&
|
||||
degradedProjection.state !== 'ready' &&
|
||||
degradedProjection.state !== 'failed' &&
|
||||
degradedProjection.state !== 'cancelled'
|
||||
? `${degradedProjection.message} (degraded recovery)`
|
||||
: 'Deterministic bootstrap recovery is degraded because persisted bootstrap state is unreadable';
|
||||
|
||||
return {
|
||||
teamName,
|
||||
|
|
@ -489,9 +520,9 @@ async function readDegradedBootstrapRuntimeState(
|
|||
progress: {
|
||||
runId: lockMetadata.runId,
|
||||
teamName,
|
||||
state: ownerAlive ? 'configuring' : 'failed',
|
||||
state: ownerAlive ? projectedState : 'failed',
|
||||
message: ownerAlive
|
||||
? 'Deterministic bootstrap recovery is degraded because persisted bootstrap state is unreadable'
|
||||
? projectedMessage
|
||||
: 'Deterministic bootstrap recovery failed because persisted bootstrap state is unreadable and the bootstrap owner is gone',
|
||||
messageSeverity: 'warning',
|
||||
error: ownerAlive
|
||||
|
|
|
|||
|
|
@ -309,13 +309,21 @@ describe('TeamBootstrapStateReader', () => {
|
|||
}),
|
||||
});
|
||||
hoisted.files.set('/mock/teams/demo/bootstrap-journal.jsonl', {
|
||||
contents: JSON.stringify({
|
||||
ts: 3,
|
||||
type: 'member',
|
||||
runId: 'run-lock',
|
||||
name: 'alice',
|
||||
action: 'spawn_started',
|
||||
}),
|
||||
contents: [
|
||||
JSON.stringify({
|
||||
ts: 2,
|
||||
type: 'phase',
|
||||
runId: 'run-lock',
|
||||
phase: 'spawning_members',
|
||||
}),
|
||||
JSON.stringify({
|
||||
ts: 3,
|
||||
type: 'member',
|
||||
runId: 'run-lock',
|
||||
name: 'alice',
|
||||
action: 'spawn_started',
|
||||
}),
|
||||
].join('\n'),
|
||||
});
|
||||
|
||||
await expect(readBootstrapRuntimeState('demo')).resolves.toMatchObject({
|
||||
|
|
@ -323,11 +331,14 @@ describe('TeamBootstrapStateReader', () => {
|
|||
isAlive: false,
|
||||
runId: 'run-lock',
|
||||
progress: {
|
||||
state: 'configuring',
|
||||
message:
|
||||
'Deterministic bootstrap recovery is degraded because persisted bootstrap state is unreadable',
|
||||
state: 'assembling',
|
||||
message: 'Spawning teammate runtimes (degraded recovery)',
|
||||
messageSeverity: 'warning',
|
||||
pid: 4242,
|
||||
warnings: [
|
||||
'Persisted deterministic bootstrap state is unreadable because bootstrap-state.json is invalid, truncated, or inaccessible.',
|
||||
'Recent deterministic bootstrap events: bootstrap phase: spawning_members | alice: spawn_started',
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue