admin-valentin/agent-ecosystem
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(team): reconcile cleanup bootstrap evidence

Browse source
This commit is contained in:
777genius 2026-05-22 21:01:01 +03:00
parent 511dd5d7a6
commit 1b836530e7
2 changed files with 410 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

98
src/main/services/team/TeamProvisioningService.ts
View file

@ -3597,7 +3597,8 @@ function isBaseAutoClearableLaunchFailureReason(reason?: string): boolean {
isOpenCodeBridgeLaunchFailureReason(reason) ||
isBootstrapMcpResourceReadFailureReason(reason) ||
isBootstrapCheckInTimeoutFailureReason(reason) ||
isBootstrapInstructionPromptFailureReason(reason)
isBootstrapInstructionPromptFailureReason(reason) ||
isLaunchCleanupBootstrapIncompleteFailureReason(reason)
);
}
@ -3618,6 +3619,58 @@ function isBootstrapInstructionPromptFailureReason(reason?: string): boolean {
return typeof reason === 'string' && isBootstrapInstructionPrompt(reason);
}
function isLaunchCleanupBootstrapIncompleteFailureReason(reason?: string): boolean {
const text = reason?.trim();
if (!text) {
return false;
}
if (
text === 'Launch ended before teammate bootstrap completed.' ||
text === 'Deterministic bootstrap failed before teammate check-in.'
) {
return true;
}
return (
text.startsWith('Launch ended before teammate bootstrap completed. ') &&
text.includes('Runtime process was alive after bootstrap failure')
);
}
function isBootstrapMemberEvidenceCurrentForMember(
current: { firstSpawnAcceptedAt?: string; lastEvaluatedAt?: string },
bootstrapMember: Pick<
PersistedTeamLaunchMemberState,
'firstSpawnAcceptedAt' | 'lastHeartbeatAt' | 'lastRuntimeAliveAt' | 'lastEvaluatedAt'
>,
evidenceKind: 'acceptance' | 'confirmation'
): boolean {
const bootstrapFirstSpawnAcceptedMs = Date.parse(bootstrapMember.firstSpawnAcceptedAt ?? '');
const bootstrapLastEvaluatedMs = Date.parse(bootstrapMember.lastEvaluatedAt ?? '');
const hasDurableBootstrapSpawnAcceptedAt =
Number.isFinite(bootstrapFirstSpawnAcceptedMs) &&
(!Number.isFinite(bootstrapLastEvaluatedMs) ||
bootstrapFirstSpawnAcceptedMs <= bootstrapLastEvaluatedMs);
const evidenceAt =
evidenceKind === 'confirmation'
? (bootstrapMember.lastHeartbeatAt ??
bootstrapMember.lastRuntimeAliveAt ??
bootstrapMember.lastEvaluatedAt)
: hasDurableBootstrapSpawnAcceptedAt
? bootstrapMember.firstSpawnAcceptedAt
: bootstrapMember.lastEvaluatedAt;
const evidenceMs = Date.parse(evidenceAt ?? '');
if (!Number.isFinite(evidenceMs)) {
return false;
}
const firstSpawnAcceptedMs = Date.parse(current.firstSpawnAcceptedAt ?? '');
const lastEvaluatedMs = Date.parse(current.lastEvaluatedAt ?? '');
const hasDurableSpawnBoundary =
Number.isFinite(firstSpawnAcceptedMs) &&
(!Number.isFinite(lastEvaluatedMs) || firstSpawnAcceptedMs <= lastEvaluatedMs);
const boundaryMs = hasDurableSpawnBoundary ? firstSpawnAcceptedMs : NaN;
return !Number.isFinite(boundaryMs) || evidenceMs >= boundaryMs;
}
function isTmuxNoServerRunningError(error: unknown): boolean {
const text = error instanceof Error ? error.message : String(error ?? '');
return (
@ -27655,11 +27708,7 @@ export class TeamProvisioningService {
private async overlayPrimaryBootstrapTruthIntoRunStatusesFromBootstrapState(
run: ProvisioningRun
): Promise<void> {
if (
!run.isLaunch ||
!Array.isArray(run.mixedSecondaryLanes) ||
run.mixedSecondaryLanes.length === 0
) {
if (!run.isLaunch) {
return;
}
@ -27684,7 +27733,7 @@ export class TeamProvisioningService {
}
const primaryMemberNames = new Set(
(run.effectiveMembers ?? [])
[...(run.effectiveMembers ?? []), ...(run.expectedMembers ?? []).map((name) => ({ name }))]
.map((member) => member.name?.trim())
.filter((name): name is string => Boolean(name))
);
@ -27703,6 +27752,9 @@ export class TeamProvisioningService {
}
const current =
run.memberSpawnStatuses.get(memberName) ?? createInitialMemberSpawnStatusEntry();
if (!isBootstrapMemberEvidenceCurrentForMember(current, bootstrapMember, 'confirmation')) {
continue;
}
if (current.launchState === 'skipped_for_launch' || current.skippedForLaunch === true) {
continue;
}
@ -27793,6 +27845,9 @@ export class TeamProvisioningService {
if (!current || bootstrapMember?.bootstrapConfirmed !== true) {
continue;
}
if (!isBootstrapMemberEvidenceCurrentForMember(current, bootstrapMember, 'confirmation')) {
continue;
}
if (
current.providerId === 'opencode' ||
isPersistedOpenCodeSecondaryLaneMember(current) ||
@ -30114,9 +30169,25 @@ export class TeamProvisioningService {
if (!current || !bootstrapMember) {
continue;
}
if (
bootstrapMember.bootstrapConfirmed === true &&
!isPersistedOpenCodeSecondaryLaneMember(current) &&
isBootstrapMemberEvidenceCurrentForMember(current, bootstrapMember, 'confirmation')
) {
const currentConfirmed =
current.bootstrapConfirmed === true || current.launchState === 'confirmed_alive';
const failureReason = current.hardFailureReason ?? current.runtimeDiagnostic;
const hasAutoClearableFailure =
(current.launchState === 'failed_to_start' || current.hardFailure === true) &&
isAutoClearableLaunchFailureReason(failureReason);
if (!currentConfirmed || hasAutoClearableFailure) {
return true;
}
}
const bootstrapProvesSpawnAcceptance =
bootstrapMember.agentToolAccepted === true ||
typeof bootstrapMember.firstSpawnAcceptedAt === 'string';
(bootstrapMember.agentToolAccepted === true ||
typeof bootstrapMember.firstSpawnAcceptedAt === 'string') &&
isBootstrapMemberEvidenceCurrentForMember(current, bootstrapMember, 'acceptance');
if (!bootstrapProvesSpawnAcceptance) {
continue;
}
@ -30327,7 +30398,11 @@ export class TeamProvisioningService {
lastEvaluatedAt: now,
};
const isOpenCodeSecondaryLaneMember = isPersistedOpenCodeSecondaryLaneMember(current);
if (bootstrapMember?.agentToolAccepted && !current.agentToolAccepted) {
if (
bootstrapMember?.agentToolAccepted &&
!current.agentToolAccepted &&
isBootstrapMemberEvidenceCurrentForMember(current, bootstrapMember, 'acceptance')
) {
current.agentToolAccepted = true;
current.firstSpawnAcceptedAt =
current.firstSpawnAcceptedAt ?? bootstrapMember.firstSpawnAcceptedAt;
@ -30335,7 +30410,8 @@ export class TeamProvisioningService {
if (
bootstrapMember?.bootstrapConfirmed &&
!current.bootstrapConfirmed &&
!isOpenCodeSecondaryLaneMember
!isOpenCodeSecondaryLaneMember &&
isBootstrapMemberEvidenceCurrentForMember(current, bootstrapMember, 'confirmation')
) {
current.bootstrapConfirmed = true;
current.lastHeartbeatAt = current.lastHeartbeatAt ?? bootstrapMember.lastHeartbeatAt;

324
test/main/services/team/TeamProvisioningService.test.ts
View file

@ -19943,6 +19943,256 @@ describe('TeamProvisioningService', () => {
});
});
it('heals cleanup-finalized launch failures when bootstrap-state confirms an Anthropic primary member', async () => {
allowConsoleLogs();
const teamName = 'zz-unit-cleanup-finalized-bootstrap-state-heals';
const leadSessionId = 'lead-session';
const projectPath = '/Users/test/proj';
const acceptedAt = new Date(Date.now() - 90_000).toISOString();
const bootstrapAt = new Date(Date.now() - 60_000).toISOString();
const cleanupAt = new Date(Date.now() - 30_000).toISOString();
const runtimePid = 45_724;
writeLaunchConfig(teamName, projectPath, leadSessionId, ['jack']);
writeLaunchState(
teamName,
leadSessionId,
{
jack: {
providerId: 'anthropic',
model: 'claude-opus-4-7',
laneId: 'primary',
laneKind: 'primary',
laneOwnerProviderId: 'anthropic',
launchState: 'failed_to_start',
agentToolAccepted: true,
runtimeAlive: false,
runtimePid,
bootstrapConfirmed: false,
hardFailure: true,
hardFailureReason: 'Launch ended before teammate bootstrap completed.',
livenessKind: 'stale_metadata',
runtimeDiagnostic: 'persisted runtime pid is not alive',
runtimeDiagnosticSeverity: 'warning',
firstSpawnAcceptedAt: acceptedAt,
lastEvaluatedAt: cleanupAt,
},
},
{ launchPhase: 'finished', updatedAt: cleanupAt }
);
writeBootstrapState(
teamName,
[
{
name: 'jack',
status: 'bootstrap_confirmed',
lastAttemptAt: Date.parse(acceptedAt),
lastObservedAt: Date.parse(bootstrapAt),
},
],
bootstrapAt
);
const svc = new TeamProvisioningService();
privateHarness(svc).getLiveTeamAgentRuntimeMetadata = vi.fn(
async () =>
new Map([
[
'jack',
{
alive: false,
backendType: 'process',
providerId: 'anthropic',
livenessKind: 'stale_metadata',
pidSource: 'persisted_metadata',
runtimeDiagnostic: 'persisted runtime pid is not alive',
runtimeDiagnosticSeverity: 'warning',
metricsPid: runtimePid,
model: 'claude-opus-4-7',
},
],
])
);
const result = await svc.getMemberSpawnStatuses(teamName);
expect(result.teamLaunchState).toBe('clean_success');
expect(result.statuses.jack).toMatchObject({
status: 'online',
launchState: 'confirmed_alive',
bootstrapConfirmed: true,
hardFailure: false,
error: undefined,
});
expect(result.statuses.jack?.hardFailureReason).toBeUndefined();
});
it('does not heal cleanup-finalized launch failures from stale bootstrap-state confirmation', async () => {
allowConsoleLogs();
const teamName = 'zz-unit-cleanup-finalized-stale-bootstrap-ignored';
const leadSessionId = 'lead-session';
const projectPath = '/Users/test/proj';
const staleBootstrapAt = '2026-05-22T14:14:58.000Z';
const acceptedAt = '2026-05-22T14:15:44.000Z';
const cleanupAt = '2026-05-22T14:16:07.000Z';
const runtimePid = 45_724;
writeLaunchConfig(teamName, projectPath, leadSessionId, ['jack']);
writeLaunchState(
teamName,
leadSessionId,
{
jack: {
providerId: 'anthropic',
model: 'claude-opus-4-7',
laneId: 'primary',
laneKind: 'primary',
laneOwnerProviderId: 'anthropic',
launchState: 'failed_to_start',
agentToolAccepted: true,
runtimeAlive: false,
runtimePid,
bootstrapConfirmed: false,
hardFailure: true,
hardFailureReason: 'Launch ended before teammate bootstrap completed.',
livenessKind: 'stale_metadata',
runtimeDiagnostic: 'persisted runtime pid is not alive',
runtimeDiagnosticSeverity: 'warning',
firstSpawnAcceptedAt: acceptedAt,
lastEvaluatedAt: cleanupAt,
},
},
{ launchPhase: 'finished', updatedAt: cleanupAt }
);
writeBootstrapState(
teamName,
[
{
name: 'jack',
status: 'bootstrap_confirmed',
lastAttemptAt: Date.parse(staleBootstrapAt),
lastObservedAt: Date.parse(staleBootstrapAt),
},
],
staleBootstrapAt
);
const svc = new TeamProvisioningService();
privateHarness(svc).getLiveTeamAgentRuntimeMetadata = vi.fn(
async () =>
new Map([
[
'jack',
{
alive: false,
backendType: 'process',
providerId: 'anthropic',
livenessKind: 'stale_metadata',
pidSource: 'persisted_metadata',
runtimeDiagnostic: 'persisted runtime pid is not alive',
runtimeDiagnosticSeverity: 'warning',
metricsPid: runtimePid,
model: 'claude-opus-4-7',
},
],
])
);
const result = await svc.getMemberSpawnStatuses(teamName);
expect(result.teamLaunchState).toBe('partial_failure');
expect(result.statuses.jack).toMatchObject({
status: 'error',
launchState: 'failed_to_start',
bootstrapConfirmed: false,
hardFailure: true,
hardFailureReason: 'Launch ended before teammate bootstrap completed.',
});
});
it('does not heal cleanup-finalized launch failures from stale member evidence in a rewritten bootstrap-state file', async () => {
allowConsoleLogs();
const teamName = 'zz-unit-cleanup-finalized-stale-member-evidence-ignored';
const leadSessionId = 'lead-session';
const projectPath = '/Users/test/proj';
const staleObservedAt = '2026-05-22T14:14:58.000Z';
const acceptedAt = '2026-05-22T14:15:44.000Z';
const cleanupAt = '2026-05-22T14:16:07.000Z';
const bootstrapFileUpdatedAt = '2026-05-22T14:16:30.000Z';
const runtimePid = 45_724;
writeLaunchConfig(teamName, projectPath, leadSessionId, ['jack']);
writeLaunchState(
teamName,
leadSessionId,
{
jack: {
providerId: 'anthropic',
model: 'claude-opus-4-7',
laneId: 'primary',
laneKind: 'primary',
laneOwnerProviderId: 'anthropic',
launchState: 'failed_to_start',
agentToolAccepted: true,
runtimeAlive: false,
runtimePid,
bootstrapConfirmed: false,
hardFailure: true,
hardFailureReason: 'Launch ended before teammate bootstrap completed.',
livenessKind: 'stale_metadata',
runtimeDiagnostic: 'persisted runtime pid is not alive',
runtimeDiagnosticSeverity: 'warning',
firstSpawnAcceptedAt: acceptedAt,
lastEvaluatedAt: cleanupAt,
},
},
{ launchPhase: 'finished', updatedAt: cleanupAt }
);
writeBootstrapState(
teamName,
[
{
name: 'jack',
status: 'bootstrap_confirmed',
lastObservedAt: Date.parse(staleObservedAt),
},
],
bootstrapFileUpdatedAt
);
const svc = new TeamProvisioningService();
privateHarness(svc).getLiveTeamAgentRuntimeMetadata = vi.fn(
async () =>
new Map([
[
'jack',
{
alive: false,
backendType: 'process',
providerId: 'anthropic',
livenessKind: 'stale_metadata',
pidSource: 'persisted_metadata',
runtimeDiagnostic: 'persisted runtime pid is not alive',
runtimeDiagnosticSeverity: 'warning',
metricsPid: runtimePid,
model: 'claude-opus-4-7',
},
],
])
);
const result = await svc.getMemberSpawnStatuses(teamName);
expect(result.teamLaunchState).toBe('partial_failure');
expect(result.statuses.jack).toMatchObject({
status: 'error',
launchState: 'failed_to_start',
bootstrapConfirmed: false,
hardFailure: true,
hardFailureReason: 'Launch ended before teammate bootstrap completed.',
});
});
it('does not heal terminal bootstrap-state failures from native app-managed proof with mismatched hashes', async () => {
allowConsoleLogs();
const teamName = 'zz-unit-bootstrap-state-native-runtime-proof-hash-mismatch';
@ -25349,6 +25599,7 @@ describe('TeamProvisioningService', () => {
hardFailure: true,
error: 'Teammate was never spawned during launch.',
hardFailureReason: 'Teammate was never spawned during launch.',
firstSpawnAcceptedAt: '2026-04-23T10:00:30.000Z',
}),
],
[
@ -25362,6 +25613,7 @@ describe('TeamProvisioningService', () => {
hardFailure: true,
error: 'Teammate was never spawned during launch.',
hardFailureReason: 'Teammate was never spawned during launch.',
firstSpawnAcceptedAt: '2026-04-23T10:00:30.000Z',
}),
],
]),
@ -25730,7 +25982,7 @@ describe('TeamProvisioningService', () => {
lastObservedAt: Date.parse('2026-04-23T09:59:00.000Z'),
},
],
'2026-04-23T09:59:00.000Z'
'2026-04-23T10:01:00.000Z'
);
const run = createMemberSpawnRun({
teamName,
@ -25749,6 +26001,7 @@ describe('TeamProvisioningService', () => {
hardFailure: true,
error: 'Teammate was never spawned during launch.',
hardFailureReason: 'Teammate was never spawned during launch.',
firstSpawnAcceptedAt: '2026-04-23T10:00:30.000Z',
}),
],
]),
@ -25786,6 +26039,75 @@ describe('TeamProvisioningService', () => {
});
});
it('applies bootstrap-state truth before persisting pure launch cleanup snapshots', async () => {
const teamName = 'pure-anthropic-cleanup-bootstrap-truth';
const acceptedAt = '2026-05-22T14:15:44.000Z';
const bootstrapAt = '2026-05-22T14:15:58.000Z';
const run = createMemberSpawnRun({
teamName,
runId: 'run-pure-anthropic-cleanup-bootstrap-truth',
startedAt: '2026-05-22T14:15:40.000Z',
expectedMembers: ['jack'],
memberSpawnStatuses: new Map([
[
'jack',
createMemberSpawnStatusEntry({
status: 'error',
launchState: 'failed_to_start',
agentToolAccepted: true,
runtimeAlive: false,
bootstrapConfirmed: false,
hardFailure: true,
error: 'Launch ended before teammate bootstrap completed.',
hardFailureReason: 'Launch ended before teammate bootstrap completed.',
firstSpawnAcceptedAt: acceptedAt,
}),
],
]),
});
run.isLaunch = true;
run.request = {
teamName,
cwd: '/Users/test/proj',
providerId: 'anthropic',
model: 'claude-opus-4-7',
members: [],
};
run.effectiveMembers = [
{ name: 'jack', providerId: 'anthropic', model: 'claude-opus-4-7' },
];
fs.mkdirSync(path.join(tempTeamsBase, teamName), { recursive: true });
writeBootstrapState(
teamName,
[
{
name: 'jack',
status: 'bootstrap_confirmed',
lastAttemptAt: Date.parse(acceptedAt),
lastObservedAt: Date.parse(bootstrapAt),
},
],
bootstrapAt
);
const svc = new TeamProvisioningService();
const snapshot = await (
svc as unknown as {
persistLaunchStateSnapshot: (
run: typeof run,
phase: 'finished'
) => Promise<unknown>;
}
).persistLaunchStateSnapshot(run, 'finished');
expect(snapshot).toBeNull();
await expect(fsPromises.readFile(getTeamLaunchStatePath(teamName), 'utf8')).rejects.toMatchObject(
{
code: 'ENOENT',
}
);
});
it('includes queued OpenCode secondary lanes in live spawn statuses during createTeam runs', async () => {
const svc = new TeamProvisioningService();
vi.spyOn(svc as any, 'refreshMemberSpawnStatusesFromLeadInbox').mockResolvedValue(undefined);