From 169e34ce7c086d0e7b3185fb6e3f2ad4c4de32a3 Mon Sep 17 00:00:00 2001 From: 777genius Date: Fri, 22 May 2026 13:34:39 +0300 Subject: [PATCH] fix(team): recover process table launch failures --- .../services/team/TeamProvisioningService.ts | 46 ++- .../team/TeamProvisioningService.test.ts | 291 ++++++++++++++++++ 2 files changed, 329 insertions(+), 8 deletions(-) diff --git a/src/main/services/team/TeamProvisioningService.ts b/src/main/services/team/TeamProvisioningService.ts index 29a41b5b..3173c969 100644 --- a/src/main/services/team/TeamProvisioningService.ts +++ b/src/main/services/team/TeamProvisioningService.ts @@ -3513,6 +3513,36 @@ function isRegisteredRuntimeMetadataFailureReason(reason?: string): boolean { return reason?.trim() === 'registered runtime metadata without live process'; } +function isProcessTableUnavailableFailureReason(reason?: string): boolean { + const text = reason?.trim(); + if (!text || !mentionsProcessTableUnavailable(text)) { + return false; + } + return ( + /^process table (?:is )?unavailable$/i.test(text) || + /^runtime pid could not be verified because process table (?:is )?unavailable$/i.test(text) + ); +} + +function stripProcessTableUnavailableDiagnosticSuffix(reason: string): string | null { + const match = /^(.*?);\s*process table (?:is )?unavailable$/i.exec(reason.trim()); + const baseReason = match?.[1]?.trim(); + return baseReason && baseReason.length > 0 ? baseReason : null; +} + +function isBaseAutoClearableLaunchFailureReason(reason?: string): boolean { + return ( + isNeverSpawnedDuringLaunchReason(reason) || + isLaunchGraceWindowFailureReason(reason) || + isConfigRegistrationFailureReason(reason) || + isRegisteredRuntimeMetadataFailureReason(reason) || + isOpenCodeBridgeLaunchFailureReason(reason) || + isBootstrapMcpResourceReadFailureReason(reason) || + isBootstrapCheckInTimeoutFailureReason(reason) || + isBootstrapInstructionPromptFailureReason(reason) + ); +} + function isBootstrapMcpResourceReadFailureReason(reason?: string): boolean { const text = reason?.trim().toLowerCase() ?? ''; return ( @@ -3539,15 +3569,15 @@ function isTmuxNoServerRunningError(error: unknown): boolean { } function isAutoClearableLaunchFailureReason(reason?: string): boolean { + const text = reason?.trim(); + if (!text) { + return false; + } + const baseReason = stripProcessTableUnavailableDiagnosticSuffix(text); return ( - isNeverSpawnedDuringLaunchReason(reason) || - isLaunchGraceWindowFailureReason(reason) || - isConfigRegistrationFailureReason(reason) || - isRegisteredRuntimeMetadataFailureReason(reason) || - isOpenCodeBridgeLaunchFailureReason(reason) || - isBootstrapMcpResourceReadFailureReason(reason) || - isBootstrapCheckInTimeoutFailureReason(reason) || - isBootstrapInstructionPromptFailureReason(reason) + isBaseAutoClearableLaunchFailureReason(text) || + isProcessTableUnavailableFailureReason(text) || + (baseReason != null && isBaseAutoClearableLaunchFailureReason(baseReason)) ); } diff --git a/test/main/services/team/TeamProvisioningService.test.ts b/test/main/services/team/TeamProvisioningService.test.ts index 39aa486d..64201867 100644 --- a/test/main/services/team/TeamProvisioningService.test.ts +++ b/test/main/services/team/TeamProvisioningService.test.ts @@ -583,6 +583,26 @@ function createMemberSpawnStatusEntry( }; } +type TeamProvisioningServicePrivateHarness = { + getLiveTeamAgentRuntimeMetadata: ( + teamName: string + ) => Promise>>; + attachLiveRuntimeMetadataToStatuses: ( + teamName: string, + statuses: Record>, + options?: Record + ) => Promise>>; + applyBootstrapTranscriptEvidenceOverlay: ( + snapshot: ReturnType | null + ) => Promise | null>; +}; + +function privateHarness( + svc: TeamProvisioningService +): TeamProvisioningServicePrivateHarness { + return svc as unknown as TeamProvisioningServicePrivateHarness; +} + function createMemberSpawnRun(params?: { runId?: string; teamName?: string; @@ -19594,6 +19614,95 @@ describe('TeamProvisioningService', () => { }); }); + it('heals process-table unavailable launch-state failures from runtime bootstrap proof', async () => { + allowConsoleLogs(); + const teamName = 'zz-unit-process-table-unavailable-proof-heals'; + const leadSessionId = 'lead-session'; + const projectPath = '/Users/test/proj'; + const acceptedAt = new Date(Date.now() - 90_000).toISOString(); + const proofAt = new Date(Date.now() - 60_000).toISOString(); + const proofToken = 'proof-token-alice'; + const bootstrapRunId = 'run-process-table-unavailable'; + const runtimePid = 35906; + const runtimeEventsPath = path.join(tempTeamsBase, teamName, 'runtime', 'alice.runtime.jsonl'); + const processTableReason = + 'runtime pid could not be verified because process table is unavailable'; + + writeLaunchConfig(teamName, projectPath, leadSessionId, ['alice']); + const configPath = path.join(tempTeamsBase, teamName, 'config.json'); + const config = JSON.parse(fs.readFileSync(configPath, 'utf8')) as { + members: Array>; + }; + config.members = config.members.map((member) => + member.name === 'alice' + ? { + ...member, + agentId: `alice@${teamName}`, + backendType: 'process', + tmuxPaneId: `process:${runtimePid}`, + runtimePid, + bootstrapExpectedAfter: acceptedAt, + bootstrapProofToken: proofToken, + bootstrapRunId, + bootstrapRuntimeEventsPath: runtimeEventsPath, + } + : member + ); + fs.writeFileSync(configPath, JSON.stringify(config), 'utf8'); + const snapshot = createPersistedLaunchSnapshot({ + teamName, + leadSessionId, + launchPhase: 'finished', + expectedMembers: ['alice'], + members: { + alice: { + name: 'alice', + launchState: 'failed_to_start', + agentToolAccepted: true, + runtimeAlive: false, + runtimePid, + runtimeRunId: bootstrapRunId, + bootstrapConfirmed: false, + hardFailure: true, + hardFailureReason: processTableReason, + livenessKind: 'registered_only', + runtimeDiagnostic: processTableReason, + runtimeDiagnosticSeverity: 'warning', + firstSpawnAcceptedAt: acceptedAt, + lastEvaluatedAt: acceptedAt, + }, + }, + }); + fs.mkdirSync(path.dirname(runtimeEventsPath), { recursive: true }); + fs.writeFileSync( + runtimeEventsPath, + `${JSON.stringify({ + version: 1, + type: 'bootstrap_confirmed', + timestamp: proofAt, + pid: runtimePid, + teamName, + agentName: 'alice', + agentId: `alice@${teamName}`, + bootstrapRunId, + source: 'member_briefing_tool_success', + bootstrapProofToken: proofToken, + })}\n`, + 'utf8' + ); + + const svc = new TeamProvisioningService(); + const result = await privateHarness(svc).applyBootstrapTranscriptEvidenceOverlay(snapshot); + + expect(result?.members.alice).toMatchObject({ + launchState: 'confirmed_alive', + bootstrapConfirmed: true, + runtimeAlive: true, + hardFailure: false, + hardFailureReason: undefined, + }); + }); + it('does not heal terminal bootstrap-state failures from native app-managed proof with mismatched hashes', async () => { allowConsoleLogs(); const teamName = 'zz-unit-bootstrap-state-native-runtime-proof-hash-mismatch'; @@ -22418,6 +22527,141 @@ describe('TeamProvisioningService', () => { }); }); + it('clears registered-only stale failure with process-table diagnostic suffix', async () => { + const svc = new TeamProvisioningService(); + const harness = privateHarness(svc); + harness.getLiveTeamAgentRuntimeMetadata = vi.fn( + async () => + new Map([ + [ + 'tom', + { + alive: true, + model: 'gpt-5.4', + livenessKind: 'runtime_process', + runtimeDiagnostic: 'verified runtime process detected', + }, + ], + ]) + ); + + const reason = 'registered runtime metadata without live process; process table unavailable'; + const result = await harness.attachLiveRuntimeMetadataToStatuses('forge-labs-10', { + tom: createMemberSpawnStatusEntry({ + status: 'error', + launchState: 'failed_to_start', + error: reason, + hardFailure: true, + hardFailureReason: reason, + livenessKind: 'registered_only', + runtimeDiagnostic: reason, + }), + }); + + expect(result.tom).toMatchObject({ + status: 'online', + launchState: 'runtime_pending_bootstrap', + runtimeAlive: true, + hardFailure: false, + hardFailureReason: undefined, + error: undefined, + runtimeModel: 'gpt-5.4', + livenessKind: 'runtime_process', + runtimeDiagnostic: 'verified runtime process detected', + livenessSource: 'process', + }); + }); + + it('clears process-table unavailable failure when a verified runtime process appears later', async () => { + const svc = new TeamProvisioningService(); + const harness = privateHarness(svc); + harness.getLiveTeamAgentRuntimeMetadata = vi.fn( + async () => + new Map([ + [ + 'alice', + { + alive: true, + model: 'gpt-5.3-codex', + livenessKind: 'runtime_process', + runtimeDiagnostic: 'verified runtime process detected', + }, + ], + ]) + ); + + const result = await harness.attachLiveRuntimeMetadataToStatuses('signal-ops-10', { + alice: createMemberSpawnStatusEntry({ + status: 'error', + launchState: 'failed_to_start', + error: 'runtime pid could not be verified because process table is unavailable', + hardFailure: true, + hardFailureReason: 'runtime pid could not be verified because process table is unavailable', + livenessKind: 'registered_only', + runtimeDiagnostic: 'runtime pid could not be verified because process table is unavailable', + runtimeDiagnosticSeverity: 'warning', + }), + }); + + expect(result.alice).toMatchObject({ + status: 'online', + launchState: 'runtime_pending_bootstrap', + runtimeAlive: true, + hardFailure: false, + hardFailureReason: undefined, + error: undefined, + runtimeModel: 'gpt-5.3-codex', + livenessKind: 'runtime_process', + runtimeDiagnostic: 'verified runtime process detected', + livenessSource: 'process', + }); + }); + + it('keeps process-table unavailable failure failed when only weak metadata is available', async () => { + const svc = new TeamProvisioningService(); + const harness = privateHarness(svc); + harness.getLiveTeamAgentRuntimeMetadata = vi.fn( + async () => + new Map([ + [ + 'alice', + { + alive: false, + livenessKind: 'registered_only', + runtimeDiagnostic: 'runtime pid could not be verified because process table is unavailable', + runtimeDiagnosticSeverity: 'warning', + }, + ], + ]) + ); + + const result = await harness.attachLiveRuntimeMetadataToStatuses('signal-ops-10', { + alice: createMemberSpawnStatusEntry({ + status: 'error', + launchState: 'failed_to_start', + error: 'runtime pid could not be verified because process table is unavailable', + hardFailure: true, + hardFailureReason: 'runtime pid could not be verified because process table is unavailable', + livenessKind: 'registered_only', + runtimeDiagnostic: 'runtime pid could not be verified because process table is unavailable', + runtimeDiagnosticSeverity: 'warning', + }), + }); + + expect(result.alice).toMatchObject({ + status: 'error', + launchState: 'failed_to_start', + runtimeAlive: false, + hardFailure: true, + hardFailureReason: 'runtime pid could not be verified because process table is unavailable', + error: 'runtime pid could not be verified because process table is unavailable', + livenessKind: 'registered_only', + runtimeDiagnostic: 'runtime pid could not be verified because process table is unavailable', + runtimeDiagnosticSeverity: 'warning', + livenessSource: undefined, + }); + }); + it('does not clear OpenCode bridge launch failure from process-only liveness', async () => { const svc = new TeamProvisioningService(); (svc as any).getLiveTeamAgentRuntimeMetadata = vi.fn( @@ -22464,6 +22708,53 @@ describe('TeamProvisioningService', () => { }); }); + it('does not clear concrete OpenCode bridge failures even when a process is visible', async () => { + const svc = new TeamProvisioningService(); + const harness = privateHarness(svc); + harness.getLiveTeamAgentRuntimeMetadata = vi.fn( + async () => + new Map([ + [ + 'bob', + { + alive: true, + model: 'openrouter/minimax/minimax-m2.5', + livenessKind: 'runtime_process', + providerId: 'opencode', + runtimeDiagnostic: 'OpenCode runtime process detected', + runtimeDiagnosticSeverity: 'info', + }, + ], + ]) + ); + + const reason = + 'OpenCode bridge failed: OpenCode app-managed bootstrap launch requires a fresh capability snapshot before state-changing launch; process table unavailable'; + const result = await harness.attachLiveRuntimeMetadataToStatuses('signal-ops-10', { + bob: createMemberSpawnStatusEntry({ + status: 'error', + launchState: 'failed_to_start', + error: reason, + hardFailure: true, + hardFailureReason: reason, + }), + }); + + expect(result.bob).toMatchObject({ + status: 'error', + launchState: 'failed_to_start', + runtimeAlive: false, + hardFailure: true, + hardFailureReason: reason, + error: reason, + runtimeModel: 'openrouter/minimax/minimax-m2.5', + livenessKind: 'runtime_process', + runtimeDiagnostic: 'OpenCode runtime process detected', + runtimeDiagnosticSeverity: 'info', + livenessSource: undefined, + }); + }); + it('maps suffixed live runtime metadata keys back onto canonical spawn statuses', async () => { const svc = new TeamProvisioningService(); (svc as any).getLiveTeamAgentRuntimeMetadata = vi.fn(