fix(team): recover process table launch failures
This commit is contained in:
777genius
parent
3eaa9889ca
commit
169e34ce7c
2 changed files with 329 additions and 8 deletions
|
|
@ -3513,6 +3513,36 @@ function isRegisteredRuntimeMetadataFailureReason(reason?: string): boolean {
|
|||
return reason?.trim() === 'registered runtime metadata without live process';
|
||||
}
|
||||
|
||||
function isProcessTableUnavailableFailureReason(reason?: string): boolean {
|
||||
const text = reason?.trim();
|
||||
if (!text || !mentionsProcessTableUnavailable(text)) {
|
||||
return false;
|
||||
}
|
||||
return (
|
||||
/^process table (?:is )?unavailable$/i.test(text) ||
|
||||
/^runtime pid could not be verified because process table (?:is )?unavailable$/i.test(text)
|
||||
);
|
||||
}
|
||||
|
||||
function stripProcessTableUnavailableDiagnosticSuffix(reason: string): string | null {
|
||||
const match = /^(.*?);\s*process table (?:is )?unavailable$/i.exec(reason.trim());
|
||||
const baseReason = match?.[1]?.trim();
|
||||
return baseReason && baseReason.length > 0 ? baseReason : null;
|
||||
}
|
||||
|
||||
function isBaseAutoClearableLaunchFailureReason(reason?: string): boolean {
|
||||
return (
|
||||
isNeverSpawnedDuringLaunchReason(reason) ||
|
||||
isLaunchGraceWindowFailureReason(reason) ||
|
||||
isConfigRegistrationFailureReason(reason) ||
|
||||
isRegisteredRuntimeMetadataFailureReason(reason) ||
|
||||
isOpenCodeBridgeLaunchFailureReason(reason) ||
|
||||
isBootstrapMcpResourceReadFailureReason(reason) ||
|
||||
isBootstrapCheckInTimeoutFailureReason(reason) ||
|
||||
isBootstrapInstructionPromptFailureReason(reason)
|
||||
);
|
||||
}
|
||||
|
||||
function isBootstrapMcpResourceReadFailureReason(reason?: string): boolean {
|
||||
const text = reason?.trim().toLowerCase() ?? '';
|
||||
return (
|
||||
|
|
@ -3539,15 +3569,15 @@ function isTmuxNoServerRunningError(error: unknown): boolean {
|
|||
}
|
||||
|
||||
function isAutoClearableLaunchFailureReason(reason?: string): boolean {
|
||||
const text = reason?.trim();
|
||||
if (!text) {
|
||||
return false;
|
||||
}
|
||||
const baseReason = stripProcessTableUnavailableDiagnosticSuffix(text);
|
||||
return (
|
||||
isNeverSpawnedDuringLaunchReason(reason) ||
|
||||
isLaunchGraceWindowFailureReason(reason) ||
|
||||
isConfigRegistrationFailureReason(reason) ||
|
||||
isRegisteredRuntimeMetadataFailureReason(reason) ||
|
||||
isOpenCodeBridgeLaunchFailureReason(reason) ||
|
||||
isBootstrapMcpResourceReadFailureReason(reason) ||
|
||||
isBootstrapCheckInTimeoutFailureReason(reason) ||
|
||||
isBootstrapInstructionPromptFailureReason(reason)
|
||||
isBaseAutoClearableLaunchFailureReason(text) ||
|
||||
isProcessTableUnavailableFailureReason(text) ||
|
||||
(baseReason != null && isBaseAutoClearableLaunchFailureReason(baseReason))
|
||||
);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -583,6 +583,26 @@ function createMemberSpawnStatusEntry(
|
|||
};
|
||||
}
|
||||
|
||||
type TeamProvisioningServicePrivateHarness = {
|
||||
getLiveTeamAgentRuntimeMetadata: (
|
||||
teamName: string
|
||||
) => Promise<Map<string, Record<string, unknown>>>;
|
||||
attachLiveRuntimeMetadataToStatuses: (
|
||||
teamName: string,
|
||||
statuses: Record<string, Record<string, unknown>>,
|
||||
options?: Record<string, unknown>
|
||||
) => Promise<Record<string, Record<string, unknown>>>;
|
||||
applyBootstrapTranscriptEvidenceOverlay: (
|
||||
snapshot: ReturnType<typeof createPersistedLaunchSnapshot> | null
|
||||
) => Promise<ReturnType<typeof createPersistedLaunchSnapshot> | null>;
|
||||
};
|
||||
|
||||
function privateHarness(
|
||||
svc: TeamProvisioningService
|
||||
): TeamProvisioningServicePrivateHarness {
|
||||
return svc as unknown as TeamProvisioningServicePrivateHarness;
|
||||
}
|
||||
|
||||
function createMemberSpawnRun(params?: {
|
||||
runId?: string;
|
||||
teamName?: string;
|
||||
|
|
@ -19594,6 +19614,95 @@ describe('TeamProvisioningService', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('heals process-table unavailable launch-state failures from runtime bootstrap proof', async () => {
|
||||
allowConsoleLogs();
|
||||
const teamName = 'zz-unit-process-table-unavailable-proof-heals';
|
||||
const leadSessionId = 'lead-session';
|
||||
const projectPath = '/Users/test/proj';
|
||||
const acceptedAt = new Date(Date.now() - 90_000).toISOString();
|
||||
const proofAt = new Date(Date.now() - 60_000).toISOString();
|
||||
const proofToken = 'proof-token-alice';
|
||||
const bootstrapRunId = 'run-process-table-unavailable';
|
||||
const runtimePid = 35906;
|
||||
const runtimeEventsPath = path.join(tempTeamsBase, teamName, 'runtime', 'alice.runtime.jsonl');
|
||||
const processTableReason =
|
||||
'runtime pid could not be verified because process table is unavailable';
|
||||
|
||||
writeLaunchConfig(teamName, projectPath, leadSessionId, ['alice']);
|
||||
const configPath = path.join(tempTeamsBase, teamName, 'config.json');
|
||||
const config = JSON.parse(fs.readFileSync(configPath, 'utf8')) as {
|
||||
members: Array<Record<string, unknown>>;
|
||||
};
|
||||
config.members = config.members.map((member) =>
|
||||
member.name === 'alice'
|
||||
? {
|
||||
...member,
|
||||
agentId: `alice@${teamName}`,
|
||||
backendType: 'process',
|
||||
tmuxPaneId: `process:${runtimePid}`,
|
||||
runtimePid,
|
||||
bootstrapExpectedAfter: acceptedAt,
|
||||
bootstrapProofToken: proofToken,
|
||||
bootstrapRunId,
|
||||
bootstrapRuntimeEventsPath: runtimeEventsPath,
|
||||
}
|
||||
: member
|
||||
);
|
||||
fs.writeFileSync(configPath, JSON.stringify(config), 'utf8');
|
||||
const snapshot = createPersistedLaunchSnapshot({
|
||||
teamName,
|
||||
leadSessionId,
|
||||
launchPhase: 'finished',
|
||||
expectedMembers: ['alice'],
|
||||
members: {
|
||||
alice: {
|
||||
name: 'alice',
|
||||
launchState: 'failed_to_start',
|
||||
agentToolAccepted: true,
|
||||
runtimeAlive: false,
|
||||
runtimePid,
|
||||
runtimeRunId: bootstrapRunId,
|
||||
bootstrapConfirmed: false,
|
||||
hardFailure: true,
|
||||
hardFailureReason: processTableReason,
|
||||
livenessKind: 'registered_only',
|
||||
runtimeDiagnostic: processTableReason,
|
||||
runtimeDiagnosticSeverity: 'warning',
|
||||
firstSpawnAcceptedAt: acceptedAt,
|
||||
lastEvaluatedAt: acceptedAt,
|
||||
},
|
||||
},
|
||||
});
|
||||
fs.mkdirSync(path.dirname(runtimeEventsPath), { recursive: true });
|
||||
fs.writeFileSync(
|
||||
runtimeEventsPath,
|
||||
`${JSON.stringify({
|
||||
version: 1,
|
||||
type: 'bootstrap_confirmed',
|
||||
timestamp: proofAt,
|
||||
pid: runtimePid,
|
||||
teamName,
|
||||
agentName: 'alice',
|
||||
agentId: `alice@${teamName}`,
|
||||
bootstrapRunId,
|
||||
source: 'member_briefing_tool_success',
|
||||
bootstrapProofToken: proofToken,
|
||||
})}\n`,
|
||||
'utf8'
|
||||
);
|
||||
|
||||
const svc = new TeamProvisioningService();
|
||||
const result = await privateHarness(svc).applyBootstrapTranscriptEvidenceOverlay(snapshot);
|
||||
|
||||
expect(result?.members.alice).toMatchObject({
|
||||
launchState: 'confirmed_alive',
|
||||
bootstrapConfirmed: true,
|
||||
runtimeAlive: true,
|
||||
hardFailure: false,
|
||||
hardFailureReason: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
it('does not heal terminal bootstrap-state failures from native app-managed proof with mismatched hashes', async () => {
|
||||
allowConsoleLogs();
|
||||
const teamName = 'zz-unit-bootstrap-state-native-runtime-proof-hash-mismatch';
|
||||
|
|
@ -22418,6 +22527,141 @@ describe('TeamProvisioningService', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('clears registered-only stale failure with process-table diagnostic suffix', async () => {
|
||||
const svc = new TeamProvisioningService();
|
||||
const harness = privateHarness(svc);
|
||||
harness.getLiveTeamAgentRuntimeMetadata = vi.fn(
|
||||
async () =>
|
||||
new Map([
|
||||
[
|
||||
'tom',
|
||||
{
|
||||
alive: true,
|
||||
model: 'gpt-5.4',
|
||||
livenessKind: 'runtime_process',
|
||||
runtimeDiagnostic: 'verified runtime process detected',
|
||||
},
|
||||
],
|
||||
])
|
||||
);
|
||||
|
||||
const reason = 'registered runtime metadata without live process; process table unavailable';
|
||||
const result = await harness.attachLiveRuntimeMetadataToStatuses('forge-labs-10', {
|
||||
tom: createMemberSpawnStatusEntry({
|
||||
status: 'error',
|
||||
launchState: 'failed_to_start',
|
||||
error: reason,
|
||||
hardFailure: true,
|
||||
hardFailureReason: reason,
|
||||
livenessKind: 'registered_only',
|
||||
runtimeDiagnostic: reason,
|
||||
}),
|
||||
});
|
||||
|
||||
expect(result.tom).toMatchObject({
|
||||
status: 'online',
|
||||
launchState: 'runtime_pending_bootstrap',
|
||||
runtimeAlive: true,
|
||||
hardFailure: false,
|
||||
hardFailureReason: undefined,
|
||||
error: undefined,
|
||||
runtimeModel: 'gpt-5.4',
|
||||
livenessKind: 'runtime_process',
|
||||
runtimeDiagnostic: 'verified runtime process detected',
|
||||
livenessSource: 'process',
|
||||
});
|
||||
});
|
||||
|
||||
it('clears process-table unavailable failure when a verified runtime process appears later', async () => {
|
||||
const svc = new TeamProvisioningService();
|
||||
const harness = privateHarness(svc);
|
||||
harness.getLiveTeamAgentRuntimeMetadata = vi.fn(
|
||||
async () =>
|
||||
new Map([
|
||||
[
|
||||
'alice',
|
||||
{
|
||||
alive: true,
|
||||
model: 'gpt-5.3-codex',
|
||||
livenessKind: 'runtime_process',
|
||||
runtimeDiagnostic: 'verified runtime process detected',
|
||||
},
|
||||
],
|
||||
])
|
||||
);
|
||||
|
||||
const result = await harness.attachLiveRuntimeMetadataToStatuses('signal-ops-10', {
|
||||
alice: createMemberSpawnStatusEntry({
|
||||
status: 'error',
|
||||
launchState: 'failed_to_start',
|
||||
error: 'runtime pid could not be verified because process table is unavailable',
|
||||
hardFailure: true,
|
||||
hardFailureReason: 'runtime pid could not be verified because process table is unavailable',
|
||||
livenessKind: 'registered_only',
|
||||
runtimeDiagnostic: 'runtime pid could not be verified because process table is unavailable',
|
||||
runtimeDiagnosticSeverity: 'warning',
|
||||
}),
|
||||
});
|
||||
|
||||
expect(result.alice).toMatchObject({
|
||||
status: 'online',
|
||||
launchState: 'runtime_pending_bootstrap',
|
||||
runtimeAlive: true,
|
||||
hardFailure: false,
|
||||
hardFailureReason: undefined,
|
||||
error: undefined,
|
||||
runtimeModel: 'gpt-5.3-codex',
|
||||
livenessKind: 'runtime_process',
|
||||
runtimeDiagnostic: 'verified runtime process detected',
|
||||
livenessSource: 'process',
|
||||
});
|
||||
});
|
||||
|
||||
it('keeps process-table unavailable failure failed when only weak metadata is available', async () => {
|
||||
const svc = new TeamProvisioningService();
|
||||
const harness = privateHarness(svc);
|
||||
harness.getLiveTeamAgentRuntimeMetadata = vi.fn(
|
||||
async () =>
|
||||
new Map([
|
||||
[
|
||||
'alice',
|
||||
{
|
||||
alive: false,
|
||||
livenessKind: 'registered_only',
|
||||
runtimeDiagnostic: 'runtime pid could not be verified because process table is unavailable',
|
||||
runtimeDiagnosticSeverity: 'warning',
|
||||
},
|
||||
],
|
||||
])
|
||||
);
|
||||
|
||||
const result = await harness.attachLiveRuntimeMetadataToStatuses('signal-ops-10', {
|
||||
alice: createMemberSpawnStatusEntry({
|
||||
status: 'error',
|
||||
launchState: 'failed_to_start',
|
||||
error: 'runtime pid could not be verified because process table is unavailable',
|
||||
hardFailure: true,
|
||||
hardFailureReason: 'runtime pid could not be verified because process table is unavailable',
|
||||
livenessKind: 'registered_only',
|
||||
runtimeDiagnostic: 'runtime pid could not be verified because process table is unavailable',
|
||||
runtimeDiagnosticSeverity: 'warning',
|
||||
}),
|
||||
});
|
||||
|
||||
expect(result.alice).toMatchObject({
|
||||
status: 'error',
|
||||
launchState: 'failed_to_start',
|
||||
runtimeAlive: false,
|
||||
hardFailure: true,
|
||||
hardFailureReason: 'runtime pid could not be verified because process table is unavailable',
|
||||
error: 'runtime pid could not be verified because process table is unavailable',
|
||||
livenessKind: 'registered_only',
|
||||
runtimeDiagnostic: 'runtime pid could not be verified because process table is unavailable',
|
||||
runtimeDiagnosticSeverity: 'warning',
|
||||
livenessSource: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
it('does not clear OpenCode bridge launch failure from process-only liveness', async () => {
|
||||
const svc = new TeamProvisioningService();
|
||||
(svc as any).getLiveTeamAgentRuntimeMetadata = vi.fn(
|
||||
|
|
@ -22464,6 +22708,53 @@ describe('TeamProvisioningService', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('does not clear concrete OpenCode bridge failures even when a process is visible', async () => {
|
||||
const svc = new TeamProvisioningService();
|
||||
const harness = privateHarness(svc);
|
||||
harness.getLiveTeamAgentRuntimeMetadata = vi.fn(
|
||||
async () =>
|
||||
new Map([
|
||||
[
|
||||
'bob',
|
||||
{
|
||||
alive: true,
|
||||
model: 'openrouter/minimax/minimax-m2.5',
|
||||
livenessKind: 'runtime_process',
|
||||
providerId: 'opencode',
|
||||
runtimeDiagnostic: 'OpenCode runtime process detected',
|
||||
runtimeDiagnosticSeverity: 'info',
|
||||
},
|
||||
],
|
||||
])
|
||||
);
|
||||
|
||||
const reason =
|
||||
'OpenCode bridge failed: OpenCode app-managed bootstrap launch requires a fresh capability snapshot before state-changing launch; process table unavailable';
|
||||
const result = await harness.attachLiveRuntimeMetadataToStatuses('signal-ops-10', {
|
||||
bob: createMemberSpawnStatusEntry({
|
||||
status: 'error',
|
||||
launchState: 'failed_to_start',
|
||||
error: reason,
|
||||
hardFailure: true,
|
||||
hardFailureReason: reason,
|
||||
}),
|
||||
});
|
||||
|
||||
expect(result.bob).toMatchObject({
|
||||
status: 'error',
|
||||
launchState: 'failed_to_start',
|
||||
runtimeAlive: false,
|
||||
hardFailure: true,
|
||||
hardFailureReason: reason,
|
||||
error: reason,
|
||||
runtimeModel: 'openrouter/minimax/minimax-m2.5',
|
||||
livenessKind: 'runtime_process',
|
||||
runtimeDiagnostic: 'OpenCode runtime process detected',
|
||||
runtimeDiagnosticSeverity: 'info',
|
||||
livenessSource: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
it('maps suffixed live runtime metadata keys back onto canonical spawn statuses', async () => {
|
||||
const svc = new TeamProvisioningService();
|
||||
(svc as any).getLiveTeamAgentRuntimeMetadata = vi.fn(
|
||||
|
|
|
|||
Loading…
Reference in a new issue