fix(team): degrade failed opencode prelaunch lanes
This commit is contained in:
parent
fa8bbcbb38
commit
f4e4ecca2e
4 changed files with 162 additions and 1 deletions
|
|
@ -1324,6 +1324,47 @@ function isNeverSpawnedDuringLaunchReason(reason?: string): boolean {
|
||||||
return reason?.trim() === 'Teammate was never spawned during launch.';
|
return reason?.trim() === 'Teammate was never spawned during launch.';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function collectRuntimeLaunchFailureDiagnostics(
|
||||||
|
result: TeamRuntimeLaunchResult,
|
||||||
|
memberName: string
|
||||||
|
): string[] {
|
||||||
|
const member = result.members[memberName];
|
||||||
|
return [...(member?.diagnostics ?? []), member?.hardFailureReason, ...result.diagnostics].filter(
|
||||||
|
(value): value is string => typeof value === 'string' && value.trim().length > 0
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function isReconciliableOpenCodeUnknownOutcome(diagnostics: readonly string[]): boolean {
|
||||||
|
return diagnostics.some((diagnostic) =>
|
||||||
|
/outcome must be reconciled before retry/i.test(diagnostic)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function isDefinitiveOpenCodePreLaunchFailure(
|
||||||
|
result: TeamRuntimeLaunchResult,
|
||||||
|
memberName: string
|
||||||
|
): boolean {
|
||||||
|
const member = result.members[memberName];
|
||||||
|
if (!member) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
const hardFailed = member.launchState === 'failed_to_start' || member.hardFailure === true;
|
||||||
|
if (!hardFailed) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
const runtimeMaterialized =
|
||||||
|
member.agentToolAccepted ||
|
||||||
|
member.runtimeAlive ||
|
||||||
|
member.bootstrapConfirmed ||
|
||||||
|
typeof member.sessionId === 'string';
|
||||||
|
if (runtimeMaterialized) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return !isReconciliableOpenCodeUnknownOutcome(
|
||||||
|
collectRuntimeLaunchFailureDiagnostics(result, memberName)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
function isLaunchGraceWindowFailureReason(reason?: string): boolean {
|
function isLaunchGraceWindowFailureReason(reason?: string): boolean {
|
||||||
return reason?.trim() === 'Teammate did not join within the launch grace window.';
|
return reason?.trim() === 'Teammate did not join within the launch grace window.';
|
||||||
}
|
}
|
||||||
|
|
@ -12438,7 +12479,20 @@ export class TeamProvisioningService {
|
||||||
lane.warnings = [...result.warnings];
|
lane.warnings = [...result.warnings];
|
||||||
lane.diagnostics = [...migration.diagnostics, ...result.diagnostics];
|
lane.diagnostics = [...migration.diagnostics, ...result.diagnostics];
|
||||||
|
|
||||||
if (result.teamLaunchState === 'partial_failure') {
|
if (isDefinitiveOpenCodePreLaunchFailure(result, lane.member.name)) {
|
||||||
|
const diagnostics = [
|
||||||
|
...migration.diagnostics,
|
||||||
|
...collectRuntimeLaunchFailureDiagnostics(result, lane.member.name),
|
||||||
|
];
|
||||||
|
await upsertOpenCodeRuntimeLaneIndexEntry({
|
||||||
|
teamsBasePath: getTeamsBasePath(),
|
||||||
|
teamName: run.teamName,
|
||||||
|
laneId: lane.laneId,
|
||||||
|
state: 'degraded',
|
||||||
|
diagnostics,
|
||||||
|
}).catch(() => undefined);
|
||||||
|
this.deleteSecondaryRuntimeRun(run.teamName, lane.laneId);
|
||||||
|
} else if (result.teamLaunchState === 'partial_failure') {
|
||||||
this.deleteSecondaryRuntimeRun(run.teamName, lane.laneId);
|
this.deleteSecondaryRuntimeRun(run.teamName, lane.laneId);
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
|
|
||||||
|
|
@ -79,6 +79,7 @@ liveDescribe('OpenCode mixed recovery live e2e', () => {
|
||||||
const bridgeEnv = {
|
const bridgeEnv = {
|
||||||
...createStableBridgeEnv(),
|
...createStableBridgeEnv(),
|
||||||
PATH: withBunOnPath(process.env.PATH ?? ''),
|
PATH: withBunOnPath(process.env.PATH ?? ''),
|
||||||
|
XDG_DATA_HOME: path.join(tempDir, 'xdg-data-single'),
|
||||||
CLAUDE_MULTIMODEL_AGENT_TEAMS_MCP_COMMAND: mcpLaunchSpec.command,
|
CLAUDE_MULTIMODEL_AGENT_TEAMS_MCP_COMMAND: mcpLaunchSpec.command,
|
||||||
CLAUDE_MULTIMODEL_AGENT_TEAMS_MCP_ENTRY: mcpLaunchSpec.args[0] ?? '',
|
CLAUDE_MULTIMODEL_AGENT_TEAMS_MCP_ENTRY: mcpLaunchSpec.args[0] ?? '',
|
||||||
};
|
};
|
||||||
|
|
@ -185,6 +186,7 @@ liveDescribe('OpenCode mixed recovery live e2e', () => {
|
||||||
const bridgeEnv = {
|
const bridgeEnv = {
|
||||||
...createStableBridgeEnv(),
|
...createStableBridgeEnv(),
|
||||||
PATH: withBunOnPath(process.env.PATH ?? ''),
|
PATH: withBunOnPath(process.env.PATH ?? ''),
|
||||||
|
XDG_DATA_HOME: path.join(tempDir, 'xdg-data-multi'),
|
||||||
CLAUDE_MULTIMODEL_AGENT_TEAMS_MCP_COMMAND: mcpLaunchSpec.command,
|
CLAUDE_MULTIMODEL_AGENT_TEAMS_MCP_COMMAND: mcpLaunchSpec.command,
|
||||||
CLAUDE_MULTIMODEL_AGENT_TEAMS_MCP_ENTRY: mcpLaunchSpec.args[0] ?? '',
|
CLAUDE_MULTIMODEL_AGENT_TEAMS_MCP_ENTRY: mcpLaunchSpec.args[0] ?? '',
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -68,6 +68,7 @@ liveDescribe('OpenCode team provisioning live e2e', () => {
|
||||||
const bridgeEnv = {
|
const bridgeEnv = {
|
||||||
...createStableBridgeEnv(),
|
...createStableBridgeEnv(),
|
||||||
PATH: withBunOnPath(process.env.PATH ?? ''),
|
PATH: withBunOnPath(process.env.PATH ?? ''),
|
||||||
|
XDG_DATA_HOME: path.join(tempDir, 'xdg-data'),
|
||||||
CLAUDE_MULTIMODEL_AGENT_TEAMS_MCP_COMMAND: mcpLaunchSpec.command,
|
CLAUDE_MULTIMODEL_AGENT_TEAMS_MCP_COMMAND: mcpLaunchSpec.command,
|
||||||
CLAUDE_MULTIMODEL_AGENT_TEAMS_MCP_ENTRY: mcpLaunchSpec.args[0] ?? '',
|
CLAUDE_MULTIMODEL_AGENT_TEAMS_MCP_ENTRY: mcpLaunchSpec.args[0] ?? '',
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -2482,6 +2482,110 @@ describe('TeamProvisioningService', () => {
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('marks an OpenCode secondary lane degraded when readiness fails before runtime materializes', async () => {
|
||||||
|
const teamName = 'mixed-prelaunch-failure';
|
||||||
|
const svc = new TeamProvisioningService();
|
||||||
|
const adapterLaunch = vi.fn(async (input: Record<string, unknown>) => ({
|
||||||
|
runId: String(input.runId),
|
||||||
|
teamName: String(input.teamName),
|
||||||
|
launchPhase: 'finished',
|
||||||
|
teamLaunchState: 'partial_failure',
|
||||||
|
members: {
|
||||||
|
bob: {
|
||||||
|
memberName: 'bob',
|
||||||
|
providerId: 'opencode',
|
||||||
|
launchState: 'failed_to_start',
|
||||||
|
agentToolAccepted: false,
|
||||||
|
runtimeAlive: false,
|
||||||
|
bootstrapConfirmed: false,
|
||||||
|
hardFailure: true,
|
||||||
|
hardFailureReason: 'unknown_error',
|
||||||
|
diagnostics: [
|
||||||
|
'OpenCode readiness bridge failed: timeout: OpenCode bridge command timed out',
|
||||||
|
'opencode_bridge_unknown_outcome: OpenCode bridge command timed out',
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
warnings: [],
|
||||||
|
diagnostics: [
|
||||||
|
'OpenCode readiness bridge failed: timeout: OpenCode bridge command timed out',
|
||||||
|
],
|
||||||
|
}));
|
||||||
|
|
||||||
|
const registry = new TeamRuntimeAdapterRegistry([
|
||||||
|
{
|
||||||
|
providerId: 'opencode',
|
||||||
|
prepare: vi.fn(),
|
||||||
|
launch: adapterLaunch,
|
||||||
|
reconcile: vi.fn(),
|
||||||
|
stop: vi.fn(),
|
||||||
|
} as any,
|
||||||
|
]);
|
||||||
|
svc.setRuntimeAdapterRegistry(registry);
|
||||||
|
|
||||||
|
(svc as any).launchStateStore = {
|
||||||
|
read: vi.fn(async () => null),
|
||||||
|
write: vi.fn(async () => {}),
|
||||||
|
clear: vi.fn(async () => {}),
|
||||||
|
};
|
||||||
|
|
||||||
|
const run = createMemberSpawnRun({
|
||||||
|
teamName,
|
||||||
|
expectedMembers: ['alice'],
|
||||||
|
});
|
||||||
|
run.isLaunch = true;
|
||||||
|
run.request = {
|
||||||
|
teamName,
|
||||||
|
cwd: '/tmp/mixed-prelaunch-failure',
|
||||||
|
providerId: 'codex',
|
||||||
|
model: 'gpt-5.4',
|
||||||
|
effort: 'high',
|
||||||
|
skipPermissions: true,
|
||||||
|
};
|
||||||
|
run.effectiveMembers = [
|
||||||
|
{
|
||||||
|
name: 'alice',
|
||||||
|
role: 'Reviewer',
|
||||||
|
providerId: 'codex',
|
||||||
|
model: 'gpt-5.4',
|
||||||
|
effort: 'high',
|
||||||
|
},
|
||||||
|
];
|
||||||
|
run.mixedSecondaryLanes = [
|
||||||
|
{
|
||||||
|
laneId: 'secondary:opencode:bob',
|
||||||
|
providerId: 'opencode',
|
||||||
|
member: {
|
||||||
|
name: 'bob',
|
||||||
|
role: 'Developer',
|
||||||
|
providerId: 'opencode',
|
||||||
|
model: 'minimax-m2.5-free',
|
||||||
|
effort: 'medium',
|
||||||
|
},
|
||||||
|
runId: null,
|
||||||
|
state: 'queued',
|
||||||
|
result: null,
|
||||||
|
warnings: [],
|
||||||
|
diagnostics: [],
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
await (svc as any).launchMixedSecondaryLaneIfNeeded(run);
|
||||||
|
await vi.waitFor(() => {
|
||||||
|
expect(adapterLaunch).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
await expect(readOpenCodeRuntimeLaneIndex(tempTeamsBase, teamName)).resolves.toMatchObject({
|
||||||
|
lanes: {
|
||||||
|
'secondary:opencode:bob': {
|
||||||
|
state: 'degraded',
|
||||||
|
diagnostics: expect.arrayContaining([
|
||||||
|
'OpenCode readiness bridge failed: timeout: OpenCode bridge command timed out',
|
||||||
|
]),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
it('starts all queued OpenCode secondary lanes without letting the first in-flight lane block its siblings', async () => {
|
it('starts all queued OpenCode secondary lanes without letting the first in-flight lane block its siblings', async () => {
|
||||||
const svc = new TeamProvisioningService();
|
const svc = new TeamProvisioningService();
|
||||||
const registry = new TeamRuntimeAdapterRegistry([
|
const registry = new TeamRuntimeAdapterRegistry([
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue