fix(opencode): recover capability snapshot launch retries

This commit is contained in:
777genius 2026-05-18 22:19:43 +03:00
parent dd4029f552
commit 3eacf81603
5 changed files with 286 additions and 11 deletions

View file

@ -68,6 +68,7 @@ export interface OpenCodeLaunchTeamCommandBody {
leadPrompt: string; leadPrompt: string;
expectedCapabilitySnapshotId: string | null; expectedCapabilitySnapshotId: string | null;
manifestHighWatermark: number | null; manifestHighWatermark: number | null;
capabilitySnapshotRecoveryAttemptId?: string;
} }
export interface OpenCodeTeamMemberLaunchCommandData { export interface OpenCodeTeamMemberLaunchCommandData {

View file

@ -103,10 +103,9 @@ const BEARER_TOKEN_PATTERN = /\bBearer\s+\S+/gi;
const SECRET_KEY_PATTERN = /\bsk-[A-Za-z0-9_-]{16,}\b/g; const SECRET_KEY_PATTERN = /\bsk-[A-Za-z0-9_-]{16,}\b/g;
const OPEN_CODE_CAPABILITY_SNAPSHOT_REFRESH_RETRY_WARNING = const OPEN_CODE_CAPABILITY_SNAPSHOT_REFRESH_RETRY_WARNING =
'OpenCode capability snapshot changed between readiness and launch; refreshed readiness and retried once.'; 'OpenCode capability snapshot changed between readiness and launch; refreshed readiness and retried once.';
const OPEN_CODE_CAPABILITY_SNAPSHOT_MISMATCH_MARKERS = [ const OPEN_CODE_CAPABILITY_SNAPSHOT_PRELAUNCH_MISMATCH_MARKERS = [
'Bridge server capability snapshot mismatch', 'Bridge server capability snapshot mismatch',
'OpenCode bridge capability snapshot precondition mismatch', 'OpenCode bridge capability snapshot precondition mismatch',
'OpenCode bridge capability snapshot mismatch',
]; ];
function resolveOpenCodeRuntimeSettlementMode( function resolveOpenCodeRuntimeSettlementMode(
@ -207,7 +206,8 @@ export class OpenCodeTeamRuntimeAdapter implements TeamLaunchRuntimeAdapter {
this.lastProjectPathByTeamName.set(input.teamName, input.cwd); this.lastProjectPathByTeamName.set(input.teamName, input.cwd);
const buildLaunchCommand = ( const buildLaunchCommand = (
snapshot: OpenCodeBridgeRuntimeSnapshot | null, snapshot: OpenCodeBridgeRuntimeSnapshot | null,
model: string model: string,
recoveryAttemptId?: string
): OpenCodeLaunchTeamCommandBody => ({ ): OpenCodeLaunchTeamCommandBody => ({
runId: input.runId, runId: input.runId,
laneId: input.laneId?.trim() || 'primary', laneId: input.laneId?.trim() || 'primary',
@ -223,12 +223,13 @@ export class OpenCodeTeamRuntimeAdapter implements TeamLaunchRuntimeAdapter {
leadPrompt: input.prompt?.trim() ?? '', leadPrompt: input.prompt?.trim() ?? '',
expectedCapabilitySnapshotId: snapshot?.capabilitySnapshotId ?? null, expectedCapabilitySnapshotId: snapshot?.capabilitySnapshotId ?? null,
manifestHighWatermark: null, manifestHighWatermark: null,
...(recoveryAttemptId ? { capabilitySnapshotRecoveryAttemptId: recoveryAttemptId } : {}),
}); });
let data = await this.bridge.launchOpenCodeTeam( let data = await this.bridge.launchOpenCodeTeam(
buildLaunchCommand(runtimeSnapshot, selectedModel) buildLaunchCommand(runtimeSnapshot, selectedModel)
); );
if (!skipReadinessPreflight && isOpenCodeCapabilitySnapshotMismatchLaunchData(data)) { if (!skipReadinessPreflight && isOpenCodePreLaunchCapabilitySnapshotMismatchData(data)) {
const refreshed = await this.prepare(input); const refreshed = await this.prepare(input);
if (!refreshed.ok) { if (!refreshed.ok) {
return blockedLaunchResult( return blockedLaunchResult(
@ -249,8 +250,16 @@ export class OpenCodeTeamRuntimeAdapter implements TeamLaunchRuntimeAdapter {
...refreshed.warnings, ...refreshed.warnings,
OPEN_CODE_CAPABILITY_SNAPSHOT_REFRESH_RETRY_WARNING, OPEN_CODE_CAPABILITY_SNAPSHOT_REFRESH_RETRY_WARNING,
]); ]);
// TODO(opencode-bridge): replace marker-based capability recovery with
// structured bridge failure details: expectedCapabilitySnapshotId,
// actualCapabilitySnapshotId, preconditionStage, and safeToRetryWithFreshCommand.
// Keep this app-side attempt id until packaged runtimes all expose that protocol.
data = await this.bridge.launchOpenCodeTeam( data = await this.bridge.launchOpenCodeTeam(
buildLaunchCommand(runtimeSnapshot, selectedModel) buildLaunchCommand(
runtimeSnapshot,
selectedModel,
`opencode-capability-recovery-${randomUUID()}`
)
); );
} }
} }
@ -1097,26 +1106,29 @@ function formatOpenCodeBridgeDiagnostic(diagnostic: {
return `${diagnostic.severity}:${diagnostic.code}: ${diagnostic.message}`; return `${diagnostic.severity}:${diagnostic.code}: ${diagnostic.message}`;
} }
function isOpenCodeCapabilitySnapshotMismatchLaunchData( function isOpenCodePreLaunchCapabilitySnapshotMismatchData(
data: OpenCodeLaunchTeamCommandData data: OpenCodeLaunchTeamCommandData
): boolean { ): boolean {
if (data.teamLaunchState !== 'failed') {
return false;
}
if ( if (
data.diagnostics.some( data.diagnostics.some(
(diagnostic) => (diagnostic) =>
isOpenCodeCapabilitySnapshotMismatchText(diagnostic.message) || isOpenCodePreLaunchCapabilitySnapshotMismatchText(diagnostic.message) ||
isOpenCodeCapabilitySnapshotMismatchText(diagnostic.code) isOpenCodePreLaunchCapabilitySnapshotMismatchText(diagnostic.code)
) )
) { ) {
return true; return true;
} }
return Object.values(data.members).some((member) => return Object.values(data.members).some((member) =>
(member.diagnostics ?? []).some(isOpenCodeCapabilitySnapshotMismatchText) (member.diagnostics ?? []).some(isOpenCodePreLaunchCapabilitySnapshotMismatchText)
); );
} }
function isOpenCodeCapabilitySnapshotMismatchText(value: string): boolean { function isOpenCodePreLaunchCapabilitySnapshotMismatchText(value: string): boolean {
const normalized = value.toLowerCase(); const normalized = value.toLowerCase();
return OPEN_CODE_CAPABILITY_SNAPSHOT_MISMATCH_MARKERS.some((marker) => return OPEN_CODE_CAPABILITY_SNAPSHOT_PRELAUNCH_MISMATCH_MARKERS.some((marker) =>
normalized.includes(marker.toLowerCase()) normalized.includes(marker.toLowerCase())
); );
} }

View file

@ -61,6 +61,37 @@ describe('OpenCodeBridgeCommandContract', () => {
expect(isOpenCodeBridgeCommandName('opencode.backfillTaskLedger')).toBe(true); expect(isOpenCodeBridgeCommandName('opencode.backfillTaskLedger')).toBe(true);
}); });
it('uses capability recovery attempt id as part of the launch idempotency key', () => {
const baseInput = {
command: 'opencode.launchTeam' as const,
teamName: 'team-a',
laneId: 'secondary:opencode:alice',
runId: 'run-1',
body: {
runId: 'run-1',
laneId: 'secondary:opencode:alice',
teamId: 'team-a',
teamName: 'team-a',
projectPath: '/repo',
selectedModel: 'openai/gpt-5.4-mini',
members: [{ name: 'alice', role: 'Developer', prompt: 'Launch' }],
leadPrompt: '',
expectedCapabilitySnapshotId: 'cap-1',
manifestHighWatermark: null,
},
};
expect(createOpenCodeBridgeIdempotencyKey(baseInput)).not.toBe(
createOpenCodeBridgeIdempotencyKey({
...baseInput,
body: {
...baseInput.body,
capabilitySnapshotRecoveryAttemptId: 'opencode-capability-recovery-test',
},
})
);
});
it('validates result request id and command against the command envelope', () => { it('validates result request id and command against the command envelope', () => {
const envelope: OpenCodeBridgeCommandEnvelope<Record<string, never>> = { const envelope: OpenCodeBridgeCommandEnvelope<Record<string, never>> = {
schemaVersion: 1, schemaVersion: 1,

View file

@ -325,6 +325,62 @@ describe('OpenCodeStateChangingBridgeCommandService', () => {
await expect(leaseStore.getActive('team-a')).resolves.toBeNull(); await expect(leaseStore.getActive('team-a')).resolves.toBeNull();
}); });
it('treats capability recovery attempt id as a fresh state-changing command body', async () => {
bridge.resultFactory = ({ body, command, options }) =>
({
ok: false,
schemaVersion: 1,
requestId: options.requestId,
command,
completedAt: '2026-04-21T12:00:10.000Z',
durationMs: 10_000,
error: {
kind: 'provider_error',
message: 'OpenCode bridge capability snapshot precondition mismatch',
retryable: true,
},
diagnostics: [],
data: body,
}) as OpenCodeBridgeResult<unknown>;
const service = createService();
const first = await service.execute(buildLaunchInput());
expect(first).toMatchObject({
ok: false,
error: { message: 'OpenCode bridge capability snapshot precondition mismatch' },
});
const firstIdempotencyKey = bridge.calls[0].body.preconditions.idempotencyKey;
await expect(ledger.getByIdempotencyKey(firstIdempotencyKey)).resolves.toMatchObject({
status: 'failed',
retryable: true,
});
await expect(service.execute(buildLaunchInput())).rejects.toThrow(
'OpenCode bridge command cannot be retried from status failed'
);
expect(bridge.calls).toHaveLength(1);
const recovery = await service.execute({
...buildLaunchInput(),
body: {
prompt: 'launch',
capabilitySnapshotRecoveryAttemptId: 'opencode-capability-recovery-test',
},
});
expect(recovery).toMatchObject({
ok: false,
error: { message: 'OpenCode bridge capability snapshot precondition mismatch' },
});
expect(bridge.calls).toHaveLength(2);
const recoveryIdempotencyKey = bridge.calls[1].body.preconditions.idempotencyKey;
expect(recoveryIdempotencyKey).not.toBe(firstIdempotencyKey);
await expect(ledger.getByIdempotencyKey(recoveryIdempotencyKey)).resolves.toMatchObject({
status: 'failed',
retryable: true,
});
await expect(leaseStore.getActive('team-a')).resolves.toBeNull();
});
function createService( function createService(
overrides: { overrides: {
leaseAcquireTimeoutMs?: number; leaseAcquireTimeoutMs?: number;

View file

@ -393,6 +393,12 @@ describe('OpenCodeTeamRuntimeAdapter', () => {
expect(launchOpenCodeTeam).toHaveBeenCalledTimes(2); expect(launchOpenCodeTeam).toHaveBeenCalledTimes(2);
expect(launchOpenCodeTeam.mock.calls[0]?.[0].expectedCapabilitySnapshotId).toBe('cap-old'); expect(launchOpenCodeTeam.mock.calls[0]?.[0].expectedCapabilitySnapshotId).toBe('cap-old');
expect(launchOpenCodeTeam.mock.calls[1]?.[0].expectedCapabilitySnapshotId).toBe('cap-new'); expect(launchOpenCodeTeam.mock.calls[1]?.[0].expectedCapabilitySnapshotId).toBe('cap-new');
expect(launchOpenCodeTeam.mock.calls[0]?.[0].capabilitySnapshotRecoveryAttemptId).toBe(
undefined
);
expect(launchOpenCodeTeam.mock.calls[1]?.[0].capabilitySnapshotRecoveryAttemptId).toMatch(
/^opencode-capability-recovery-/
);
}); });
it('refreshes readiness and retries once when the launch command sees a newer capability snapshot', async () => { it('refreshes readiness and retries once when the launch command sees a newer capability snapshot', async () => {
@ -406,6 +412,156 @@ describe('OpenCodeTeamRuntimeAdapter', () => {
expect(launchOpenCodeTeam).toHaveBeenCalledTimes(2); expect(launchOpenCodeTeam).toHaveBeenCalledTimes(2);
expect(launchOpenCodeTeam.mock.calls[0]?.[0].expectedCapabilitySnapshotId).toBe('cap-old'); expect(launchOpenCodeTeam.mock.calls[0]?.[0].expectedCapabilitySnapshotId).toBe('cap-old');
expect(launchOpenCodeTeam.mock.calls[1]?.[0].expectedCapabilitySnapshotId).toBe('cap-new'); expect(launchOpenCodeTeam.mock.calls[1]?.[0].expectedCapabilitySnapshotId).toBe('cap-new');
expect(launchOpenCodeTeam.mock.calls[1]?.[0].capabilitySnapshotRecoveryAttemptId).toMatch(
/^opencode-capability-recovery-/
);
});
it('uses a fresh recovery attempt id when capability refresh returns the same snapshot', async () => {
let readinessCalls = 0;
const checkReadiness = vi.fn<
OpenCodeTeamRuntimeBridgePort['checkOpenCodeTeamLaunchReadiness']
>(() => {
readinessCalls += 1;
return Promise.resolve(readiness({ state: 'ready', launchAllowed: true }));
});
const launchOpenCodeTeam = vi.fn<
NonNullable<OpenCodeTeamRuntimeBridgePort['launchOpenCodeTeam']>
>((input) =>
Promise.resolve(
input.capabilitySnapshotRecoveryAttemptId
? successfulOpenCodeLaunchData()
: failedCapabilitySnapshotLaunchData(
'OpenCode bridge capability snapshot precondition mismatch'
)
)
);
const adapter = new OpenCodeTeamRuntimeAdapter({
checkOpenCodeTeamLaunchReadiness: checkReadiness,
getLastOpenCodeRuntimeSnapshot: vi.fn(() => runtimeSnapshot('cap-1')),
launchOpenCodeTeam,
});
const result = await adapter.launch(launchInput());
expect(result.teamLaunchState).toBe('clean_success');
expect(readinessCalls).toBe(2);
expect(launchOpenCodeTeam).toHaveBeenCalledTimes(2);
expect(launchOpenCodeTeam.mock.calls[0]?.[0].expectedCapabilitySnapshotId).toBe('cap-1');
expect(launchOpenCodeTeam.mock.calls[1]?.[0].expectedCapabilitySnapshotId).toBe('cap-1');
expect(launchOpenCodeTeam.mock.calls[0]?.[0].capabilitySnapshotRecoveryAttemptId).toBe(
undefined
);
expect(launchOpenCodeTeam.mock.calls[1]?.[0].capabilitySnapshotRecoveryAttemptId).toMatch(
/^opencode-capability-recovery-/
);
});
it('retries pre-launch capability mismatch reported in member diagnostics', async () => {
const checkReadiness = vi.fn<
OpenCodeTeamRuntimeBridgePort['checkOpenCodeTeamLaunchReadiness']
>(() => Promise.resolve(readiness({ state: 'ready', launchAllowed: true })));
const launchOpenCodeTeam = vi.fn<
NonNullable<OpenCodeTeamRuntimeBridgePort['launchOpenCodeTeam']>
>((input) =>
Promise.resolve(
input.capabilitySnapshotRecoveryAttemptId
? successfulOpenCodeLaunchData()
: failedMemberCapabilitySnapshotLaunchData(
'OpenCode bridge capability snapshot precondition mismatch'
)
)
);
const adapter = new OpenCodeTeamRuntimeAdapter({
checkOpenCodeTeamLaunchReadiness: checkReadiness,
getLastOpenCodeRuntimeSnapshot: vi.fn(() => runtimeSnapshot('cap-1')),
launchOpenCodeTeam,
});
const result = await adapter.launch(launchInput());
expect(result.teamLaunchState).toBe('clean_success');
expect(checkReadiness).toHaveBeenCalledTimes(2);
expect(launchOpenCodeTeam).toHaveBeenCalledTimes(2);
expect(launchOpenCodeTeam.mock.calls[1]?.[0].capabilitySnapshotRecoveryAttemptId).toMatch(
/^opencode-capability-recovery-/
);
});
it('does not retry a successful launch just because stale diagnostics mention pre-launch mismatch', async () => {
const checkReadiness = vi.fn<
OpenCodeTeamRuntimeBridgePort['checkOpenCodeTeamLaunchReadiness']
>(() => Promise.resolve(readiness({ state: 'ready', launchAllowed: true })));
const launchOpenCodeTeam = vi.fn<
NonNullable<OpenCodeTeamRuntimeBridgePort['launchOpenCodeTeam']>
>(() =>
Promise.resolve({
...successfulOpenCodeLaunchData(),
diagnostics: [
{
code: 'stale_note',
severity: 'warning',
message: 'OpenCode bridge capability snapshot precondition mismatch',
},
],
})
);
const adapter = new OpenCodeTeamRuntimeAdapter({
checkOpenCodeTeamLaunchReadiness: checkReadiness,
getLastOpenCodeRuntimeSnapshot: vi.fn(() => runtimeSnapshot('cap-1')),
launchOpenCodeTeam,
});
const result = await adapter.launch(launchInput());
expect(result.teamLaunchState).toBe('clean_success');
expect(checkReadiness).toHaveBeenCalledTimes(1);
expect(launchOpenCodeTeam).toHaveBeenCalledTimes(1);
});
it('does not retry post-result capability snapshot mismatch', async () => {
const { result, checkReadiness, launchOpenCodeTeam } =
await launchWithStaleCapabilitySnapshotRecovery(
'OpenCode bridge capability snapshot mismatch'
);
expect(result.teamLaunchState).toBe('partial_failure');
expect(checkReadiness).toHaveBeenCalledTimes(1);
expect(launchOpenCodeTeam).toHaveBeenCalledTimes(1);
expect(result.diagnostics).toContain(
'error:opencode_bridge: OpenCode bridge failed: OpenCode bridge capability snapshot mismatch'
);
});
it('keeps the original precondition mismatch when the recovery retry also fails', async () => {
const checkReadiness = vi.fn<
OpenCodeTeamRuntimeBridgePort['checkOpenCodeTeamLaunchReadiness']
>(() => Promise.resolve(readiness({ state: 'ready', launchAllowed: true })));
const launchOpenCodeTeam = vi.fn<
NonNullable<OpenCodeTeamRuntimeBridgePort['launchOpenCodeTeam']>
>(() =>
Promise.resolve(
failedCapabilitySnapshotLaunchData(
'OpenCode bridge capability snapshot precondition mismatch'
)
)
);
const adapter = new OpenCodeTeamRuntimeAdapter({
checkOpenCodeTeamLaunchReadiness: checkReadiness,
getLastOpenCodeRuntimeSnapshot: vi.fn(() => runtimeSnapshot('cap-1')),
launchOpenCodeTeam,
});
const result = await adapter.launch(launchInput());
expect(result.teamLaunchState).toBe('partial_failure');
expect(launchOpenCodeTeam).toHaveBeenCalledTimes(2);
expect(result.diagnostics).toContain(
'error:opencode_bridge: OpenCode bridge failed: OpenCode bridge capability snapshot precondition mismatch'
);
expect(result.diagnostics.join('\n')).not.toContain(
'OpenCode bridge command cannot be retried from status failed'
);
}); });
it('does not mark the lane clean_success when ready bridge data omits an expected member', async () => { it('does not mark the lane clean_success when ready bridge data omits an expected member', async () => {
@ -1207,6 +1363,25 @@ function failedCapabilitySnapshotLaunchData(message: string): OpenCodeLaunchTeam
}; };
} }
function failedMemberCapabilitySnapshotLaunchData(message: string): OpenCodeLaunchTeamCommandData {
return {
runId: 'run-1',
teamLaunchState: 'failed',
members: {
alice: {
sessionId: 'oc-session-1',
launchState: 'failed',
runtimePid: 123,
model: 'openai/gpt-5.4-mini',
evidence: [],
diagnostics: [message],
},
},
warnings: [],
diagnostics: [],
};
}
function bridgePort( function bridgePort(
readinessResult: OpenCodeTeamLaunchReadiness, readinessResult: OpenCodeTeamLaunchReadiness,
overrides: Partial<OpenCodeTeamRuntimeBridgePort> = {} overrides: Partial<OpenCodeTeamRuntimeBridgePort> = {}