From 3eacf81603103d1e723287320872bcb110ee1a2d Mon Sep 17 00:00:00 2001 From: 777genius Date: Mon, 18 May 2026 22:19:43 +0300 Subject: [PATCH] fix(opencode): recover capability snapshot launch retries --- .../bridge/OpenCodeBridgeCommandContract.ts | 1 + .../runtime/OpenCodeTeamRuntimeAdapter.ts | 34 ++-- .../OpenCodeBridgeCommandContract.test.ts | 31 ++++ ...eStateChangingBridgeCommandService.test.ts | 56 ++++++ .../team/OpenCodeTeamRuntimeAdapter.test.ts | 175 ++++++++++++++++++ 5 files changed, 286 insertions(+), 11 deletions(-) diff --git a/src/main/services/team/opencode/bridge/OpenCodeBridgeCommandContract.ts b/src/main/services/team/opencode/bridge/OpenCodeBridgeCommandContract.ts index 52490fca..0d0c50e2 100644 --- a/src/main/services/team/opencode/bridge/OpenCodeBridgeCommandContract.ts +++ b/src/main/services/team/opencode/bridge/OpenCodeBridgeCommandContract.ts @@ -68,6 +68,7 @@ export interface OpenCodeLaunchTeamCommandBody { leadPrompt: string; expectedCapabilitySnapshotId: string | null; manifestHighWatermark: number | null; + capabilitySnapshotRecoveryAttemptId?: string; } export interface OpenCodeTeamMemberLaunchCommandData { diff --git a/src/main/services/team/runtime/OpenCodeTeamRuntimeAdapter.ts b/src/main/services/team/runtime/OpenCodeTeamRuntimeAdapter.ts index 1d179f68..51563623 100644 --- a/src/main/services/team/runtime/OpenCodeTeamRuntimeAdapter.ts +++ b/src/main/services/team/runtime/OpenCodeTeamRuntimeAdapter.ts @@ -103,10 +103,9 @@ const BEARER_TOKEN_PATTERN = /\bBearer\s+\S+/gi; const SECRET_KEY_PATTERN = /\bsk-[A-Za-z0-9_-]{16,}\b/g; const OPEN_CODE_CAPABILITY_SNAPSHOT_REFRESH_RETRY_WARNING = 'OpenCode capability snapshot changed between readiness and launch; refreshed readiness and retried once.'; -const OPEN_CODE_CAPABILITY_SNAPSHOT_MISMATCH_MARKERS = [ +const OPEN_CODE_CAPABILITY_SNAPSHOT_PRELAUNCH_MISMATCH_MARKERS = [ 'Bridge server capability snapshot mismatch', 'OpenCode bridge capability snapshot precondition mismatch', - 'OpenCode bridge capability snapshot mismatch', ]; function resolveOpenCodeRuntimeSettlementMode( @@ -207,7 +206,8 @@ export class OpenCodeTeamRuntimeAdapter implements TeamLaunchRuntimeAdapter { this.lastProjectPathByTeamName.set(input.teamName, input.cwd); const buildLaunchCommand = ( snapshot: OpenCodeBridgeRuntimeSnapshot | null, - model: string + model: string, + recoveryAttemptId?: string ): OpenCodeLaunchTeamCommandBody => ({ runId: input.runId, laneId: input.laneId?.trim() || 'primary', @@ -223,12 +223,13 @@ export class OpenCodeTeamRuntimeAdapter implements TeamLaunchRuntimeAdapter { leadPrompt: input.prompt?.trim() ?? '', expectedCapabilitySnapshotId: snapshot?.capabilitySnapshotId ?? null, manifestHighWatermark: null, + ...(recoveryAttemptId ? { capabilitySnapshotRecoveryAttemptId: recoveryAttemptId } : {}), }); let data = await this.bridge.launchOpenCodeTeam( buildLaunchCommand(runtimeSnapshot, selectedModel) ); - if (!skipReadinessPreflight && isOpenCodeCapabilitySnapshotMismatchLaunchData(data)) { + if (!skipReadinessPreflight && isOpenCodePreLaunchCapabilitySnapshotMismatchData(data)) { const refreshed = await this.prepare(input); if (!refreshed.ok) { return blockedLaunchResult( @@ -249,8 +250,16 @@ export class OpenCodeTeamRuntimeAdapter implements TeamLaunchRuntimeAdapter { ...refreshed.warnings, OPEN_CODE_CAPABILITY_SNAPSHOT_REFRESH_RETRY_WARNING, ]); + // TODO(opencode-bridge): replace marker-based capability recovery with + // structured bridge failure details: expectedCapabilitySnapshotId, + // actualCapabilitySnapshotId, preconditionStage, and safeToRetryWithFreshCommand. + // Keep this app-side attempt id until packaged runtimes all expose that protocol. data = await this.bridge.launchOpenCodeTeam( - buildLaunchCommand(runtimeSnapshot, selectedModel) + buildLaunchCommand( + runtimeSnapshot, + selectedModel, + `opencode-capability-recovery-${randomUUID()}` + ) ); } } @@ -1097,26 +1106,29 @@ function formatOpenCodeBridgeDiagnostic(diagnostic: { return `${diagnostic.severity}:${diagnostic.code}: ${diagnostic.message}`; } -function isOpenCodeCapabilitySnapshotMismatchLaunchData( +function isOpenCodePreLaunchCapabilitySnapshotMismatchData( data: OpenCodeLaunchTeamCommandData ): boolean { + if (data.teamLaunchState !== 'failed') { + return false; + } if ( data.diagnostics.some( (diagnostic) => - isOpenCodeCapabilitySnapshotMismatchText(diagnostic.message) || - isOpenCodeCapabilitySnapshotMismatchText(diagnostic.code) + isOpenCodePreLaunchCapabilitySnapshotMismatchText(diagnostic.message) || + isOpenCodePreLaunchCapabilitySnapshotMismatchText(diagnostic.code) ) ) { return true; } return Object.values(data.members).some((member) => - (member.diagnostics ?? []).some(isOpenCodeCapabilitySnapshotMismatchText) + (member.diagnostics ?? []).some(isOpenCodePreLaunchCapabilitySnapshotMismatchText) ); } -function isOpenCodeCapabilitySnapshotMismatchText(value: string): boolean { +function isOpenCodePreLaunchCapabilitySnapshotMismatchText(value: string): boolean { const normalized = value.toLowerCase(); - return OPEN_CODE_CAPABILITY_SNAPSHOT_MISMATCH_MARKERS.some((marker) => + return OPEN_CODE_CAPABILITY_SNAPSHOT_PRELAUNCH_MISMATCH_MARKERS.some((marker) => normalized.includes(marker.toLowerCase()) ); } diff --git a/test/main/services/team/OpenCodeBridgeCommandContract.test.ts b/test/main/services/team/OpenCodeBridgeCommandContract.test.ts index ec2aab58..f5f56267 100644 --- a/test/main/services/team/OpenCodeBridgeCommandContract.test.ts +++ b/test/main/services/team/OpenCodeBridgeCommandContract.test.ts @@ -61,6 +61,37 @@ describe('OpenCodeBridgeCommandContract', () => { expect(isOpenCodeBridgeCommandName('opencode.backfillTaskLedger')).toBe(true); }); + it('uses capability recovery attempt id as part of the launch idempotency key', () => { + const baseInput = { + command: 'opencode.launchTeam' as const, + teamName: 'team-a', + laneId: 'secondary:opencode:alice', + runId: 'run-1', + body: { + runId: 'run-1', + laneId: 'secondary:opencode:alice', + teamId: 'team-a', + teamName: 'team-a', + projectPath: '/repo', + selectedModel: 'openai/gpt-5.4-mini', + members: [{ name: 'alice', role: 'Developer', prompt: 'Launch' }], + leadPrompt: '', + expectedCapabilitySnapshotId: 'cap-1', + manifestHighWatermark: null, + }, + }; + + expect(createOpenCodeBridgeIdempotencyKey(baseInput)).not.toBe( + createOpenCodeBridgeIdempotencyKey({ + ...baseInput, + body: { + ...baseInput.body, + capabilitySnapshotRecoveryAttemptId: 'opencode-capability-recovery-test', + }, + }) + ); + }); + it('validates result request id and command against the command envelope', () => { const envelope: OpenCodeBridgeCommandEnvelope> = { schemaVersion: 1, diff --git a/test/main/services/team/OpenCodeStateChangingBridgeCommandService.test.ts b/test/main/services/team/OpenCodeStateChangingBridgeCommandService.test.ts index 548f6050..c61e1f88 100644 --- a/test/main/services/team/OpenCodeStateChangingBridgeCommandService.test.ts +++ b/test/main/services/team/OpenCodeStateChangingBridgeCommandService.test.ts @@ -325,6 +325,62 @@ describe('OpenCodeStateChangingBridgeCommandService', () => { await expect(leaseStore.getActive('team-a')).resolves.toBeNull(); }); + it('treats capability recovery attempt id as a fresh state-changing command body', async () => { + bridge.resultFactory = ({ body, command, options }) => + ({ + ok: false, + schemaVersion: 1, + requestId: options.requestId, + command, + completedAt: '2026-04-21T12:00:10.000Z', + durationMs: 10_000, + error: { + kind: 'provider_error', + message: 'OpenCode bridge capability snapshot precondition mismatch', + retryable: true, + }, + diagnostics: [], + data: body, + }) as OpenCodeBridgeResult; + const service = createService(); + + const first = await service.execute(buildLaunchInput()); + expect(first).toMatchObject({ + ok: false, + error: { message: 'OpenCode bridge capability snapshot precondition mismatch' }, + }); + const firstIdempotencyKey = bridge.calls[0].body.preconditions.idempotencyKey; + await expect(ledger.getByIdempotencyKey(firstIdempotencyKey)).resolves.toMatchObject({ + status: 'failed', + retryable: true, + }); + + await expect(service.execute(buildLaunchInput())).rejects.toThrow( + 'OpenCode bridge command cannot be retried from status failed' + ); + expect(bridge.calls).toHaveLength(1); + + const recovery = await service.execute({ + ...buildLaunchInput(), + body: { + prompt: 'launch', + capabilitySnapshotRecoveryAttemptId: 'opencode-capability-recovery-test', + }, + }); + expect(recovery).toMatchObject({ + ok: false, + error: { message: 'OpenCode bridge capability snapshot precondition mismatch' }, + }); + expect(bridge.calls).toHaveLength(2); + const recoveryIdempotencyKey = bridge.calls[1].body.preconditions.idempotencyKey; + expect(recoveryIdempotencyKey).not.toBe(firstIdempotencyKey); + await expect(ledger.getByIdempotencyKey(recoveryIdempotencyKey)).resolves.toMatchObject({ + status: 'failed', + retryable: true, + }); + await expect(leaseStore.getActive('team-a')).resolves.toBeNull(); + }); + function createService( overrides: { leaseAcquireTimeoutMs?: number; diff --git a/test/main/services/team/OpenCodeTeamRuntimeAdapter.test.ts b/test/main/services/team/OpenCodeTeamRuntimeAdapter.test.ts index 4e4584da..93ad5984 100644 --- a/test/main/services/team/OpenCodeTeamRuntimeAdapter.test.ts +++ b/test/main/services/team/OpenCodeTeamRuntimeAdapter.test.ts @@ -393,6 +393,12 @@ describe('OpenCodeTeamRuntimeAdapter', () => { expect(launchOpenCodeTeam).toHaveBeenCalledTimes(2); expect(launchOpenCodeTeam.mock.calls[0]?.[0].expectedCapabilitySnapshotId).toBe('cap-old'); expect(launchOpenCodeTeam.mock.calls[1]?.[0].expectedCapabilitySnapshotId).toBe('cap-new'); + expect(launchOpenCodeTeam.mock.calls[0]?.[0].capabilitySnapshotRecoveryAttemptId).toBe( + undefined + ); + expect(launchOpenCodeTeam.mock.calls[1]?.[0].capabilitySnapshotRecoveryAttemptId).toMatch( + /^opencode-capability-recovery-/ + ); }); it('refreshes readiness and retries once when the launch command sees a newer capability snapshot', async () => { @@ -406,6 +412,156 @@ describe('OpenCodeTeamRuntimeAdapter', () => { expect(launchOpenCodeTeam).toHaveBeenCalledTimes(2); expect(launchOpenCodeTeam.mock.calls[0]?.[0].expectedCapabilitySnapshotId).toBe('cap-old'); expect(launchOpenCodeTeam.mock.calls[1]?.[0].expectedCapabilitySnapshotId).toBe('cap-new'); + expect(launchOpenCodeTeam.mock.calls[1]?.[0].capabilitySnapshotRecoveryAttemptId).toMatch( + /^opencode-capability-recovery-/ + ); + }); + + it('uses a fresh recovery attempt id when capability refresh returns the same snapshot', async () => { + let readinessCalls = 0; + const checkReadiness = vi.fn< + OpenCodeTeamRuntimeBridgePort['checkOpenCodeTeamLaunchReadiness'] + >(() => { + readinessCalls += 1; + return Promise.resolve(readiness({ state: 'ready', launchAllowed: true })); + }); + const launchOpenCodeTeam = vi.fn< + NonNullable + >((input) => + Promise.resolve( + input.capabilitySnapshotRecoveryAttemptId + ? successfulOpenCodeLaunchData() + : failedCapabilitySnapshotLaunchData( + 'OpenCode bridge capability snapshot precondition mismatch' + ) + ) + ); + const adapter = new OpenCodeTeamRuntimeAdapter({ + checkOpenCodeTeamLaunchReadiness: checkReadiness, + getLastOpenCodeRuntimeSnapshot: vi.fn(() => runtimeSnapshot('cap-1')), + launchOpenCodeTeam, + }); + + const result = await adapter.launch(launchInput()); + + expect(result.teamLaunchState).toBe('clean_success'); + expect(readinessCalls).toBe(2); + expect(launchOpenCodeTeam).toHaveBeenCalledTimes(2); + expect(launchOpenCodeTeam.mock.calls[0]?.[0].expectedCapabilitySnapshotId).toBe('cap-1'); + expect(launchOpenCodeTeam.mock.calls[1]?.[0].expectedCapabilitySnapshotId).toBe('cap-1'); + expect(launchOpenCodeTeam.mock.calls[0]?.[0].capabilitySnapshotRecoveryAttemptId).toBe( + undefined + ); + expect(launchOpenCodeTeam.mock.calls[1]?.[0].capabilitySnapshotRecoveryAttemptId).toMatch( + /^opencode-capability-recovery-/ + ); + }); + + it('retries pre-launch capability mismatch reported in member diagnostics', async () => { + const checkReadiness = vi.fn< + OpenCodeTeamRuntimeBridgePort['checkOpenCodeTeamLaunchReadiness'] + >(() => Promise.resolve(readiness({ state: 'ready', launchAllowed: true }))); + const launchOpenCodeTeam = vi.fn< + NonNullable + >((input) => + Promise.resolve( + input.capabilitySnapshotRecoveryAttemptId + ? successfulOpenCodeLaunchData() + : failedMemberCapabilitySnapshotLaunchData( + 'OpenCode bridge capability snapshot precondition mismatch' + ) + ) + ); + const adapter = new OpenCodeTeamRuntimeAdapter({ + checkOpenCodeTeamLaunchReadiness: checkReadiness, + getLastOpenCodeRuntimeSnapshot: vi.fn(() => runtimeSnapshot('cap-1')), + launchOpenCodeTeam, + }); + + const result = await adapter.launch(launchInput()); + + expect(result.teamLaunchState).toBe('clean_success'); + expect(checkReadiness).toHaveBeenCalledTimes(2); + expect(launchOpenCodeTeam).toHaveBeenCalledTimes(2); + expect(launchOpenCodeTeam.mock.calls[1]?.[0].capabilitySnapshotRecoveryAttemptId).toMatch( + /^opencode-capability-recovery-/ + ); + }); + + it('does not retry a successful launch just because stale diagnostics mention pre-launch mismatch', async () => { + const checkReadiness = vi.fn< + OpenCodeTeamRuntimeBridgePort['checkOpenCodeTeamLaunchReadiness'] + >(() => Promise.resolve(readiness({ state: 'ready', launchAllowed: true }))); + const launchOpenCodeTeam = vi.fn< + NonNullable + >(() => + Promise.resolve({ + ...successfulOpenCodeLaunchData(), + diagnostics: [ + { + code: 'stale_note', + severity: 'warning', + message: 'OpenCode bridge capability snapshot precondition mismatch', + }, + ], + }) + ); + const adapter = new OpenCodeTeamRuntimeAdapter({ + checkOpenCodeTeamLaunchReadiness: checkReadiness, + getLastOpenCodeRuntimeSnapshot: vi.fn(() => runtimeSnapshot('cap-1')), + launchOpenCodeTeam, + }); + + const result = await adapter.launch(launchInput()); + + expect(result.teamLaunchState).toBe('clean_success'); + expect(checkReadiness).toHaveBeenCalledTimes(1); + expect(launchOpenCodeTeam).toHaveBeenCalledTimes(1); + }); + + it('does not retry post-result capability snapshot mismatch', async () => { + const { result, checkReadiness, launchOpenCodeTeam } = + await launchWithStaleCapabilitySnapshotRecovery( + 'OpenCode bridge capability snapshot mismatch' + ); + + expect(result.teamLaunchState).toBe('partial_failure'); + expect(checkReadiness).toHaveBeenCalledTimes(1); + expect(launchOpenCodeTeam).toHaveBeenCalledTimes(1); + expect(result.diagnostics).toContain( + 'error:opencode_bridge: OpenCode bridge failed: OpenCode bridge capability snapshot mismatch' + ); + }); + + it('keeps the original precondition mismatch when the recovery retry also fails', async () => { + const checkReadiness = vi.fn< + OpenCodeTeamRuntimeBridgePort['checkOpenCodeTeamLaunchReadiness'] + >(() => Promise.resolve(readiness({ state: 'ready', launchAllowed: true }))); + const launchOpenCodeTeam = vi.fn< + NonNullable + >(() => + Promise.resolve( + failedCapabilitySnapshotLaunchData( + 'OpenCode bridge capability snapshot precondition mismatch' + ) + ) + ); + const adapter = new OpenCodeTeamRuntimeAdapter({ + checkOpenCodeTeamLaunchReadiness: checkReadiness, + getLastOpenCodeRuntimeSnapshot: vi.fn(() => runtimeSnapshot('cap-1')), + launchOpenCodeTeam, + }); + + const result = await adapter.launch(launchInput()); + + expect(result.teamLaunchState).toBe('partial_failure'); + expect(launchOpenCodeTeam).toHaveBeenCalledTimes(2); + expect(result.diagnostics).toContain( + 'error:opencode_bridge: OpenCode bridge failed: OpenCode bridge capability snapshot precondition mismatch' + ); + expect(result.diagnostics.join('\n')).not.toContain( + 'OpenCode bridge command cannot be retried from status failed' + ); }); it('does not mark the lane clean_success when ready bridge data omits an expected member', async () => { @@ -1207,6 +1363,25 @@ function failedCapabilitySnapshotLaunchData(message: string): OpenCodeLaunchTeam }; } +function failedMemberCapabilitySnapshotLaunchData(message: string): OpenCodeLaunchTeamCommandData { + return { + runId: 'run-1', + teamLaunchState: 'failed', + members: { + alice: { + sessionId: 'oc-session-1', + launchState: 'failed', + runtimePid: 123, + model: 'openai/gpt-5.4-mini', + evidence: [], + diagnostics: [message], + }, + }, + warnings: [], + diagnostics: [], + }; +} + function bridgePort( readinessResult: OpenCodeTeamLaunchReadiness, overrides: Partial = {}