From 5c65f550674ef82768c3cc241a11802f194c29ef Mon Sep 17 00:00:00 2001 From: 777genius Date: Mon, 4 May 2026 14:48:55 +0300 Subject: [PATCH] feat(team): retry failed opencode secondary lanes --- src/main/ipc/teams.ts | 20 + .../services/team/TeamProvisioningService.ts | 362 ++++++++++++++++++ src/preload/constants/ipcChannels.ts | 3 + src/preload/index.ts | 8 + src/renderer/api/httpClient.ts | 3 + .../components/team/TeamProvisioningPanel.tsx | 110 +++++- .../team/useTeamProvisioningPresentation.ts | 32 +- src/renderer/store/slices/teamSlice.ts | 17 + .../utils/teamProvisioningPresentation.ts | 97 +++++ src/shared/types/api.ts | 4 + src/shared/types/team.ts | 8 + .../team/TeamProvisioningService.test.ts | 60 +++ .../team/TeamProvisioningBanner.test.ts | 53 +++ test/renderer/store/teamSlice.test.ts | 41 ++ .../teamProvisioningPresentation.test.ts | 122 ++++++ 15 files changed, 912 insertions(+), 28 deletions(-) diff --git a/src/main/ipc/teams.ts b/src/main/ipc/teams.ts index 9165f98d..c754e437 100644 --- a/src/main/ipc/teams.ts +++ b/src/main/ipc/teams.ts @@ -60,6 +60,7 @@ import { TEAM_RESTART_MEMBER, TEAM_RESTORE, TEAM_RESTORE_TASK, + TEAM_RETRY_FAILED_OPENCODE_SECONDARY_LANES, TEAM_SAVE_TASK_ATTACHMENT, TEAM_SEND_MESSAGE, TEAM_SET_CHANGE_PRESENCE_TRACKING, @@ -188,6 +189,7 @@ import type { MemberLogSummary, MemberSpawnStatusesSnapshot, MessagesPage, + RetryFailedOpenCodeSecondaryLanesResult, SendMessageRequest, SendMessageResult, TaskAttachmentMeta, @@ -708,6 +710,10 @@ export function registerTeamHandlers(ipcMain: IpcMain): void { ipcMain.handle(TEAM_LEAD_CONTEXT, handleLeadContext); ipcMain.handle(TEAM_MEMBER_SPAWN_STATUSES, handleMemberSpawnStatuses); ipcMain.handle(TEAM_GET_AGENT_RUNTIME, handleGetAgentRuntime); + ipcMain.handle( + TEAM_RETRY_FAILED_OPENCODE_SECONDARY_LANES, + handleRetryFailedOpenCodeSecondaryLanes + ); ipcMain.handle(TEAM_RESTART_MEMBER, handleRestartMember); ipcMain.handle(TEAM_SKIP_MEMBER_FOR_LAUNCH, handleSkipMemberForLaunch); ipcMain.handle(TEAM_SOFT_DELETE_TASK, handleSoftDeleteTask); @@ -789,6 +795,7 @@ export function removeTeamHandlers(ipcMain: IpcMain): void { ipcMain.removeHandler(TEAM_LEAD_CONTEXT); ipcMain.removeHandler(TEAM_MEMBER_SPAWN_STATUSES); ipcMain.removeHandler(TEAM_GET_AGENT_RUNTIME); + ipcMain.removeHandler(TEAM_RETRY_FAILED_OPENCODE_SECONDARY_LANES); ipcMain.removeHandler(TEAM_RESTART_MEMBER); ipcMain.removeHandler(TEAM_SKIP_MEMBER_FOR_LAUNCH); ipcMain.removeHandler(TEAM_SOFT_DELETE_TASK); @@ -3833,6 +3840,19 @@ async function handleRestartMember( ); } +async function handleRetryFailedOpenCodeSecondaryLanes( + _event: IpcMainInvokeEvent, + teamName: unknown +): Promise> { + const validatedTeamName = validateTeamName(teamName); + if (!validatedTeamName.valid) { + return { success: false, error: validatedTeamName.error ?? 'Invalid teamName' }; + } + return wrapTeamHandler('retryFailedOpenCodeSecondaryLanes', async () => + getTeamProvisioningService().retryFailedOpenCodeSecondaryLanes(validatedTeamName.value!) + ); +} + async function handleSkipMemberForLaunch( _event: IpcMainInvokeEvent, teamName: unknown, diff --git a/src/main/services/team/TeamProvisioningService.ts b/src/main/services/team/TeamProvisioningService.ts index a0a175dc..61435cf8 100644 --- a/src/main/services/team/TeamProvisioningService.ts +++ b/src/main/services/team/TeamProvisioningService.ts @@ -367,6 +367,7 @@ import type { PersistedTeamLaunchSnapshot, PersistedTeamLaunchSummary, ProviderModelLaunchIdentity, + RetryFailedOpenCodeSecondaryLanesResult, TaskRef, TeamAgentRuntimeBackendType, TeamAgentRuntimeDiagnosticSeverity, @@ -1699,6 +1700,16 @@ interface MixedSecondaryRuntimeLaneState { launchFinishedAtMs?: number; } +interface OpenCodeSecondaryRetryCandidate { + memberName: string; + laneId: string; +} + +interface OpenCodeSecondaryRetryOutcome { + launchState: MemberLaunchState; + reason?: string; +} + function formatOpenCodeLaneTimingMs(value: number | null | undefined): string { return typeof value === 'number' && Number.isFinite(value) ? `${Math.max(0, Math.round(value))}ms` @@ -5043,6 +5054,10 @@ export class TeamProvisioningService { private readonly launchStateStore = new TeamLaunchStateStore(); private readonly launchStateStoreQueue = new Map>(); private readonly launchStateWrittenRunIdByTeam = new Map(); + private readonly failedOpenCodeSecondaryRetryInFlightByTeam = new Map< + string, + Promise + >(); private readonly memberLogsFinder: TeamMemberLogsFinder; private readonly transcriptProjectResolver: TeamTranscriptProjectResolver; private teamChangeEmitter: ((event: TeamChangeEvent) => void) | null = null; @@ -12190,6 +12205,353 @@ export class TeamProvisioningService { } } + async retryFailedOpenCodeSecondaryLanes( + teamName: string + ): Promise { + const existing = this.failedOpenCodeSecondaryRetryInFlightByTeam.get(teamName); + if (existing) { + return existing; + } + + const retry = this.retryFailedOpenCodeSecondaryLanesNow(teamName).finally(() => { + this.failedOpenCodeSecondaryRetryInFlightByTeam.delete(teamName); + }); + this.failedOpenCodeSecondaryRetryInFlightByTeam.set(teamName, retry); + return retry; + } + + private async retryFailedOpenCodeSecondaryLanesNow( + teamName: string + ): Promise { + const run = this.getMutableAliveRunOrThrow(teamName); + if (this.getProvisioningRunId(teamName)) { + throw new Error('Team launch is still in progress'); + } + + const result: RetryFailedOpenCodeSecondaryLanesResult = { + attempted: [], + confirmed: [], + pending: [], + failed: [], + skipped: [], + }; + const candidates = await this.collectFailedOpenCodeSecondaryRetryCandidates(run); + + for (const candidate of candidates) { + if (!this.isCurrentTrackedRun(run) || run.processKilled || run.cancelRequested) { + result.skipped.push({ + memberName: candidate.memberName, + reason: 'Team stopped during retry', + }); + continue; + } + + try { + await this.reattachOpenCodeOwnedMemberLane(teamName, candidate.memberName, { + reason: 'manual_restart', + }); + result.attempted.push(candidate.memberName); + + const outcome = await this.readOpenCodeSecondaryRetryOutcome( + run, + candidate.memberName, + candidate.laneId + ); + if (outcome.launchState === 'confirmed_alive') { + result.confirmed.push(candidate.memberName); + } else if (outcome.launchState === 'failed_to_start') { + result.failed.push({ + memberName: candidate.memberName, + error: outcome.reason ?? 'OpenCode retry failed', + }); + } else if (outcome.launchState === 'skipped_for_launch') { + result.skipped.push({ + memberName: candidate.memberName, + reason: outcome.reason ?? 'Teammate is skipped for this launch', + }); + } else { + result.pending.push(candidate.memberName); + } + } catch (error) { + result.failed.push({ + memberName: candidate.memberName, + error: error instanceof Error ? error.message : String(error), + }); + } + } + + await this.notifyLeadAboutConfirmedOpenCodeRetries(run, result); + return result; + } + + private async collectFailedOpenCodeSecondaryRetryCandidates( + run: ProvisioningRun + ): Promise { + const teamName = run.teamName; + const leadProviderId = resolveTeamProviderId(run.request.providerId); + if (leadProviderId === 'opencode') { + throw new Error( + 'Retrying OpenCode secondary lanes is only supported for mixed teams with a non-OpenCode lead.' + ); + } + if (!this.getOpenCodeRuntimeAdapter()) { + throw new Error('OpenCode runtime adapter is not available for secondary lane retry.'); + } + + const config = await this.readConfigForStrictDecision(teamName); + if (!config) { + throw new Error(`Team "${teamName}" configuration is no longer available`); + } + const metaMembers = await this.membersMetaStore.getMembers(teamName).catch(() => []); + const persistedSnapshot = await this.launchStateStore.read(teamName).catch(() => null); + + const names = new Set(); + for (const member of config.members ?? []) { + const name = member.name?.trim(); + if (name) { + names.add(name); + } + } + for (const member of metaMembers) { + const name = member.name?.trim(); + if (name) { + names.add(name); + } + } + for (const lane of run.mixedSecondaryLanes ?? []) { + const name = lane.member.name?.trim(); + if (name) { + names.add(name); + } + } + for (const name of persistedSnapshot?.expectedMembers ?? []) { + if (name.trim()) { + names.add(name.trim()); + } + } + for (const name of Object.keys(persistedSnapshot?.members ?? {})) { + if (name.trim()) { + names.add(name.trim()); + } + } + + const candidates: OpenCodeSecondaryRetryCandidate[] = []; + for (const memberName of [...names].sort((left, right) => left.localeCompare(right))) { + const configuredMember = this.resolveEffectiveConfiguredMember( + config.members ?? [], + metaMembers, + memberName + ); + if (!configuredMember || configuredMember.removedAt) { + continue; + } + if (isLeadMember({ name: memberName, agentType: configuredMember.agentType })) { + continue; + } + if (normalizeOptionalTeamProviderId(configuredMember.providerId) !== 'opencode') { + continue; + } + + const laneIdentity = buildPlannedMemberLaneIdentity({ + leadProviderId, + member: { + name: memberName, + providerId: 'opencode', + }, + }); + if ( + laneIdentity.laneKind !== 'secondary' || + laneIdentity.laneOwnerProviderId !== 'opencode' + ) { + continue; + } + + const existingLane = (run.mixedSecondaryLanes ?? []).find( + (lane) => + lane.laneId === laneIdentity.laneId || + matchesTeamMemberIdentity(lane.member.name, memberName) + ); + const liveEntry = run.memberSpawnStatuses.get(memberName); + const persistedMember = + persistedSnapshot?.members[memberName] ?? + Object.values(persistedSnapshot?.members ?? {}).find( + (member) => member.laneId === laneIdentity.laneId + ); + + if ( + this.isRetryableFailedOpenCodeSecondaryLane({ + liveEntry, + persistedMember, + existingLane, + }) + ) { + candidates.push({ memberName, laneId: laneIdentity.laneId }); + } + } + return candidates; + } + + private isRetryableFailedOpenCodeSecondaryLane(input: { + liveEntry?: MemberSpawnStatusEntry; + persistedMember?: PersistedTeamLaunchMemberState; + existingLane?: MixedSecondaryRuntimeLaneState; + }): boolean { + const { liveEntry, persistedMember, existingLane } = input; + if (existingLane?.state === 'queued' || existingLane?.state === 'launching') { + return false; + } + if ( + liveEntry?.launchState === 'skipped_for_launch' || + liveEntry?.skippedForLaunch === true || + persistedMember?.launchState === 'skipped_for_launch' || + persistedMember?.skippedForLaunch === true + ) { + return false; + } + if ( + liveEntry?.launchState === 'runtime_pending_permission' || + liveEntry?.launchState === 'runtime_pending_bootstrap' || + persistedMember?.launchState === 'runtime_pending_permission' || + persistedMember?.launchState === 'runtime_pending_bootstrap' || + (liveEntry?.pendingPermissionRequestIds?.length ?? 0) > 0 || + (persistedMember?.pendingPermissionRequestIds?.length ?? 0) > 0 + ) { + return false; + } + if (liveEntry?.launchState === 'starting' || liveEntry?.status === 'spawning') { + return false; + } + if ( + liveEntry?.launchState === 'confirmed_alive' || + liveEntry?.bootstrapConfirmed === true || + persistedMember?.launchState === 'confirmed_alive' || + persistedMember?.bootstrapConfirmed === true + ) { + return false; + } + + return ( + liveEntry?.launchState === 'failed_to_start' || + liveEntry?.status === 'error' || + persistedMember?.launchState === 'failed_to_start' || + persistedMember?.hardFailure === true + ); + } + + private async readOpenCodeSecondaryRetryOutcome( + run: ProvisioningRun, + memberName: string, + laneId: string + ): Promise { + const lane = (run.mixedSecondaryLanes ?? []).find( + (candidate) => + candidate.laneId === laneId || matchesTeamMemberIdentity(candidate.member.name, memberName) + ); + const memberEvidence = + lane?.result?.members[memberName] ?? + Object.values(lane?.result?.members ?? {}).find((member) => + matchesTeamMemberIdentity(member.memberName, memberName) + ); + const persistedSnapshot = await this.launchStateStore.read(run.teamName).catch(() => null); + const persistedMember = + persistedSnapshot?.members[memberName] ?? + Object.values(persistedSnapshot?.members ?? {}).find((member) => member.laneId === laneId); + const liveEntry = run.memberSpawnStatuses.get(memberName); + + if ( + memberEvidence?.launchState === 'confirmed_alive' || + memberEvidence?.bootstrapConfirmed === true || + liveEntry?.launchState === 'confirmed_alive' || + liveEntry?.bootstrapConfirmed === true || + persistedMember?.launchState === 'confirmed_alive' || + persistedMember?.bootstrapConfirmed === true + ) { + return { launchState: 'confirmed_alive' }; + } + + if ( + liveEntry?.launchState === 'skipped_for_launch' || + liveEntry?.skippedForLaunch === true || + persistedMember?.launchState === 'skipped_for_launch' || + persistedMember?.skippedForLaunch === true + ) { + return { + launchState: 'skipped_for_launch', + reason: liveEntry?.skipReason ?? persistedMember?.skipReason, + }; + } + + if ( + memberEvidence?.launchState === 'failed_to_start' || + memberEvidence?.hardFailure === true || + liveEntry?.launchState === 'failed_to_start' || + liveEntry?.status === 'error' || + persistedMember?.launchState === 'failed_to_start' || + persistedMember?.hardFailure === true + ) { + return { + launchState: 'failed_to_start', + reason: this.selectOpenCodeSecondaryRetryFailureReason({ + memberEvidence, + liveEntry, + persistedMember, + }), + }; + } + + return { + launchState: + memberEvidence?.launchState ?? + liveEntry?.launchState ?? + persistedMember?.launchState ?? + 'runtime_pending_bootstrap', + }; + } + + private selectOpenCodeSecondaryRetryFailureReason(input: { + memberEvidence?: TeamRuntimeMemberLaunchEvidence; + liveEntry?: MemberSpawnStatusEntry; + persistedMember?: PersistedTeamLaunchMemberState; + }): string | undefined { + const diagnostics = [ + input.memberEvidence?.hardFailureReason, + input.memberEvidence?.runtimeDiagnostic, + ...(input.memberEvidence?.diagnostics ?? []), + input.liveEntry?.hardFailureReason, + input.liveEntry?.runtimeDiagnostic, + input.liveEntry?.error, + input.persistedMember?.hardFailureReason, + input.persistedMember?.runtimeDiagnostic, + ]; + return diagnostics + .find( + (diagnostic): diagnostic is string => + typeof diagnostic === 'string' && diagnostic.trim().length > 0 + ) + ?.trim(); + } + + private async notifyLeadAboutConfirmedOpenCodeRetries( + run: ProvisioningRun, + result: RetryFailedOpenCodeSecondaryLanesResult + ): Promise { + if (result.confirmed.length === 0) { + return; + } + const confirmedNames = result.confirmed.map((name) => `@${name}`).join(', '); + const message = [ + `Системное замечание: повторный запуск OpenCode-тиммейтов подтверждён: ${confirmedNames}.`, + `Их можно снова считать доступными.`, + ].join(' '); + await this.sendMessageToRun(run, message).catch((error: unknown) => + logger.warn( + `[${run.teamName}] failed to send OpenCode retry recovery notice to lead: ${ + error instanceof Error ? error.message : String(error) + }` + ) + ); + } + async skipMemberForLaunch(teamName: string, memberName: string): Promise { const normalizedMemberName = memberName.trim(); if (!normalizedMemberName) { diff --git a/src/preload/constants/ipcChannels.ts b/src/preload/constants/ipcChannels.ts index 43fadd19..9264004d 100644 --- a/src/preload/constants/ipcChannels.ts +++ b/src/preload/constants/ipcChannels.ts @@ -394,6 +394,9 @@ export const TEAM_GET_AGENT_RUNTIME = 'team:getAgentRuntime'; /** Restart a specific teammate runtime */ export const TEAM_RESTART_MEMBER = 'team:restartMember'; +/** Retry failed OpenCode-owned secondary runtime lanes */ +export const TEAM_RETRY_FAILED_OPENCODE_SECONDARY_LANES = 'team:retryFailedOpenCodeSecondaryLanes'; + /** Skip a failed teammate for the current launch */ export const TEAM_SKIP_MEMBER_FOR_LAUNCH = 'team:skipMemberForLaunch'; diff --git a/src/preload/index.ts b/src/preload/index.ts index c248d009..0f3b5abb 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -168,6 +168,7 @@ import { TEAM_RESTART_MEMBER, TEAM_RESTORE, TEAM_RESTORE_TASK, + TEAM_RETRY_FAILED_OPENCODE_SECONDARY_LANES, TEAM_SAVE_TASK_ATTACHMENT, TEAM_SEND_MESSAGE, TEAM_SET_CHANGE_PRESENCE_TRACKING, @@ -283,6 +284,7 @@ import type { ProjectBranchChangeEvent, RejectResult, ReplaceMembersRequest, + RetryFailedOpenCodeSecondaryLanesResult, Schedule, ScheduleChangeEvent, ScheduleRun, @@ -1118,6 +1120,12 @@ const electronAPI: ElectronAPI = { getTeamAgentRuntime: async (teamName: string) => { return invokeIpcWithResult(TEAM_GET_AGENT_RUNTIME, teamName); }, + retryFailedOpenCodeSecondaryLanes: async (teamName: string) => { + return invokeIpcWithResult( + TEAM_RETRY_FAILED_OPENCODE_SECONDARY_LANES, + teamName + ); + }, restartMember: async (teamName: string, memberName: string) => { return invokeIpcWithResult(TEAM_RESTART_MEMBER, teamName, memberName); }, diff --git a/src/renderer/api/httpClient.ts b/src/renderer/api/httpClient.ts index 2a2ad181..49b67a91 100644 --- a/src/renderer/api/httpClient.ts +++ b/src/renderer/api/httpClient.ts @@ -981,6 +981,9 @@ export class HttpAPIClient implements ElectronAPI { restartMember: async (): Promise => { throw new Error('Member restart is not available in browser mode'); }, + retryFailedOpenCodeSecondaryLanes: async () => { + throw new Error('OpenCode secondary retry is not available in browser mode'); + }, skipMemberForLaunch: async (): Promise => { throw new Error('Member launch skip is not available in browser mode'); }, diff --git a/src/renderer/components/team/TeamProvisioningPanel.tsx b/src/renderer/components/team/TeamProvisioningPanel.tsx index 69f29554..4a2a24df 100644 --- a/src/renderer/components/team/TeamProvisioningPanel.tsx +++ b/src/renderer/components/team/TeamProvisioningPanel.tsx @@ -7,6 +7,8 @@ import { X } from 'lucide-react'; import { ProvisioningProgressBlock } from './ProvisioningProgressBlock'; import { useTeamProvisioningPresentation } from './useTeamProvisioningPresentation'; +import type { RetryFailedOpenCodeSecondaryLanesResult } from '@shared/types'; + export interface TeamProvisioningPanelProps { teamName: string; surface?: 'raised' | 'flat'; @@ -15,6 +17,27 @@ export interface TeamProvisioningPanelProps { defaultLogsOpen?: boolean; } +function formatOpenCodeSecondaryRetryResult( + result: RetryFailedOpenCodeSecondaryLanesResult +): string { + const parts: string[] = []; + if (result.confirmed.length > 0) { + parts.push(`${result.confirmed.length} confirmed`); + } + if (result.pending.length > 0) { + parts.push(`${result.pending.length} pending`); + } + if (result.failed.length > 0) { + parts.push(`${result.failed.length} failed`); + } + if (result.skipped.length > 0) { + parts.push(`${result.skipped.length} skipped`); + } + return parts.length > 0 + ? `OpenCode retry: ${parts.join(', ')}` + : 'No retryable OpenCode failures'; +} + export const TeamProvisioningPanel = memo(function TeamProvisioningPanel({ teamName, surface = 'flat', @@ -22,13 +45,19 @@ export const TeamProvisioningPanel = memo(function TeamProvisioningPanel({ className, defaultLogsOpen, }: TeamProvisioningPanelProps): React.JSX.Element | null { - const { presentation, cancelProvisioning, runInstanceKey } = + const { presentation, cancelProvisioning, retryFailedOpenCodeSecondaryLanes, runInstanceKey } = useTeamProvisioningPresentation(teamName); const [dismissed, setDismissed] = useState(false); + const [retryingOpenCode, setRetryingOpenCode] = useState(false); + const [openCodeRetryMessage, setOpenCodeRetryMessage] = useState(null); + const [openCodeRetryError, setOpenCodeRetryError] = useState(null); const lastActiveStepRef = useRef(-1); useEffect(() => { setDismissed(false); + setRetryingOpenCode(false); + setOpenCodeRetryMessage(null); + setOpenCodeRetryError(null); }, [runInstanceKey]); if (!presentation || dismissed) { @@ -40,6 +69,48 @@ export const TeamProvisioningPanel = memo(function TeamProvisioningPanel({ } const showRunningState = presentation.isActive || presentation.hasMembersStillJoining; + const canRetryFailedOpenCode = + !presentation.isActive && + presentation.retryableOpenCodeSecondaryFailedCount > 0 && + Boolean(retryFailedOpenCodeSecondaryLanes); + + const retryOpenCodeAction = canRetryFailedOpenCode ? ( +
+

+ {openCodeRetryError ?? + openCodeRetryMessage ?? + `${presentation.retryableOpenCodeSecondaryFailedCount} failed OpenCode teammate${ + presentation.retryableOpenCodeSecondaryFailedCount === 1 ? '' : 's' + } can be retried.`} +

+ +
+ ) : null; const block = ( ); - if (!presentation.isFailed) { + if (!presentation.isFailed && !retryOpenCodeAction) { return block; } return (
-
-

- {presentation.progress.message} -

- {dismissible ? ( - - ) : null} -
+ {presentation.isFailed ? ( +
+

+ {presentation.progress.message} +

+ {dismissible ? ( + + ) : null} +
+ ) : null} {block} + {retryOpenCodeAction}
); }); diff --git a/src/renderer/components/team/useTeamProvisioningPresentation.ts b/src/renderer/components/team/useTeamProvisioningPresentation.ts index 0cdf27ec..970cc9f8 100644 --- a/src/renderer/components/team/useTeamProvisioningPresentation.ts +++ b/src/renderer/components/team/useTeamProvisioningPresentation.ts @@ -9,22 +9,33 @@ import { buildTeamProvisioningPresentation } from '@renderer/utils/teamProvision import { useShallow } from 'zustand/react/shallow'; import type { TeamProvisioningPresentation } from '@renderer/utils/teamProvisioningPresentation'; +import type { RetryFailedOpenCodeSecondaryLanesResult } from '@shared/types'; export function useTeamProvisioningPresentation(teamName: string): { presentation: TeamProvisioningPresentation | null; cancelProvisioning: ((runId: string) => Promise) | null; + retryFailedOpenCodeSecondaryLanes: + | ((teamName: string) => Promise) + | null; runInstanceKey: string | null; } { - const { progress, cancelProvisioning, teamMembers, memberSpawnStatuses, memberSpawnSnapshot } = - useStore( - useShallow((s) => ({ - progress: getCurrentProvisioningProgressForTeam(s, teamName), - cancelProvisioning: s.cancelProvisioning, - teamMembers: selectTeamMemberSnapshotsForName(s, teamName), - memberSpawnStatuses: s.memberSpawnStatusesByTeam[teamName], - memberSpawnSnapshot: s.memberSpawnSnapshotsByTeam[teamName], - })) - ); + const { + progress, + cancelProvisioning, + retryFailedOpenCodeSecondaryLanes, + teamMembers, + memberSpawnStatuses, + memberSpawnSnapshot, + } = useStore( + useShallow((s) => ({ + progress: getCurrentProvisioningProgressForTeam(s, teamName), + cancelProvisioning: s.cancelProvisioning, + retryFailedOpenCodeSecondaryLanes: s.retryFailedOpenCodeSecondaryLanes, + teamMembers: selectTeamMemberSnapshotsForName(s, teamName), + memberSpawnStatuses: s.memberSpawnStatusesByTeam[teamName], + memberSpawnSnapshot: s.memberSpawnSnapshotsByTeam[teamName], + })) + ); const presentation = useMemo( () => @@ -40,6 +51,7 @@ export function useTeamProvisioningPresentation(teamName: string): { return { presentation, cancelProvisioning, + retryFailedOpenCodeSecondaryLanes: retryFailedOpenCodeSecondaryLanes ?? null, runInstanceKey: progress ? `${teamName}:${progress.runId}:${progress.startedAt}` : null, }; } diff --git a/src/renderer/store/slices/teamSlice.ts b/src/renderer/store/slices/teamSlice.ts index f1cd371d..1c0effeb 100644 --- a/src/renderer/store/slices/teamSlice.ts +++ b/src/renderer/store/slices/teamSlice.ts @@ -46,6 +46,7 @@ import type { NotificationTarget, PersistedTeamLaunchSummary, ResolvedTeamMember, + RetryFailedOpenCodeSecondaryLanesResult, SendMessageRequest, SendMessageResult, TaskChangePresenceState, @@ -2500,6 +2501,9 @@ export interface TeamSlice { memberName: string, role: string | undefined ) => Promise; + retryFailedOpenCodeSecondaryLanes: ( + teamName: string + ) => Promise; addTaskRelationship: ( teamName: string, taskId: string, @@ -4679,6 +4683,19 @@ export const createTeamSlice: StateCreator = (set, } }, + retryFailedOpenCodeSecondaryLanes: async (teamName: string) => { + try { + return await unwrapIpc('team:retryFailedOpenCodeSecondaryLanes', () => + api.teams.retryFailedOpenCodeSecondaryLanes(teamName) + ); + } finally { + await Promise.allSettled([ + get().fetchMemberSpawnStatuses(teamName), + get().fetchTeamAgentRuntime(teamName), + ]); + } + }, + skipMemberForLaunch: async (teamName: string, memberName: string) => { try { await unwrapIpc('team:skipMemberForLaunch', () => diff --git a/src/renderer/utils/teamProvisioningPresentation.ts b/src/renderer/utils/teamProvisioningPresentation.ts index 58b22116..f1b3c2ed 100644 --- a/src/renderer/utils/teamProvisioningPresentation.ts +++ b/src/renderer/utils/teamProvisioningPresentation.ts @@ -10,6 +10,7 @@ import type { MemberSpawnStatusesSnapshot, TeamProvisioningProgress, } from '@shared/types'; +import { isLeadMember } from '@shared/utils/leadDetection'; type MemberSpawnStatusCollection = | Record @@ -69,6 +70,42 @@ function isSkippedSpawnEntry(entry: MemberSpawnStatusEntry | undefined): boolean return entry?.launchState === 'skipped_for_launch' || entry?.skippedForLaunch === true; } +function isOpenCodeSecondaryRetryCandidate(params: { + member: ProvisioningMemberLike | undefined; + entry: MemberSpawnStatusEntry | undefined; +}): boolean { + const { member, entry } = params; + if (!member || !entry) { + return false; + } + if (member.providerId !== 'opencode' || member.removedAt) { + return false; + } + if (isLeadMember({ name: member.name, agentType: member.agentType })) { + return false; + } + if (member.laneKind && member.laneKind !== 'secondary') { + return false; + } + if (member.laneOwnerProviderId && member.laneOwnerProviderId !== 'opencode') { + return false; + } + if ( + entry.launchState === 'skipped_for_launch' || + entry.skippedForLaunch === true || + entry.launchState === 'runtime_pending_permission' || + entry.launchState === 'runtime_pending_bootstrap' || + (entry.pendingPermissionRequestIds?.length ?? 0) > 0 || + entry.launchState === 'starting' || + entry.status === 'spawning' || + entry.launchState === 'confirmed_alive' || + entry.bootstrapConfirmed === true + ) { + return false; + } + return entry.launchState === 'failed_to_start' || entry.status === 'error'; +} + function shouldPreferSnapshotEntryOverLive(params: { liveEntry: MemberSpawnStatusEntry | undefined; snapshotEntry: MemberSpawnStatusEntry | undefined; @@ -480,6 +517,51 @@ function getSkippedSpawnDetails(params: { .sort((left, right) => left.name.localeCompare(right.name)); } +function getRetryableOpenCodeSecondaryFailedNames(params: { + members: readonly ProvisioningMemberLike[]; + memberSpawnStatuses: MemberSpawnStatusCollection; + memberSpawnSnapshotStatuses?: MemberSpawnStatusesSnapshot['statuses']; + memberSpawnSnapshotUpdatedAt?: string; +}): string[] { + const membersByName = new Map( + params.members + .map((member) => [member.name.trim(), member] as const) + .filter(([name]) => name.length > 0) + ); + const names = new Set(membersByName.keys()); + if (params.memberSpawnStatuses instanceof Map) { + for (const name of params.memberSpawnStatuses.keys()) { + names.add(name); + } + } else if (params.memberSpawnStatuses) { + for (const name of Object.keys(params.memberSpawnStatuses)) { + names.add(name); + } + } + for (const name of Object.keys(params.memberSpawnSnapshotStatuses ?? {})) { + names.add(name); + } + + return [...names] + .filter((name) => { + const liveEntry = + params.memberSpawnStatuses instanceof Map + ? params.memberSpawnStatuses.get(name) + : params.memberSpawnStatuses?.[name]; + const snapshotEntry = params.memberSpawnSnapshotStatuses?.[name]; + const entry = getPreferredSpawnEntry({ + liveEntry, + snapshotEntry, + snapshotUpdatedAt: params.memberSpawnSnapshotUpdatedAt, + }); + return isOpenCodeSecondaryRetryCandidate({ + member: membersByName.get(name), + entry, + }); + }) + .sort((left, right) => left.localeCompare(right)); +} + function normalizeFailureReason(reason: string): string { return reason.replace(/\s+/g, ' ').trim(); } @@ -581,6 +663,8 @@ export interface TeamProvisioningPresentation { allTeammatesConfirmedAlive: boolean; hasMembersStillJoining: boolean; remainingJoinCount: number; + retryableOpenCodeSecondaryFailedCount: number; + retryableOpenCodeSecondaryFailedNames: string[]; panelTitle: string; panelMessage?: string | null; panelMessageSeverity?: 'error' | 'warning' | 'info'; @@ -674,6 +758,13 @@ export function buildTeamProvisioningPresentation({ memberSpawnSnapshotStatuses: memberSpawnSnapshot?.statuses, memberSpawnSnapshotUpdatedAt: memberSpawnSnapshot?.updatedAt, }); + const retryableOpenCodeSecondaryFailedNames = getRetryableOpenCodeSecondaryFailedNames({ + members, + memberSpawnStatuses, + memberSpawnSnapshotStatuses: memberSpawnSnapshot?.statuses, + memberSpawnSnapshotUpdatedAt: memberSpawnSnapshot?.updatedAt, + }); + const retryableOpenCodeSecondaryFailedCount = retryableOpenCodeSecondaryFailedNames.length; const { allTeammatesConfirmedAlive, hasMembersStillJoining, remainingJoinCount } = getLaunchJoinState({ @@ -712,6 +803,8 @@ export function buildTeamProvisioningPresentation({ allTeammatesConfirmedAlive, hasMembersStillJoining, remainingJoinCount, + retryableOpenCodeSecondaryFailedCount, + retryableOpenCodeSecondaryFailedNames, panelTitle: 'Launch failed', panelMessage: progress.error ?? failedSpawnPanelMessage ?? genericFailedSpawnPanelMessage, panelTone: 'error', @@ -800,6 +893,8 @@ export function buildTeamProvisioningPresentation({ allTeammatesConfirmedAlive, hasMembersStillJoining, remainingJoinCount, + retryableOpenCodeSecondaryFailedCount, + retryableOpenCodeSecondaryFailedNames, panelTitle: 'Launch details', panelMessage: failedSpawnCount > 0 || skippedSpawnCount > 0 || hasMembersStillJoining @@ -875,6 +970,8 @@ export function buildTeamProvisioningPresentation({ allTeammatesConfirmedAlive, hasMembersStillJoining, remainingJoinCount, + retryableOpenCodeSecondaryFailedCount, + retryableOpenCodeSecondaryFailedNames, panelTitle: openCodeSecondaryWaitPhrase ? 'Core team ready' : 'Launching team', panelMessage: failedSpawnCount > 0 diff --git a/src/shared/types/api.ts b/src/shared/types/api.ts index d679956b..42a19156 100644 --- a/src/shared/types/api.ts +++ b/src/shared/types/api.ts @@ -60,6 +60,7 @@ import type { MessagesPage, ProjectBranchChangeEvent, ReplaceMembersRequest, + RetryFailedOpenCodeSecondaryLanesResult, SendMessageRequest, SendMessageResult, TaskAttachmentMeta, @@ -555,6 +556,9 @@ export interface TeamsAPI { getLeadContext: (teamName: string) => Promise; getMemberSpawnStatuses: (teamName: string) => Promise; getTeamAgentRuntime: (teamName: string) => Promise; + retryFailedOpenCodeSecondaryLanes: ( + teamName: string + ) => Promise; restartMember: (teamName: string, memberName: string) => Promise; skipMemberForLaunch: (teamName: string, memberName: string) => Promise; softDeleteTask: (teamName: string, taskId: string) => Promise; diff --git a/src/shared/types/team.ts b/src/shared/types/team.ts index 0ebede0a..a9224eae 100644 --- a/src/shared/types/team.ts +++ b/src/shared/types/team.ts @@ -1079,6 +1079,14 @@ export interface MemberSpawnStatusesSnapshot { source?: 'live' | 'persisted' | 'merged'; } +export interface RetryFailedOpenCodeSecondaryLanesResult { + attempted: string[]; + confirmed: string[]; + pending: string[]; + failed: Array<{ memberName: string; error: string }>; + skipped: Array<{ memberName: string; reason: string }>; +} + export type MemberSpawnLivenessSource = 'heartbeat' | 'process'; export type TeamAgentRuntimeBackendType = 'lead' | 'tmux' | 'iterm2' | 'in-process' | 'process'; diff --git a/test/main/services/team/TeamProvisioningService.test.ts b/test/main/services/team/TeamProvisioningService.test.ts index daced49e..5f597bf6 100644 --- a/test/main/services/team/TeamProvisioningService.test.ts +++ b/test/main/services/team/TeamProvisioningService.test.ts @@ -16195,4 +16195,64 @@ describe('TeamProvisioningService', () => { }); expect(run.expectedMembers).toEqual(['alice', 'jack']); }); + + it('bulk retries failed OpenCode secondary lanes sequentially and classifies outcomes', async () => { + const svc = new TeamProvisioningService(); + const run = createMemberSpawnRun({ + teamName: 'mixed-retry-team', + runId: 'run-mixed-retry', + expectedMembers: ['alice', 'tom', 'nova'], + }); + run.isLaunch = true; + run.provisioningComplete = true; + + (svc as any).runs.set(run.runId, run); + (svc as any).aliveRunByTeam.set(run.teamName, run.runId); + + vi.spyOn(svc as any, 'collectFailedOpenCodeSecondaryRetryCandidates').mockResolvedValue([ + { memberName: 'alice', laneId: 'secondary:opencode:alice' }, + { memberName: 'tom', laneId: 'secondary:opencode:tom' }, + { memberName: 'nova', laneId: 'secondary:opencode:nova' }, + ]); + const reattach = vi + .spyOn(svc as any, 'reattachOpenCodeOwnedMemberLane') + .mockResolvedValueOnce(undefined) + .mockResolvedValueOnce(undefined) + .mockRejectedValueOnce(new Error('OpenCode bridge crashed')); + vi.spyOn(svc as any, 'readOpenCodeSecondaryRetryOutcome') + .mockResolvedValueOnce({ launchState: 'confirmed_alive' }) + .mockResolvedValueOnce({ + launchState: 'failed_to_start', + reason: 'Latest assistant message reported OpenRouter credits exhausted', + }); + const notify = vi + .spyOn(svc as any, 'notifyLeadAboutConfirmedOpenCodeRetries') + .mockResolvedValue(undefined); + + const result = await svc.retryFailedOpenCodeSecondaryLanes(run.teamName); + + expect(reattach).toHaveBeenNthCalledWith(1, run.teamName, 'alice', { + reason: 'manual_restart', + }); + expect(reattach).toHaveBeenNthCalledWith(2, run.teamName, 'tom', { + reason: 'manual_restart', + }); + expect(reattach).toHaveBeenNthCalledWith(3, run.teamName, 'nova', { + reason: 'manual_restart', + }); + expect(result).toEqual({ + attempted: ['alice', 'tom'], + confirmed: ['alice'], + pending: [], + failed: [ + { + memberName: 'tom', + error: 'Latest assistant message reported OpenRouter credits exhausted', + }, + { memberName: 'nova', error: 'OpenCode bridge crashed' }, + ], + skipped: [], + }); + expect(notify).toHaveBeenCalledWith(run, result); + }); }); diff --git a/test/renderer/components/team/TeamProvisioningBanner.test.ts b/test/renderer/components/team/TeamProvisioningBanner.test.ts index a9af3989..72e42e9e 100644 --- a/test/renderer/components/team/TeamProvisioningBanner.test.ts +++ b/test/renderer/components/team/TeamProvisioningBanner.test.ts @@ -5,6 +5,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; const storeState = { progress: null as Record | null, cancelProvisioning: vi.fn(), + retryFailedOpenCodeSecondaryLanes: vi.fn(), selectedTeamName: 'northstar-core', selectedTeamData: { members: [ @@ -106,6 +107,13 @@ describe('TeamProvisioningBanner launch-step alignment', () => { cliLogsTail: '', assistantOutput: '', }; + storeState.retryFailedOpenCodeSecondaryLanes.mockResolvedValue({ + attempted: [], + confirmed: [], + pending: [], + failed: [], + skipped: [], + }); storeState.memberSpawnStatusesByTeam['northstar-core'] = {}; storeState.selectedTeamData.members = [ { name: 'team-lead', agentType: 'team-lead' }, @@ -408,6 +416,51 @@ describe('TeamProvisioningBanner launch-step alignment', () => { }); }); + it('renders a bulk retry action for failed OpenCode secondary teammates', async () => { + vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true); + storeState.selectedTeamData.members = [ + { name: 'team-lead', agentType: 'team-lead', providerId: 'anthropic' }, + { + name: 'alice', + agentType: 'developer', + providerId: 'opencode', + laneKind: 'secondary', + laneOwnerProviderId: 'opencode', + }, + ]; + storeState.teamDataCacheByName['northstar-core'] = { + members: [...storeState.selectedTeamData.members], + }; + storeState.memberSpawnStatusesByTeam['northstar-core'] = { + alice: { + status: 'error', + launchState: 'failed_to_start', + updatedAt: '2026-04-09T10:00:00.000Z', + runtimeAlive: false, + bootstrapConfirmed: false, + hardFailure: true, + hardFailureReason: 'OpenRouter credits exhausted', + agentToolAccepted: false, + }, + } as Record; + + const host = document.createElement('div'); + document.body.appendChild(host); + const root = createRoot(host); + + await act(async () => { + root.render(React.createElement(TeamProvisioningBanner, { teamName: 'northstar-core' })); + await Promise.resolve(); + }); + + expect(host.textContent).toContain('Retry failed OpenCode teammates'); + + await act(async () => { + root.unmount(); + await Promise.resolve(); + }); + }); + it('uses info severity while runtimes are online but teammate contact is still pending', async () => { vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true); storeState.memberSpawnSnapshotsByTeam['northstar-core'] = { diff --git a/test/renderer/store/teamSlice.test.ts b/test/renderer/store/teamSlice.test.ts index 670c550b..12ad5420 100644 --- a/test/renderer/store/teamSlice.test.ts +++ b/test/renderer/store/teamSlice.test.ts @@ -34,6 +34,7 @@ const hoisted = vi.hoisted(() => ({ restoreTeam: vi.fn(), permanentlyDeleteTeam: vi.fn(), sendMessage: vi.fn(), + retryFailedOpenCodeSecondaryLanes: vi.fn(), restartMember: vi.fn(), skipMemberForLaunch: vi.fn(), requestReview: vi.fn(), @@ -69,6 +70,7 @@ vi.mock('@renderer/api', () => ({ restoreTeam: hoisted.restoreTeam, permanentlyDeleteTeam: hoisted.permanentlyDeleteTeam, sendMessage: hoisted.sendMessage, + retryFailedOpenCodeSecondaryLanes: hoisted.retryFailedOpenCodeSecondaryLanes, restartMember: hoisted.restartMember, skipMemberForLaunch: hoisted.skipMemberForLaunch, requestReview: hoisted.requestReview, @@ -328,6 +330,13 @@ describe('teamSlice actions', () => { hoisted.deleteTeam.mockResolvedValue(undefined); hoisted.restoreTeam.mockResolvedValue(undefined); hoisted.permanentlyDeleteTeam.mockResolvedValue(undefined); + hoisted.retryFailedOpenCodeSecondaryLanes.mockResolvedValue({ + attempted: [], + confirmed: [], + pending: [], + failed: [], + skipped: [], + }); hoisted.restartMember.mockResolvedValue(undefined); hoisted.skipMemberForLaunch.mockResolvedValue(undefined); }); @@ -3394,6 +3403,38 @@ describe('teamSlice actions', () => { expect(store.getState().teamAgentRuntimeByTeam['my-team']).toEqual(createRuntimeSnapshot()); }); + it('retryFailedOpenCodeSecondaryLanes refreshes only spawn statuses and runtime snapshot', async () => { + const store = createSliceStore(); + const refreshSpawnStatuses = vi.fn(async (_teamName: string) => undefined); + const refreshRuntimeSnapshot = vi.fn(async (_teamName: string) => undefined); + const refreshTeamData = vi.fn(async (_teamName: string) => undefined); + const fetchTeams = vi.fn(async () => undefined); + store.setState({ + fetchMemberSpawnStatuses: refreshSpawnStatuses, + fetchTeamAgentRuntime: refreshRuntimeSnapshot, + refreshTeamData, + fetchTeams, + }); + hoisted.retryFailedOpenCodeSecondaryLanes.mockResolvedValueOnce({ + attempted: ['alice'], + confirmed: [], + pending: [], + failed: [{ memberName: 'alice', error: 'OpenRouter credits exhausted' }], + skipped: [], + }); + + const result = await store.getState().retryFailedOpenCodeSecondaryLanes('my-team'); + + expect(result.failed).toEqual([ + { memberName: 'alice', error: 'OpenRouter credits exhausted' }, + ]); + expect(hoisted.retryFailedOpenCodeSecondaryLanes).toHaveBeenCalledWith('my-team'); + expect(refreshSpawnStatuses).toHaveBeenCalledWith('my-team'); + expect(refreshRuntimeSnapshot).toHaveBeenCalledWith('my-team'); + expect(refreshTeamData).not.toHaveBeenCalled(); + expect(fetchTeams).not.toHaveBeenCalled(); + }); + it('restartMember refreshes spawn statuses and runtime snapshot even when restart fails', async () => { const store = createSliceStore(); const refreshSpawnStatuses = vi.fn(async (_teamName: string) => undefined); diff --git a/test/renderer/utils/teamProvisioningPresentation.test.ts b/test/renderer/utils/teamProvisioningPresentation.test.ts index 38f03e30..569fd476 100644 --- a/test/renderer/utils/teamProvisioningPresentation.test.ts +++ b/test/renderer/utils/teamProvisioningPresentation.test.ts @@ -91,6 +91,128 @@ describe('buildTeamProvisioningPresentation', () => { expect(presentation?.defaultLiveOutputOpen).toBe(false); }); + it('counts retryable failed OpenCode secondary teammates conservatively', () => { + const presentation = buildTeamProvisioningPresentation({ + progress: { + runId: 'run-opencode-retry', + teamName: 'mixed-team', + state: 'ready', + startedAt: '2026-04-13T10:00:00.000Z', + updatedAt: '2026-04-13T10:00:08.000Z', + message: 'Launch completed with teammate errors', + messageSeverity: 'warning', + pid: 4321, + cliLogsTail: '', + assistantOutput: '', + }, + members: [ + { + name: 'team-lead', + agentType: 'team-lead', + providerId: 'anthropic', + }, + { + name: 'alice', + agentType: 'developer', + providerId: 'opencode', + laneKind: 'secondary', + laneOwnerProviderId: 'opencode', + }, + { + name: 'bob', + agentType: 'developer', + providerId: 'anthropic', + laneKind: 'primary', + }, + ], + memberSpawnStatuses: { + alice: { + status: 'error', + launchState: 'failed_to_start', + hardFailureReason: 'OpenRouter credits exhausted', + updatedAt: '2026-04-13T10:00:05.000Z', + runtimeAlive: false, + bootstrapConfirmed: false, + hardFailure: true, + agentToolAccepted: false, + }, + bob: { + status: 'error', + launchState: 'failed_to_start', + hardFailureReason: 'Primary lane failed', + updatedAt: '2026-04-13T10:00:05.000Z', + runtimeAlive: false, + bootstrapConfirmed: false, + hardFailure: true, + agentToolAccepted: false, + }, + }, + memberSpawnSnapshot: undefined, + }); + + expect(presentation?.retryableOpenCodeSecondaryFailedNames).toEqual(['alice']); + expect(presentation?.retryableOpenCodeSecondaryFailedCount).toBe(1); + }); + + it('does not count skipped or permission-blocked OpenCode failures as bulk retry candidates', () => { + const presentation = buildTeamProvisioningPresentation({ + progress: { + runId: 'run-opencode-no-retry', + teamName: 'mixed-team', + state: 'ready', + startedAt: '2026-04-13T10:00:00.000Z', + updatedAt: '2026-04-13T10:00:08.000Z', + message: 'Launch completed with teammate errors', + messageSeverity: 'warning', + pid: 4321, + cliLogsTail: '', + assistantOutput: '', + }, + members: [ + { + name: 'alice', + agentType: 'developer', + providerId: 'opencode', + laneKind: 'secondary', + laneOwnerProviderId: 'opencode', + }, + { + name: 'tom', + agentType: 'developer', + providerId: 'opencode', + laneKind: 'secondary', + laneOwnerProviderId: 'opencode', + }, + ], + memberSpawnStatuses: { + alice: { + status: 'skipped', + launchState: 'skipped_for_launch', + skippedForLaunch: true, + updatedAt: '2026-04-13T10:00:05.000Z', + runtimeAlive: false, + bootstrapConfirmed: false, + hardFailure: false, + agentToolAccepted: false, + }, + tom: { + status: 'waiting', + launchState: 'runtime_pending_permission', + pendingPermissionRequestIds: ['perm-1'], + updatedAt: '2026-04-13T10:00:05.000Z', + runtimeAlive: true, + bootstrapConfirmed: false, + hardFailure: false, + agentToolAccepted: true, + }, + }, + memberSpawnSnapshot: undefined, + }); + + expect(presentation?.retryableOpenCodeSecondaryFailedNames).toEqual([]); + expect(presentation?.retryableOpenCodeSecondaryFailedCount).toBe(0); + }); + it('does not truncate long failed teammate reasons in the panel message', () => { const reason = 'You are bootstrapping into team "relay-works-10" as member "alice". Your first action is to call the MCP tool member_briefing on the agent-teams server with teamName="relay-works-10" and memberName="alice". If tool search shows only the prefixed MCP name, use mcp__agent-teams__member_briefing.';