feat(team): retry failed opencode secondary lanes
This commit is contained in:
777genius
parent
b1b2e696e5
commit
5c65f55067
15 changed files with 912 additions and 28 deletions
|
|
@ -60,6 +60,7 @@ import {
|
|||
TEAM_RESTART_MEMBER,
|
||||
TEAM_RESTORE,
|
||||
TEAM_RESTORE_TASK,
|
||||
TEAM_RETRY_FAILED_OPENCODE_SECONDARY_LANES,
|
||||
TEAM_SAVE_TASK_ATTACHMENT,
|
||||
TEAM_SEND_MESSAGE,
|
||||
TEAM_SET_CHANGE_PRESENCE_TRACKING,
|
||||
|
|
@ -188,6 +189,7 @@ import type {
|
|||
MemberLogSummary,
|
||||
MemberSpawnStatusesSnapshot,
|
||||
MessagesPage,
|
||||
RetryFailedOpenCodeSecondaryLanesResult,
|
||||
SendMessageRequest,
|
||||
SendMessageResult,
|
||||
TaskAttachmentMeta,
|
||||
|
|
@ -708,6 +710,10 @@ export function registerTeamHandlers(ipcMain: IpcMain): void {
|
|||
ipcMain.handle(TEAM_LEAD_CONTEXT, handleLeadContext);
|
||||
ipcMain.handle(TEAM_MEMBER_SPAWN_STATUSES, handleMemberSpawnStatuses);
|
||||
ipcMain.handle(TEAM_GET_AGENT_RUNTIME, handleGetAgentRuntime);
|
||||
ipcMain.handle(
|
||||
TEAM_RETRY_FAILED_OPENCODE_SECONDARY_LANES,
|
||||
handleRetryFailedOpenCodeSecondaryLanes
|
||||
);
|
||||
ipcMain.handle(TEAM_RESTART_MEMBER, handleRestartMember);
|
||||
ipcMain.handle(TEAM_SKIP_MEMBER_FOR_LAUNCH, handleSkipMemberForLaunch);
|
||||
ipcMain.handle(TEAM_SOFT_DELETE_TASK, handleSoftDeleteTask);
|
||||
|
|
@ -789,6 +795,7 @@ export function removeTeamHandlers(ipcMain: IpcMain): void {
|
|||
ipcMain.removeHandler(TEAM_LEAD_CONTEXT);
|
||||
ipcMain.removeHandler(TEAM_MEMBER_SPAWN_STATUSES);
|
||||
ipcMain.removeHandler(TEAM_GET_AGENT_RUNTIME);
|
||||
ipcMain.removeHandler(TEAM_RETRY_FAILED_OPENCODE_SECONDARY_LANES);
|
||||
ipcMain.removeHandler(TEAM_RESTART_MEMBER);
|
||||
ipcMain.removeHandler(TEAM_SKIP_MEMBER_FOR_LAUNCH);
|
||||
ipcMain.removeHandler(TEAM_SOFT_DELETE_TASK);
|
||||
|
|
@ -3833,6 +3840,19 @@ async function handleRestartMember(
|
|||
);
|
||||
}
|
||||
|
||||
async function handleRetryFailedOpenCodeSecondaryLanes(
|
||||
_event: IpcMainInvokeEvent,
|
||||
teamName: unknown
|
||||
): Promise<IpcResult<RetryFailedOpenCodeSecondaryLanesResult>> {
|
||||
const validatedTeamName = validateTeamName(teamName);
|
||||
if (!validatedTeamName.valid) {
|
||||
return { success: false, error: validatedTeamName.error ?? 'Invalid teamName' };
|
||||
}
|
||||
return wrapTeamHandler('retryFailedOpenCodeSecondaryLanes', async () =>
|
||||
getTeamProvisioningService().retryFailedOpenCodeSecondaryLanes(validatedTeamName.value!)
|
||||
);
|
||||
}
|
||||
|
||||
async function handleSkipMemberForLaunch(
|
||||
_event: IpcMainInvokeEvent,
|
||||
teamName: unknown,
|
||||
|
|
|
|||
|
|
@ -367,6 +367,7 @@ import type {
|
|||
PersistedTeamLaunchSnapshot,
|
||||
PersistedTeamLaunchSummary,
|
||||
ProviderModelLaunchIdentity,
|
||||
RetryFailedOpenCodeSecondaryLanesResult,
|
||||
TaskRef,
|
||||
TeamAgentRuntimeBackendType,
|
||||
TeamAgentRuntimeDiagnosticSeverity,
|
||||
|
|
@ -1699,6 +1700,16 @@ interface MixedSecondaryRuntimeLaneState {
|
|||
launchFinishedAtMs?: number;
|
||||
}
|
||||
|
||||
interface OpenCodeSecondaryRetryCandidate {
|
||||
memberName: string;
|
||||
laneId: string;
|
||||
}
|
||||
|
||||
interface OpenCodeSecondaryRetryOutcome {
|
||||
launchState: MemberLaunchState;
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
function formatOpenCodeLaneTimingMs(value: number | null | undefined): string {
|
||||
return typeof value === 'number' && Number.isFinite(value)
|
||||
? `${Math.max(0, Math.round(value))}ms`
|
||||
|
|
@ -5043,6 +5054,10 @@ export class TeamProvisioningService {
|
|||
private readonly launchStateStore = new TeamLaunchStateStore();
|
||||
private readonly launchStateStoreQueue = new Map<string, Promise<unknown>>();
|
||||
private readonly launchStateWrittenRunIdByTeam = new Map<string, string>();
|
||||
private readonly failedOpenCodeSecondaryRetryInFlightByTeam = new Map<
|
||||
string,
|
||||
Promise<RetryFailedOpenCodeSecondaryLanesResult>
|
||||
>();
|
||||
private readonly memberLogsFinder: TeamMemberLogsFinder;
|
||||
private readonly transcriptProjectResolver: TeamTranscriptProjectResolver;
|
||||
private teamChangeEmitter: ((event: TeamChangeEvent) => void) | null = null;
|
||||
|
|
@ -12190,6 +12205,353 @@ export class TeamProvisioningService {
|
|||
}
|
||||
}
|
||||
|
||||
async retryFailedOpenCodeSecondaryLanes(
|
||||
teamName: string
|
||||
): Promise<RetryFailedOpenCodeSecondaryLanesResult> {
|
||||
const existing = this.failedOpenCodeSecondaryRetryInFlightByTeam.get(teamName);
|
||||
if (existing) {
|
||||
return existing;
|
||||
}
|
||||
|
||||
const retry = this.retryFailedOpenCodeSecondaryLanesNow(teamName).finally(() => {
|
||||
this.failedOpenCodeSecondaryRetryInFlightByTeam.delete(teamName);
|
||||
});
|
||||
this.failedOpenCodeSecondaryRetryInFlightByTeam.set(teamName, retry);
|
||||
return retry;
|
||||
}
|
||||
|
||||
private async retryFailedOpenCodeSecondaryLanesNow(
|
||||
teamName: string
|
||||
): Promise<RetryFailedOpenCodeSecondaryLanesResult> {
|
||||
const run = this.getMutableAliveRunOrThrow(teamName);
|
||||
if (this.getProvisioningRunId(teamName)) {
|
||||
throw new Error('Team launch is still in progress');
|
||||
}
|
||||
|
||||
const result: RetryFailedOpenCodeSecondaryLanesResult = {
|
||||
attempted: [],
|
||||
confirmed: [],
|
||||
pending: [],
|
||||
failed: [],
|
||||
skipped: [],
|
||||
};
|
||||
const candidates = await this.collectFailedOpenCodeSecondaryRetryCandidates(run);
|
||||
|
||||
for (const candidate of candidates) {
|
||||
if (!this.isCurrentTrackedRun(run) || run.processKilled || run.cancelRequested) {
|
||||
result.skipped.push({
|
||||
memberName: candidate.memberName,
|
||||
reason: 'Team stopped during retry',
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
await this.reattachOpenCodeOwnedMemberLane(teamName, candidate.memberName, {
|
||||
reason: 'manual_restart',
|
||||
});
|
||||
result.attempted.push(candidate.memberName);
|
||||
|
||||
const outcome = await this.readOpenCodeSecondaryRetryOutcome(
|
||||
run,
|
||||
candidate.memberName,
|
||||
candidate.laneId
|
||||
);
|
||||
if (outcome.launchState === 'confirmed_alive') {
|
||||
result.confirmed.push(candidate.memberName);
|
||||
} else if (outcome.launchState === 'failed_to_start') {
|
||||
result.failed.push({
|
||||
memberName: candidate.memberName,
|
||||
error: outcome.reason ?? 'OpenCode retry failed',
|
||||
});
|
||||
} else if (outcome.launchState === 'skipped_for_launch') {
|
||||
result.skipped.push({
|
||||
memberName: candidate.memberName,
|
||||
reason: outcome.reason ?? 'Teammate is skipped for this launch',
|
||||
});
|
||||
} else {
|
||||
result.pending.push(candidate.memberName);
|
||||
}
|
||||
} catch (error) {
|
||||
result.failed.push({
|
||||
memberName: candidate.memberName,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
await this.notifyLeadAboutConfirmedOpenCodeRetries(run, result);
|
||||
return result;
|
||||
}
|
||||
|
||||
private async collectFailedOpenCodeSecondaryRetryCandidates(
|
||||
run: ProvisioningRun
|
||||
): Promise<OpenCodeSecondaryRetryCandidate[]> {
|
||||
const teamName = run.teamName;
|
||||
const leadProviderId = resolveTeamProviderId(run.request.providerId);
|
||||
if (leadProviderId === 'opencode') {
|
||||
throw new Error(
|
||||
'Retrying OpenCode secondary lanes is only supported for mixed teams with a non-OpenCode lead.'
|
||||
);
|
||||
}
|
||||
if (!this.getOpenCodeRuntimeAdapter()) {
|
||||
throw new Error('OpenCode runtime adapter is not available for secondary lane retry.');
|
||||
}
|
||||
|
||||
const config = await this.readConfigForStrictDecision(teamName);
|
||||
if (!config) {
|
||||
throw new Error(`Team "${teamName}" configuration is no longer available`);
|
||||
}
|
||||
const metaMembers = await this.membersMetaStore.getMembers(teamName).catch(() => []);
|
||||
const persistedSnapshot = await this.launchStateStore.read(teamName).catch(() => null);
|
||||
|
||||
const names = new Set<string>();
|
||||
for (const member of config.members ?? []) {
|
||||
const name = member.name?.trim();
|
||||
if (name) {
|
||||
names.add(name);
|
||||
}
|
||||
}
|
||||
for (const member of metaMembers) {
|
||||
const name = member.name?.trim();
|
||||
if (name) {
|
||||
names.add(name);
|
||||
}
|
||||
}
|
||||
for (const lane of run.mixedSecondaryLanes ?? []) {
|
||||
const name = lane.member.name?.trim();
|
||||
if (name) {
|
||||
names.add(name);
|
||||
}
|
||||
}
|
||||
for (const name of persistedSnapshot?.expectedMembers ?? []) {
|
||||
if (name.trim()) {
|
||||
names.add(name.trim());
|
||||
}
|
||||
}
|
||||
for (const name of Object.keys(persistedSnapshot?.members ?? {})) {
|
||||
if (name.trim()) {
|
||||
names.add(name.trim());
|
||||
}
|
||||
}
|
||||
|
||||
const candidates: OpenCodeSecondaryRetryCandidate[] = [];
|
||||
for (const memberName of [...names].sort((left, right) => left.localeCompare(right))) {
|
||||
const configuredMember = this.resolveEffectiveConfiguredMember(
|
||||
config.members ?? [],
|
||||
metaMembers,
|
||||
memberName
|
||||
);
|
||||
if (!configuredMember || configuredMember.removedAt) {
|
||||
continue;
|
||||
}
|
||||
if (isLeadMember({ name: memberName, agentType: configuredMember.agentType })) {
|
||||
continue;
|
||||
}
|
||||
if (normalizeOptionalTeamProviderId(configuredMember.providerId) !== 'opencode') {
|
||||
continue;
|
||||
}
|
||||
|
||||
const laneIdentity = buildPlannedMemberLaneIdentity({
|
||||
leadProviderId,
|
||||
member: {
|
||||
name: memberName,
|
||||
providerId: 'opencode',
|
||||
},
|
||||
});
|
||||
if (
|
||||
laneIdentity.laneKind !== 'secondary' ||
|
||||
laneIdentity.laneOwnerProviderId !== 'opencode'
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const existingLane = (run.mixedSecondaryLanes ?? []).find(
|
||||
(lane) =>
|
||||
lane.laneId === laneIdentity.laneId ||
|
||||
matchesTeamMemberIdentity(lane.member.name, memberName)
|
||||
);
|
||||
const liveEntry = run.memberSpawnStatuses.get(memberName);
|
||||
const persistedMember =
|
||||
persistedSnapshot?.members[memberName] ??
|
||||
Object.values(persistedSnapshot?.members ?? {}).find(
|
||||
(member) => member.laneId === laneIdentity.laneId
|
||||
);
|
||||
|
||||
if (
|
||||
this.isRetryableFailedOpenCodeSecondaryLane({
|
||||
liveEntry,
|
||||
persistedMember,
|
||||
existingLane,
|
||||
})
|
||||
) {
|
||||
candidates.push({ memberName, laneId: laneIdentity.laneId });
|
||||
}
|
||||
}
|
||||
return candidates;
|
||||
}
|
||||
|
||||
private isRetryableFailedOpenCodeSecondaryLane(input: {
|
||||
liveEntry?: MemberSpawnStatusEntry;
|
||||
persistedMember?: PersistedTeamLaunchMemberState;
|
||||
existingLane?: MixedSecondaryRuntimeLaneState;
|
||||
}): boolean {
|
||||
const { liveEntry, persistedMember, existingLane } = input;
|
||||
if (existingLane?.state === 'queued' || existingLane?.state === 'launching') {
|
||||
return false;
|
||||
}
|
||||
if (
|
||||
liveEntry?.launchState === 'skipped_for_launch' ||
|
||||
liveEntry?.skippedForLaunch === true ||
|
||||
persistedMember?.launchState === 'skipped_for_launch' ||
|
||||
persistedMember?.skippedForLaunch === true
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
if (
|
||||
liveEntry?.launchState === 'runtime_pending_permission' ||
|
||||
liveEntry?.launchState === 'runtime_pending_bootstrap' ||
|
||||
persistedMember?.launchState === 'runtime_pending_permission' ||
|
||||
persistedMember?.launchState === 'runtime_pending_bootstrap' ||
|
||||
(liveEntry?.pendingPermissionRequestIds?.length ?? 0) > 0 ||
|
||||
(persistedMember?.pendingPermissionRequestIds?.length ?? 0) > 0
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
if (liveEntry?.launchState === 'starting' || liveEntry?.status === 'spawning') {
|
||||
return false;
|
||||
}
|
||||
if (
|
||||
liveEntry?.launchState === 'confirmed_alive' ||
|
||||
liveEntry?.bootstrapConfirmed === true ||
|
||||
persistedMember?.launchState === 'confirmed_alive' ||
|
||||
persistedMember?.bootstrapConfirmed === true
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return (
|
||||
liveEntry?.launchState === 'failed_to_start' ||
|
||||
liveEntry?.status === 'error' ||
|
||||
persistedMember?.launchState === 'failed_to_start' ||
|
||||
persistedMember?.hardFailure === true
|
||||
);
|
||||
}
|
||||
|
||||
private async readOpenCodeSecondaryRetryOutcome(
|
||||
run: ProvisioningRun,
|
||||
memberName: string,
|
||||
laneId: string
|
||||
): Promise<OpenCodeSecondaryRetryOutcome> {
|
||||
const lane = (run.mixedSecondaryLanes ?? []).find(
|
||||
(candidate) =>
|
||||
candidate.laneId === laneId || matchesTeamMemberIdentity(candidate.member.name, memberName)
|
||||
);
|
||||
const memberEvidence =
|
||||
lane?.result?.members[memberName] ??
|
||||
Object.values(lane?.result?.members ?? {}).find((member) =>
|
||||
matchesTeamMemberIdentity(member.memberName, memberName)
|
||||
);
|
||||
const persistedSnapshot = await this.launchStateStore.read(run.teamName).catch(() => null);
|
||||
const persistedMember =
|
||||
persistedSnapshot?.members[memberName] ??
|
||||
Object.values(persistedSnapshot?.members ?? {}).find((member) => member.laneId === laneId);
|
||||
const liveEntry = run.memberSpawnStatuses.get(memberName);
|
||||
|
||||
if (
|
||||
memberEvidence?.launchState === 'confirmed_alive' ||
|
||||
memberEvidence?.bootstrapConfirmed === true ||
|
||||
liveEntry?.launchState === 'confirmed_alive' ||
|
||||
liveEntry?.bootstrapConfirmed === true ||
|
||||
persistedMember?.launchState === 'confirmed_alive' ||
|
||||
persistedMember?.bootstrapConfirmed === true
|
||||
) {
|
||||
return { launchState: 'confirmed_alive' };
|
||||
}
|
||||
|
||||
if (
|
||||
liveEntry?.launchState === 'skipped_for_launch' ||
|
||||
liveEntry?.skippedForLaunch === true ||
|
||||
persistedMember?.launchState === 'skipped_for_launch' ||
|
||||
persistedMember?.skippedForLaunch === true
|
||||
) {
|
||||
return {
|
||||
launchState: 'skipped_for_launch',
|
||||
reason: liveEntry?.skipReason ?? persistedMember?.skipReason,
|
||||
};
|
||||
}
|
||||
|
||||
if (
|
||||
memberEvidence?.launchState === 'failed_to_start' ||
|
||||
memberEvidence?.hardFailure === true ||
|
||||
liveEntry?.launchState === 'failed_to_start' ||
|
||||
liveEntry?.status === 'error' ||
|
||||
persistedMember?.launchState === 'failed_to_start' ||
|
||||
persistedMember?.hardFailure === true
|
||||
) {
|
||||
return {
|
||||
launchState: 'failed_to_start',
|
||||
reason: this.selectOpenCodeSecondaryRetryFailureReason({
|
||||
memberEvidence,
|
||||
liveEntry,
|
||||
persistedMember,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
launchState:
|
||||
memberEvidence?.launchState ??
|
||||
liveEntry?.launchState ??
|
||||
persistedMember?.launchState ??
|
||||
'runtime_pending_bootstrap',
|
||||
};
|
||||
}
|
||||
|
||||
private selectOpenCodeSecondaryRetryFailureReason(input: {
|
||||
memberEvidence?: TeamRuntimeMemberLaunchEvidence;
|
||||
liveEntry?: MemberSpawnStatusEntry;
|
||||
persistedMember?: PersistedTeamLaunchMemberState;
|
||||
}): string | undefined {
|
||||
const diagnostics = [
|
||||
input.memberEvidence?.hardFailureReason,
|
||||
input.memberEvidence?.runtimeDiagnostic,
|
||||
...(input.memberEvidence?.diagnostics ?? []),
|
||||
input.liveEntry?.hardFailureReason,
|
||||
input.liveEntry?.runtimeDiagnostic,
|
||||
input.liveEntry?.error,
|
||||
input.persistedMember?.hardFailureReason,
|
||||
input.persistedMember?.runtimeDiagnostic,
|
||||
];
|
||||
return diagnostics
|
||||
.find(
|
||||
(diagnostic): diagnostic is string =>
|
||||
typeof diagnostic === 'string' && diagnostic.trim().length > 0
|
||||
)
|
||||
?.trim();
|
||||
}
|
||||
|
||||
private async notifyLeadAboutConfirmedOpenCodeRetries(
|
||||
run: ProvisioningRun,
|
||||
result: RetryFailedOpenCodeSecondaryLanesResult
|
||||
): Promise<void> {
|
||||
if (result.confirmed.length === 0) {
|
||||
return;
|
||||
}
|
||||
const confirmedNames = result.confirmed.map((name) => `@${name}`).join(', ');
|
||||
const message = [
|
||||
`Системное замечание: повторный запуск OpenCode-тиммейтов подтверждён: ${confirmedNames}.`,
|
||||
`Их можно снова считать доступными.`,
|
||||
].join(' ');
|
||||
await this.sendMessageToRun(run, message).catch((error: unknown) =>
|
||||
logger.warn(
|
||||
`[${run.teamName}] failed to send OpenCode retry recovery notice to lead: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}`
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
async skipMemberForLaunch(teamName: string, memberName: string): Promise<void> {
|
||||
const normalizedMemberName = memberName.trim();
|
||||
if (!normalizedMemberName) {
|
||||
|
|
|
|||
|
|
@ -394,6 +394,9 @@ export const TEAM_GET_AGENT_RUNTIME = 'team:getAgentRuntime';
|
|||
/** Restart a specific teammate runtime */
|
||||
export const TEAM_RESTART_MEMBER = 'team:restartMember';
|
||||
|
||||
/** Retry failed OpenCode-owned secondary runtime lanes */
|
||||
export const TEAM_RETRY_FAILED_OPENCODE_SECONDARY_LANES = 'team:retryFailedOpenCodeSecondaryLanes';
|
||||
|
||||
/** Skip a failed teammate for the current launch */
|
||||
export const TEAM_SKIP_MEMBER_FOR_LAUNCH = 'team:skipMemberForLaunch';
|
||||
|
||||
|
|
|
|||
|
|
@ -168,6 +168,7 @@ import {
|
|||
TEAM_RESTART_MEMBER,
|
||||
TEAM_RESTORE,
|
||||
TEAM_RESTORE_TASK,
|
||||
TEAM_RETRY_FAILED_OPENCODE_SECONDARY_LANES,
|
||||
TEAM_SAVE_TASK_ATTACHMENT,
|
||||
TEAM_SEND_MESSAGE,
|
||||
TEAM_SET_CHANGE_PRESENCE_TRACKING,
|
||||
|
|
@ -283,6 +284,7 @@ import type {
|
|||
ProjectBranchChangeEvent,
|
||||
RejectResult,
|
||||
ReplaceMembersRequest,
|
||||
RetryFailedOpenCodeSecondaryLanesResult,
|
||||
Schedule,
|
||||
ScheduleChangeEvent,
|
||||
ScheduleRun,
|
||||
|
|
@ -1118,6 +1120,12 @@ const electronAPI: ElectronAPI = {
|
|||
getTeamAgentRuntime: async (teamName: string) => {
|
||||
return invokeIpcWithResult<TeamAgentRuntimeSnapshot>(TEAM_GET_AGENT_RUNTIME, teamName);
|
||||
},
|
||||
retryFailedOpenCodeSecondaryLanes: async (teamName: string) => {
|
||||
return invokeIpcWithResult<RetryFailedOpenCodeSecondaryLanesResult>(
|
||||
TEAM_RETRY_FAILED_OPENCODE_SECONDARY_LANES,
|
||||
teamName
|
||||
);
|
||||
},
|
||||
restartMember: async (teamName: string, memberName: string) => {
|
||||
return invokeIpcWithResult<void>(TEAM_RESTART_MEMBER, teamName, memberName);
|
||||
},
|
||||
|
|
|
|||
|
|
@ -981,6 +981,9 @@ export class HttpAPIClient implements ElectronAPI {
|
|||
restartMember: async (): Promise<void> => {
|
||||
throw new Error('Member restart is not available in browser mode');
|
||||
},
|
||||
retryFailedOpenCodeSecondaryLanes: async () => {
|
||||
throw new Error('OpenCode secondary retry is not available in browser mode');
|
||||
},
|
||||
skipMemberForLaunch: async (): Promise<void> => {
|
||||
throw new Error('Member launch skip is not available in browser mode');
|
||||
},
|
||||
|
|
|
|||
|
|
@ -7,6 +7,8 @@ import { X } from 'lucide-react';
|
|||
import { ProvisioningProgressBlock } from './ProvisioningProgressBlock';
|
||||
import { useTeamProvisioningPresentation } from './useTeamProvisioningPresentation';
|
||||
|
||||
import type { RetryFailedOpenCodeSecondaryLanesResult } from '@shared/types';
|
||||
|
||||
export interface TeamProvisioningPanelProps {
|
||||
teamName: string;
|
||||
surface?: 'raised' | 'flat';
|
||||
|
|
@ -15,6 +17,27 @@ export interface TeamProvisioningPanelProps {
|
|||
defaultLogsOpen?: boolean;
|
||||
}
|
||||
|
||||
function formatOpenCodeSecondaryRetryResult(
|
||||
result: RetryFailedOpenCodeSecondaryLanesResult
|
||||
): string {
|
||||
const parts: string[] = [];
|
||||
if (result.confirmed.length > 0) {
|
||||
parts.push(`${result.confirmed.length} confirmed`);
|
||||
}
|
||||
if (result.pending.length > 0) {
|
||||
parts.push(`${result.pending.length} pending`);
|
||||
}
|
||||
if (result.failed.length > 0) {
|
||||
parts.push(`${result.failed.length} failed`);
|
||||
}
|
||||
if (result.skipped.length > 0) {
|
||||
parts.push(`${result.skipped.length} skipped`);
|
||||
}
|
||||
return parts.length > 0
|
||||
? `OpenCode retry: ${parts.join(', ')}`
|
||||
: 'No retryable OpenCode failures';
|
||||
}
|
||||
|
||||
export const TeamProvisioningPanel = memo(function TeamProvisioningPanel({
|
||||
teamName,
|
||||
surface = 'flat',
|
||||
|
|
@ -22,13 +45,19 @@ export const TeamProvisioningPanel = memo(function TeamProvisioningPanel({
|
|||
className,
|
||||
defaultLogsOpen,
|
||||
}: TeamProvisioningPanelProps): React.JSX.Element | null {
|
||||
const { presentation, cancelProvisioning, runInstanceKey } =
|
||||
const { presentation, cancelProvisioning, retryFailedOpenCodeSecondaryLanes, runInstanceKey } =
|
||||
useTeamProvisioningPresentation(teamName);
|
||||
const [dismissed, setDismissed] = useState(false);
|
||||
const [retryingOpenCode, setRetryingOpenCode] = useState(false);
|
||||
const [openCodeRetryMessage, setOpenCodeRetryMessage] = useState<string | null>(null);
|
||||
const [openCodeRetryError, setOpenCodeRetryError] = useState<string | null>(null);
|
||||
const lastActiveStepRef = useRef(-1);
|
||||
|
||||
useEffect(() => {
|
||||
setDismissed(false);
|
||||
setRetryingOpenCode(false);
|
||||
setOpenCodeRetryMessage(null);
|
||||
setOpenCodeRetryError(null);
|
||||
}, [runInstanceKey]);
|
||||
|
||||
if (!presentation || dismissed) {
|
||||
|
|
@ -40,6 +69,48 @@ export const TeamProvisioningPanel = memo(function TeamProvisioningPanel({
|
|||
}
|
||||
|
||||
const showRunningState = presentation.isActive || presentation.hasMembersStillJoining;
|
||||
const canRetryFailedOpenCode =
|
||||
!presentation.isActive &&
|
||||
presentation.retryableOpenCodeSecondaryFailedCount > 0 &&
|
||||
Boolean(retryFailedOpenCodeSecondaryLanes);
|
||||
|
||||
const retryOpenCodeAction = canRetryFailedOpenCode ? (
|
||||
<div className="flex flex-wrap items-center gap-2 rounded-md border border-amber-500/30 bg-amber-500/10 px-3 py-2">
|
||||
<p className="min-w-0 flex-1 text-xs text-[var(--step-warning-text)]">
|
||||
{openCodeRetryError ??
|
||||
openCodeRetryMessage ??
|
||||
`${presentation.retryableOpenCodeSecondaryFailedCount} failed OpenCode teammate${
|
||||
presentation.retryableOpenCodeSecondaryFailedCount === 1 ? '' : 's'
|
||||
} can be retried.`}
|
||||
</p>
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
className="h-7 shrink-0 border-amber-500/40 px-2 text-xs text-[var(--step-warning-text)] hover:bg-amber-500/10"
|
||||
disabled={retryingOpenCode}
|
||||
onClick={() => {
|
||||
if (!retryFailedOpenCodeSecondaryLanes || retryingOpenCode) {
|
||||
return;
|
||||
}
|
||||
setRetryingOpenCode(true);
|
||||
setOpenCodeRetryError(null);
|
||||
setOpenCodeRetryMessage(null);
|
||||
void retryFailedOpenCodeSecondaryLanes(teamName)
|
||||
.then((result) => {
|
||||
setOpenCodeRetryMessage(formatOpenCodeSecondaryRetryResult(result));
|
||||
})
|
||||
.catch((error: unknown) => {
|
||||
setOpenCodeRetryError(error instanceof Error ? error.message : String(error));
|
||||
})
|
||||
.finally(() => {
|
||||
setRetryingOpenCode(false);
|
||||
});
|
||||
}}
|
||||
>
|
||||
{retryingOpenCode ? 'Retrying OpenCode...' : 'Retry failed OpenCode teammates'}
|
||||
</Button>
|
||||
</div>
|
||||
) : null;
|
||||
|
||||
const block = (
|
||||
<ProvisioningProgressBlock
|
||||
|
|
@ -81,32 +152,35 @@ export const TeamProvisioningPanel = memo(function TeamProvisioningPanel({
|
|||
}
|
||||
: null
|
||||
}
|
||||
className={!presentation.isFailed ? className : undefined}
|
||||
className={!presentation.isFailed && !retryOpenCodeAction ? className : undefined}
|
||||
/>
|
||||
);
|
||||
|
||||
if (!presentation.isFailed) {
|
||||
if (!presentation.isFailed && !retryOpenCodeAction) {
|
||||
return block;
|
||||
}
|
||||
|
||||
return (
|
||||
<div className={cn('space-y-2', className)}>
|
||||
<div className="flex items-center gap-2 rounded-md border border-red-500/40 bg-red-500/10 px-3 py-2">
|
||||
<p className="flex-1 text-xs text-[var(--step-error-text)]">
|
||||
{presentation.progress.message}
|
||||
</p>
|
||||
{dismissible ? (
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
className="h-6 shrink-0 border-red-500/40 px-2 text-xs text-[var(--step-error-text)] hover:bg-red-500/10"
|
||||
onClick={() => setDismissed(true)}
|
||||
>
|
||||
<X size={12} />
|
||||
</Button>
|
||||
) : null}
|
||||
</div>
|
||||
{presentation.isFailed ? (
|
||||
<div className="flex items-center gap-2 rounded-md border border-red-500/40 bg-red-500/10 px-3 py-2">
|
||||
<p className="flex-1 text-xs text-[var(--step-error-text)]">
|
||||
{presentation.progress.message}
|
||||
</p>
|
||||
{dismissible ? (
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
className="h-6 shrink-0 border-red-500/40 px-2 text-xs text-[var(--step-error-text)] hover:bg-red-500/10"
|
||||
onClick={() => setDismissed(true)}
|
||||
>
|
||||
<X size={12} />
|
||||
</Button>
|
||||
) : null}
|
||||
</div>
|
||||
) : null}
|
||||
{block}
|
||||
{retryOpenCodeAction}
|
||||
</div>
|
||||
);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -9,22 +9,33 @@ import { buildTeamProvisioningPresentation } from '@renderer/utils/teamProvision
|
|||
import { useShallow } from 'zustand/react/shallow';
|
||||
|
||||
import type { TeamProvisioningPresentation } from '@renderer/utils/teamProvisioningPresentation';
|
||||
import type { RetryFailedOpenCodeSecondaryLanesResult } from '@shared/types';
|
||||
|
||||
export function useTeamProvisioningPresentation(teamName: string): {
|
||||
presentation: TeamProvisioningPresentation | null;
|
||||
cancelProvisioning: ((runId: string) => Promise<void>) | null;
|
||||
retryFailedOpenCodeSecondaryLanes:
|
||||
| ((teamName: string) => Promise<RetryFailedOpenCodeSecondaryLanesResult>)
|
||||
| null;
|
||||
runInstanceKey: string | null;
|
||||
} {
|
||||
const { progress, cancelProvisioning, teamMembers, memberSpawnStatuses, memberSpawnSnapshot } =
|
||||
useStore(
|
||||
useShallow((s) => ({
|
||||
progress: getCurrentProvisioningProgressForTeam(s, teamName),
|
||||
cancelProvisioning: s.cancelProvisioning,
|
||||
teamMembers: selectTeamMemberSnapshotsForName(s, teamName),
|
||||
memberSpawnStatuses: s.memberSpawnStatusesByTeam[teamName],
|
||||
memberSpawnSnapshot: s.memberSpawnSnapshotsByTeam[teamName],
|
||||
}))
|
||||
);
|
||||
const {
|
||||
progress,
|
||||
cancelProvisioning,
|
||||
retryFailedOpenCodeSecondaryLanes,
|
||||
teamMembers,
|
||||
memberSpawnStatuses,
|
||||
memberSpawnSnapshot,
|
||||
} = useStore(
|
||||
useShallow((s) => ({
|
||||
progress: getCurrentProvisioningProgressForTeam(s, teamName),
|
||||
cancelProvisioning: s.cancelProvisioning,
|
||||
retryFailedOpenCodeSecondaryLanes: s.retryFailedOpenCodeSecondaryLanes,
|
||||
teamMembers: selectTeamMemberSnapshotsForName(s, teamName),
|
||||
memberSpawnStatuses: s.memberSpawnStatusesByTeam[teamName],
|
||||
memberSpawnSnapshot: s.memberSpawnSnapshotsByTeam[teamName],
|
||||
}))
|
||||
);
|
||||
|
||||
const presentation = useMemo(
|
||||
() =>
|
||||
|
|
@ -40,6 +51,7 @@ export function useTeamProvisioningPresentation(teamName: string): {
|
|||
return {
|
||||
presentation,
|
||||
cancelProvisioning,
|
||||
retryFailedOpenCodeSecondaryLanes: retryFailedOpenCodeSecondaryLanes ?? null,
|
||||
runInstanceKey: progress ? `${teamName}:${progress.runId}:${progress.startedAt}` : null,
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -46,6 +46,7 @@ import type {
|
|||
NotificationTarget,
|
||||
PersistedTeamLaunchSummary,
|
||||
ResolvedTeamMember,
|
||||
RetryFailedOpenCodeSecondaryLanesResult,
|
||||
SendMessageRequest,
|
||||
SendMessageResult,
|
||||
TaskChangePresenceState,
|
||||
|
|
@ -2500,6 +2501,9 @@ export interface TeamSlice {
|
|||
memberName: string,
|
||||
role: string | undefined
|
||||
) => Promise<void>;
|
||||
retryFailedOpenCodeSecondaryLanes: (
|
||||
teamName: string
|
||||
) => Promise<RetryFailedOpenCodeSecondaryLanesResult>;
|
||||
addTaskRelationship: (
|
||||
teamName: string,
|
||||
taskId: string,
|
||||
|
|
@ -4679,6 +4683,19 @@ export const createTeamSlice: StateCreator<AppState, [], [], TeamSlice> = (set,
|
|||
}
|
||||
},
|
||||
|
||||
retryFailedOpenCodeSecondaryLanes: async (teamName: string) => {
|
||||
try {
|
||||
return await unwrapIpc('team:retryFailedOpenCodeSecondaryLanes', () =>
|
||||
api.teams.retryFailedOpenCodeSecondaryLanes(teamName)
|
||||
);
|
||||
} finally {
|
||||
await Promise.allSettled([
|
||||
get().fetchMemberSpawnStatuses(teamName),
|
||||
get().fetchTeamAgentRuntime(teamName),
|
||||
]);
|
||||
}
|
||||
},
|
||||
|
||||
skipMemberForLaunch: async (teamName: string, memberName: string) => {
|
||||
try {
|
||||
await unwrapIpc('team:skipMemberForLaunch', () =>
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import type {
|
|||
MemberSpawnStatusesSnapshot,
|
||||
TeamProvisioningProgress,
|
||||
} from '@shared/types';
|
||||
import { isLeadMember } from '@shared/utils/leadDetection';
|
||||
|
||||
type MemberSpawnStatusCollection =
|
||||
| Record<string, MemberSpawnStatusEntry>
|
||||
|
|
@ -69,6 +70,42 @@ function isSkippedSpawnEntry(entry: MemberSpawnStatusEntry | undefined): boolean
|
|||
return entry?.launchState === 'skipped_for_launch' || entry?.skippedForLaunch === true;
|
||||
}
|
||||
|
||||
function isOpenCodeSecondaryRetryCandidate(params: {
|
||||
member: ProvisioningMemberLike | undefined;
|
||||
entry: MemberSpawnStatusEntry | undefined;
|
||||
}): boolean {
|
||||
const { member, entry } = params;
|
||||
if (!member || !entry) {
|
||||
return false;
|
||||
}
|
||||
if (member.providerId !== 'opencode' || member.removedAt) {
|
||||
return false;
|
||||
}
|
||||
if (isLeadMember({ name: member.name, agentType: member.agentType })) {
|
||||
return false;
|
||||
}
|
||||
if (member.laneKind && member.laneKind !== 'secondary') {
|
||||
return false;
|
||||
}
|
||||
if (member.laneOwnerProviderId && member.laneOwnerProviderId !== 'opencode') {
|
||||
return false;
|
||||
}
|
||||
if (
|
||||
entry.launchState === 'skipped_for_launch' ||
|
||||
entry.skippedForLaunch === true ||
|
||||
entry.launchState === 'runtime_pending_permission' ||
|
||||
entry.launchState === 'runtime_pending_bootstrap' ||
|
||||
(entry.pendingPermissionRequestIds?.length ?? 0) > 0 ||
|
||||
entry.launchState === 'starting' ||
|
||||
entry.status === 'spawning' ||
|
||||
entry.launchState === 'confirmed_alive' ||
|
||||
entry.bootstrapConfirmed === true
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
return entry.launchState === 'failed_to_start' || entry.status === 'error';
|
||||
}
|
||||
|
||||
function shouldPreferSnapshotEntryOverLive(params: {
|
||||
liveEntry: MemberSpawnStatusEntry | undefined;
|
||||
snapshotEntry: MemberSpawnStatusEntry | undefined;
|
||||
|
|
@ -480,6 +517,51 @@ function getSkippedSpawnDetails(params: {
|
|||
.sort((left, right) => left.name.localeCompare(right.name));
|
||||
}
|
||||
|
||||
function getRetryableOpenCodeSecondaryFailedNames(params: {
|
||||
members: readonly ProvisioningMemberLike[];
|
||||
memberSpawnStatuses: MemberSpawnStatusCollection;
|
||||
memberSpawnSnapshotStatuses?: MemberSpawnStatusesSnapshot['statuses'];
|
||||
memberSpawnSnapshotUpdatedAt?: string;
|
||||
}): string[] {
|
||||
const membersByName = new Map(
|
||||
params.members
|
||||
.map((member) => [member.name.trim(), member] as const)
|
||||
.filter(([name]) => name.length > 0)
|
||||
);
|
||||
const names = new Set<string>(membersByName.keys());
|
||||
if (params.memberSpawnStatuses instanceof Map) {
|
||||
for (const name of params.memberSpawnStatuses.keys()) {
|
||||
names.add(name);
|
||||
}
|
||||
} else if (params.memberSpawnStatuses) {
|
||||
for (const name of Object.keys(params.memberSpawnStatuses)) {
|
||||
names.add(name);
|
||||
}
|
||||
}
|
||||
for (const name of Object.keys(params.memberSpawnSnapshotStatuses ?? {})) {
|
||||
names.add(name);
|
||||
}
|
||||
|
||||
return [...names]
|
||||
.filter((name) => {
|
||||
const liveEntry =
|
||||
params.memberSpawnStatuses instanceof Map
|
||||
? params.memberSpawnStatuses.get(name)
|
||||
: params.memberSpawnStatuses?.[name];
|
||||
const snapshotEntry = params.memberSpawnSnapshotStatuses?.[name];
|
||||
const entry = getPreferredSpawnEntry({
|
||||
liveEntry,
|
||||
snapshotEntry,
|
||||
snapshotUpdatedAt: params.memberSpawnSnapshotUpdatedAt,
|
||||
});
|
||||
return isOpenCodeSecondaryRetryCandidate({
|
||||
member: membersByName.get(name),
|
||||
entry,
|
||||
});
|
||||
})
|
||||
.sort((left, right) => left.localeCompare(right));
|
||||
}
|
||||
|
||||
function normalizeFailureReason(reason: string): string {
|
||||
return reason.replace(/\s+/g, ' ').trim();
|
||||
}
|
||||
|
|
@ -581,6 +663,8 @@ export interface TeamProvisioningPresentation {
|
|||
allTeammatesConfirmedAlive: boolean;
|
||||
hasMembersStillJoining: boolean;
|
||||
remainingJoinCount: number;
|
||||
retryableOpenCodeSecondaryFailedCount: number;
|
||||
retryableOpenCodeSecondaryFailedNames: string[];
|
||||
panelTitle: string;
|
||||
panelMessage?: string | null;
|
||||
panelMessageSeverity?: 'error' | 'warning' | 'info';
|
||||
|
|
@ -674,6 +758,13 @@ export function buildTeamProvisioningPresentation({
|
|||
memberSpawnSnapshotStatuses: memberSpawnSnapshot?.statuses,
|
||||
memberSpawnSnapshotUpdatedAt: memberSpawnSnapshot?.updatedAt,
|
||||
});
|
||||
const retryableOpenCodeSecondaryFailedNames = getRetryableOpenCodeSecondaryFailedNames({
|
||||
members,
|
||||
memberSpawnStatuses,
|
||||
memberSpawnSnapshotStatuses: memberSpawnSnapshot?.statuses,
|
||||
memberSpawnSnapshotUpdatedAt: memberSpawnSnapshot?.updatedAt,
|
||||
});
|
||||
const retryableOpenCodeSecondaryFailedCount = retryableOpenCodeSecondaryFailedNames.length;
|
||||
|
||||
const { allTeammatesConfirmedAlive, hasMembersStillJoining, remainingJoinCount } =
|
||||
getLaunchJoinState({
|
||||
|
|
@ -712,6 +803,8 @@ export function buildTeamProvisioningPresentation({
|
|||
allTeammatesConfirmedAlive,
|
||||
hasMembersStillJoining,
|
||||
remainingJoinCount,
|
||||
retryableOpenCodeSecondaryFailedCount,
|
||||
retryableOpenCodeSecondaryFailedNames,
|
||||
panelTitle: 'Launch failed',
|
||||
panelMessage: progress.error ?? failedSpawnPanelMessage ?? genericFailedSpawnPanelMessage,
|
||||
panelTone: 'error',
|
||||
|
|
@ -800,6 +893,8 @@ export function buildTeamProvisioningPresentation({
|
|||
allTeammatesConfirmedAlive,
|
||||
hasMembersStillJoining,
|
||||
remainingJoinCount,
|
||||
retryableOpenCodeSecondaryFailedCount,
|
||||
retryableOpenCodeSecondaryFailedNames,
|
||||
panelTitle: 'Launch details',
|
||||
panelMessage:
|
||||
failedSpawnCount > 0 || skippedSpawnCount > 0 || hasMembersStillJoining
|
||||
|
|
@ -875,6 +970,8 @@ export function buildTeamProvisioningPresentation({
|
|||
allTeammatesConfirmedAlive,
|
||||
hasMembersStillJoining,
|
||||
remainingJoinCount,
|
||||
retryableOpenCodeSecondaryFailedCount,
|
||||
retryableOpenCodeSecondaryFailedNames,
|
||||
panelTitle: openCodeSecondaryWaitPhrase ? 'Core team ready' : 'Launching team',
|
||||
panelMessage:
|
||||
failedSpawnCount > 0
|
||||
|
|
|
|||
|
|
@ -60,6 +60,7 @@ import type {
|
|||
MessagesPage,
|
||||
ProjectBranchChangeEvent,
|
||||
ReplaceMembersRequest,
|
||||
RetryFailedOpenCodeSecondaryLanesResult,
|
||||
SendMessageRequest,
|
||||
SendMessageResult,
|
||||
TaskAttachmentMeta,
|
||||
|
|
@ -555,6 +556,9 @@ export interface TeamsAPI {
|
|||
getLeadContext: (teamName: string) => Promise<LeadContextUsageSnapshot>;
|
||||
getMemberSpawnStatuses: (teamName: string) => Promise<MemberSpawnStatusesSnapshot>;
|
||||
getTeamAgentRuntime: (teamName: string) => Promise<TeamAgentRuntimeSnapshot>;
|
||||
retryFailedOpenCodeSecondaryLanes: (
|
||||
teamName: string
|
||||
) => Promise<RetryFailedOpenCodeSecondaryLanesResult>;
|
||||
restartMember: (teamName: string, memberName: string) => Promise<void>;
|
||||
skipMemberForLaunch: (teamName: string, memberName: string) => Promise<void>;
|
||||
softDeleteTask: (teamName: string, taskId: string) => Promise<void>;
|
||||
|
|
|
|||
|
|
@ -1079,6 +1079,14 @@ export interface MemberSpawnStatusesSnapshot {
|
|||
source?: 'live' | 'persisted' | 'merged';
|
||||
}
|
||||
|
||||
export interface RetryFailedOpenCodeSecondaryLanesResult {
|
||||
attempted: string[];
|
||||
confirmed: string[];
|
||||
pending: string[];
|
||||
failed: Array<{ memberName: string; error: string }>;
|
||||
skipped: Array<{ memberName: string; reason: string }>;
|
||||
}
|
||||
|
||||
export type MemberSpawnLivenessSource = 'heartbeat' | 'process';
|
||||
|
||||
export type TeamAgentRuntimeBackendType = 'lead' | 'tmux' | 'iterm2' | 'in-process' | 'process';
|
||||
|
|
|
|||
|
|
@ -16195,4 +16195,64 @@ describe('TeamProvisioningService', () => {
|
|||
});
|
||||
expect(run.expectedMembers).toEqual(['alice', 'jack']);
|
||||
});
|
||||
|
||||
it('bulk retries failed OpenCode secondary lanes sequentially and classifies outcomes', async () => {
|
||||
const svc = new TeamProvisioningService();
|
||||
const run = createMemberSpawnRun({
|
||||
teamName: 'mixed-retry-team',
|
||||
runId: 'run-mixed-retry',
|
||||
expectedMembers: ['alice', 'tom', 'nova'],
|
||||
});
|
||||
run.isLaunch = true;
|
||||
run.provisioningComplete = true;
|
||||
|
||||
(svc as any).runs.set(run.runId, run);
|
||||
(svc as any).aliveRunByTeam.set(run.teamName, run.runId);
|
||||
|
||||
vi.spyOn(svc as any, 'collectFailedOpenCodeSecondaryRetryCandidates').mockResolvedValue([
|
||||
{ memberName: 'alice', laneId: 'secondary:opencode:alice' },
|
||||
{ memberName: 'tom', laneId: 'secondary:opencode:tom' },
|
||||
{ memberName: 'nova', laneId: 'secondary:opencode:nova' },
|
||||
]);
|
||||
const reattach = vi
|
||||
.spyOn(svc as any, 'reattachOpenCodeOwnedMemberLane')
|
||||
.mockResolvedValueOnce(undefined)
|
||||
.mockResolvedValueOnce(undefined)
|
||||
.mockRejectedValueOnce(new Error('OpenCode bridge crashed'));
|
||||
vi.spyOn(svc as any, 'readOpenCodeSecondaryRetryOutcome')
|
||||
.mockResolvedValueOnce({ launchState: 'confirmed_alive' })
|
||||
.mockResolvedValueOnce({
|
||||
launchState: 'failed_to_start',
|
||||
reason: 'Latest assistant message reported OpenRouter credits exhausted',
|
||||
});
|
||||
const notify = vi
|
||||
.spyOn(svc as any, 'notifyLeadAboutConfirmedOpenCodeRetries')
|
||||
.mockResolvedValue(undefined);
|
||||
|
||||
const result = await svc.retryFailedOpenCodeSecondaryLanes(run.teamName);
|
||||
|
||||
expect(reattach).toHaveBeenNthCalledWith(1, run.teamName, 'alice', {
|
||||
reason: 'manual_restart',
|
||||
});
|
||||
expect(reattach).toHaveBeenNthCalledWith(2, run.teamName, 'tom', {
|
||||
reason: 'manual_restart',
|
||||
});
|
||||
expect(reattach).toHaveBeenNthCalledWith(3, run.teamName, 'nova', {
|
||||
reason: 'manual_restart',
|
||||
});
|
||||
expect(result).toEqual({
|
||||
attempted: ['alice', 'tom'],
|
||||
confirmed: ['alice'],
|
||||
pending: [],
|
||||
failed: [
|
||||
{
|
||||
memberName: 'tom',
|
||||
error: 'Latest assistant message reported OpenRouter credits exhausted',
|
||||
},
|
||||
{ memberName: 'nova', error: 'OpenCode bridge crashed' },
|
||||
],
|
||||
skipped: [],
|
||||
});
|
||||
expect(notify).toHaveBeenCalledWith(run, result);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|||
const storeState = {
|
||||
progress: null as Record<string, unknown> | null,
|
||||
cancelProvisioning: vi.fn(),
|
||||
retryFailedOpenCodeSecondaryLanes: vi.fn(),
|
||||
selectedTeamName: 'northstar-core',
|
||||
selectedTeamData: {
|
||||
members: [
|
||||
|
|
@ -106,6 +107,13 @@ describe('TeamProvisioningBanner launch-step alignment', () => {
|
|||
cliLogsTail: '',
|
||||
assistantOutput: '',
|
||||
};
|
||||
storeState.retryFailedOpenCodeSecondaryLanes.mockResolvedValue({
|
||||
attempted: [],
|
||||
confirmed: [],
|
||||
pending: [],
|
||||
failed: [],
|
||||
skipped: [],
|
||||
});
|
||||
storeState.memberSpawnStatusesByTeam['northstar-core'] = {};
|
||||
storeState.selectedTeamData.members = [
|
||||
{ name: 'team-lead', agentType: 'team-lead' },
|
||||
|
|
@ -408,6 +416,51 @@ describe('TeamProvisioningBanner launch-step alignment', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('renders a bulk retry action for failed OpenCode secondary teammates', async () => {
|
||||
vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true);
|
||||
storeState.selectedTeamData.members = [
|
||||
{ name: 'team-lead', agentType: 'team-lead', providerId: 'anthropic' },
|
||||
{
|
||||
name: 'alice',
|
||||
agentType: 'developer',
|
||||
providerId: 'opencode',
|
||||
laneKind: 'secondary',
|
||||
laneOwnerProviderId: 'opencode',
|
||||
},
|
||||
];
|
||||
storeState.teamDataCacheByName['northstar-core'] = {
|
||||
members: [...storeState.selectedTeamData.members],
|
||||
};
|
||||
storeState.memberSpawnStatusesByTeam['northstar-core'] = {
|
||||
alice: {
|
||||
status: 'error',
|
||||
launchState: 'failed_to_start',
|
||||
updatedAt: '2026-04-09T10:00:00.000Z',
|
||||
runtimeAlive: false,
|
||||
bootstrapConfirmed: false,
|
||||
hardFailure: true,
|
||||
hardFailureReason: 'OpenRouter credits exhausted',
|
||||
agentToolAccepted: false,
|
||||
},
|
||||
} as Record<string, unknown>;
|
||||
|
||||
const host = document.createElement('div');
|
||||
document.body.appendChild(host);
|
||||
const root = createRoot(host);
|
||||
|
||||
await act(async () => {
|
||||
root.render(React.createElement(TeamProvisioningBanner, { teamName: 'northstar-core' }));
|
||||
await Promise.resolve();
|
||||
});
|
||||
|
||||
expect(host.textContent).toContain('Retry failed OpenCode teammates');
|
||||
|
||||
await act(async () => {
|
||||
root.unmount();
|
||||
await Promise.resolve();
|
||||
});
|
||||
});
|
||||
|
||||
it('uses info severity while runtimes are online but teammate contact is still pending', async () => {
|
||||
vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true);
|
||||
storeState.memberSpawnSnapshotsByTeam['northstar-core'] = {
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ const hoisted = vi.hoisted(() => ({
|
|||
restoreTeam: vi.fn(),
|
||||
permanentlyDeleteTeam: vi.fn(),
|
||||
sendMessage: vi.fn(),
|
||||
retryFailedOpenCodeSecondaryLanes: vi.fn(),
|
||||
restartMember: vi.fn(),
|
||||
skipMemberForLaunch: vi.fn(),
|
||||
requestReview: vi.fn(),
|
||||
|
|
@ -69,6 +70,7 @@ vi.mock('@renderer/api', () => ({
|
|||
restoreTeam: hoisted.restoreTeam,
|
||||
permanentlyDeleteTeam: hoisted.permanentlyDeleteTeam,
|
||||
sendMessage: hoisted.sendMessage,
|
||||
retryFailedOpenCodeSecondaryLanes: hoisted.retryFailedOpenCodeSecondaryLanes,
|
||||
restartMember: hoisted.restartMember,
|
||||
skipMemberForLaunch: hoisted.skipMemberForLaunch,
|
||||
requestReview: hoisted.requestReview,
|
||||
|
|
@ -328,6 +330,13 @@ describe('teamSlice actions', () => {
|
|||
hoisted.deleteTeam.mockResolvedValue(undefined);
|
||||
hoisted.restoreTeam.mockResolvedValue(undefined);
|
||||
hoisted.permanentlyDeleteTeam.mockResolvedValue(undefined);
|
||||
hoisted.retryFailedOpenCodeSecondaryLanes.mockResolvedValue({
|
||||
attempted: [],
|
||||
confirmed: [],
|
||||
pending: [],
|
||||
failed: [],
|
||||
skipped: [],
|
||||
});
|
||||
hoisted.restartMember.mockResolvedValue(undefined);
|
||||
hoisted.skipMemberForLaunch.mockResolvedValue(undefined);
|
||||
});
|
||||
|
|
@ -3394,6 +3403,38 @@ describe('teamSlice actions', () => {
|
|||
expect(store.getState().teamAgentRuntimeByTeam['my-team']).toEqual(createRuntimeSnapshot());
|
||||
});
|
||||
|
||||
it('retryFailedOpenCodeSecondaryLanes refreshes only spawn statuses and runtime snapshot', async () => {
|
||||
const store = createSliceStore();
|
||||
const refreshSpawnStatuses = vi.fn(async (_teamName: string) => undefined);
|
||||
const refreshRuntimeSnapshot = vi.fn(async (_teamName: string) => undefined);
|
||||
const refreshTeamData = vi.fn(async (_teamName: string) => undefined);
|
||||
const fetchTeams = vi.fn(async () => undefined);
|
||||
store.setState({
|
||||
fetchMemberSpawnStatuses: refreshSpawnStatuses,
|
||||
fetchTeamAgentRuntime: refreshRuntimeSnapshot,
|
||||
refreshTeamData,
|
||||
fetchTeams,
|
||||
});
|
||||
hoisted.retryFailedOpenCodeSecondaryLanes.mockResolvedValueOnce({
|
||||
attempted: ['alice'],
|
||||
confirmed: [],
|
||||
pending: [],
|
||||
failed: [{ memberName: 'alice', error: 'OpenRouter credits exhausted' }],
|
||||
skipped: [],
|
||||
});
|
||||
|
||||
const result = await store.getState().retryFailedOpenCodeSecondaryLanes('my-team');
|
||||
|
||||
expect(result.failed).toEqual([
|
||||
{ memberName: 'alice', error: 'OpenRouter credits exhausted' },
|
||||
]);
|
||||
expect(hoisted.retryFailedOpenCodeSecondaryLanes).toHaveBeenCalledWith('my-team');
|
||||
expect(refreshSpawnStatuses).toHaveBeenCalledWith('my-team');
|
||||
expect(refreshRuntimeSnapshot).toHaveBeenCalledWith('my-team');
|
||||
expect(refreshTeamData).not.toHaveBeenCalled();
|
||||
expect(fetchTeams).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('restartMember refreshes spawn statuses and runtime snapshot even when restart fails', async () => {
|
||||
const store = createSliceStore();
|
||||
const refreshSpawnStatuses = vi.fn(async (_teamName: string) => undefined);
|
||||
|
|
|
|||
|
|
@ -91,6 +91,128 @@ describe('buildTeamProvisioningPresentation', () => {
|
|||
expect(presentation?.defaultLiveOutputOpen).toBe(false);
|
||||
});
|
||||
|
||||
it('counts retryable failed OpenCode secondary teammates conservatively', () => {
|
||||
const presentation = buildTeamProvisioningPresentation({
|
||||
progress: {
|
||||
runId: 'run-opencode-retry',
|
||||
teamName: 'mixed-team',
|
||||
state: 'ready',
|
||||
startedAt: '2026-04-13T10:00:00.000Z',
|
||||
updatedAt: '2026-04-13T10:00:08.000Z',
|
||||
message: 'Launch completed with teammate errors',
|
||||
messageSeverity: 'warning',
|
||||
pid: 4321,
|
||||
cliLogsTail: '',
|
||||
assistantOutput: '',
|
||||
},
|
||||
members: [
|
||||
{
|
||||
name: 'team-lead',
|
||||
agentType: 'team-lead',
|
||||
providerId: 'anthropic',
|
||||
},
|
||||
{
|
||||
name: 'alice',
|
||||
agentType: 'developer',
|
||||
providerId: 'opencode',
|
||||
laneKind: 'secondary',
|
||||
laneOwnerProviderId: 'opencode',
|
||||
},
|
||||
{
|
||||
name: 'bob',
|
||||
agentType: 'developer',
|
||||
providerId: 'anthropic',
|
||||
laneKind: 'primary',
|
||||
},
|
||||
],
|
||||
memberSpawnStatuses: {
|
||||
alice: {
|
||||
status: 'error',
|
||||
launchState: 'failed_to_start',
|
||||
hardFailureReason: 'OpenRouter credits exhausted',
|
||||
updatedAt: '2026-04-13T10:00:05.000Z',
|
||||
runtimeAlive: false,
|
||||
bootstrapConfirmed: false,
|
||||
hardFailure: true,
|
||||
agentToolAccepted: false,
|
||||
},
|
||||
bob: {
|
||||
status: 'error',
|
||||
launchState: 'failed_to_start',
|
||||
hardFailureReason: 'Primary lane failed',
|
||||
updatedAt: '2026-04-13T10:00:05.000Z',
|
||||
runtimeAlive: false,
|
||||
bootstrapConfirmed: false,
|
||||
hardFailure: true,
|
||||
agentToolAccepted: false,
|
||||
},
|
||||
},
|
||||
memberSpawnSnapshot: undefined,
|
||||
});
|
||||
|
||||
expect(presentation?.retryableOpenCodeSecondaryFailedNames).toEqual(['alice']);
|
||||
expect(presentation?.retryableOpenCodeSecondaryFailedCount).toBe(1);
|
||||
});
|
||||
|
||||
it('does not count skipped or permission-blocked OpenCode failures as bulk retry candidates', () => {
|
||||
const presentation = buildTeamProvisioningPresentation({
|
||||
progress: {
|
||||
runId: 'run-opencode-no-retry',
|
||||
teamName: 'mixed-team',
|
||||
state: 'ready',
|
||||
startedAt: '2026-04-13T10:00:00.000Z',
|
||||
updatedAt: '2026-04-13T10:00:08.000Z',
|
||||
message: 'Launch completed with teammate errors',
|
||||
messageSeverity: 'warning',
|
||||
pid: 4321,
|
||||
cliLogsTail: '',
|
||||
assistantOutput: '',
|
||||
},
|
||||
members: [
|
||||
{
|
||||
name: 'alice',
|
||||
agentType: 'developer',
|
||||
providerId: 'opencode',
|
||||
laneKind: 'secondary',
|
||||
laneOwnerProviderId: 'opencode',
|
||||
},
|
||||
{
|
||||
name: 'tom',
|
||||
agentType: 'developer',
|
||||
providerId: 'opencode',
|
||||
laneKind: 'secondary',
|
||||
laneOwnerProviderId: 'opencode',
|
||||
},
|
||||
],
|
||||
memberSpawnStatuses: {
|
||||
alice: {
|
||||
status: 'skipped',
|
||||
launchState: 'skipped_for_launch',
|
||||
skippedForLaunch: true,
|
||||
updatedAt: '2026-04-13T10:00:05.000Z',
|
||||
runtimeAlive: false,
|
||||
bootstrapConfirmed: false,
|
||||
hardFailure: false,
|
||||
agentToolAccepted: false,
|
||||
},
|
||||
tom: {
|
||||
status: 'waiting',
|
||||
launchState: 'runtime_pending_permission',
|
||||
pendingPermissionRequestIds: ['perm-1'],
|
||||
updatedAt: '2026-04-13T10:00:05.000Z',
|
||||
runtimeAlive: true,
|
||||
bootstrapConfirmed: false,
|
||||
hardFailure: false,
|
||||
agentToolAccepted: true,
|
||||
},
|
||||
},
|
||||
memberSpawnSnapshot: undefined,
|
||||
});
|
||||
|
||||
expect(presentation?.retryableOpenCodeSecondaryFailedNames).toEqual([]);
|
||||
expect(presentation?.retryableOpenCodeSecondaryFailedCount).toBe(0);
|
||||
});
|
||||
|
||||
it('does not truncate long failed teammate reasons in the panel message', () => {
|
||||
const reason =
|
||||
'You are bootstrapping into team "relay-works-10" as member "alice". Your first action is to call the MCP tool member_briefing on the agent-teams server with teamName="relay-works-10" and memberName="alice". If tool search shows only the prefixed MCP name, use mcp__agent-teams__member_briefing.';
|
||||
|
|
|
|||
Loading…
Reference in a new issue