fix(team): harden launch failure recovery copy
This commit is contained in:
parent
5c5b5c7afc
commit
e6e3ae9f54
5 changed files with 207 additions and 19 deletions
|
|
@ -1560,7 +1560,10 @@ function isConfigRegistrationFailureReason(reason?: string): boolean {
|
||||||
|
|
||||||
function isTmuxNoServerRunningError(error: unknown): boolean {
|
function isTmuxNoServerRunningError(error: unknown): boolean {
|
||||||
const text = error instanceof Error ? error.message : String(error ?? '');
|
const text = error instanceof Error ? error.message : String(error ?? '');
|
||||||
return /no server running on /i.test(text);
|
return (
|
||||||
|
/no server running on /i.test(text) ||
|
||||||
|
/error connecting to .*no such file or directory/i.test(text)
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
function isAutoClearableLaunchFailureReason(reason?: string): boolean {
|
function isAutoClearableLaunchFailureReason(reason?: string): boolean {
|
||||||
|
|
@ -2194,6 +2197,7 @@ function extractHeartbeatTimestamp(text: string, fallback?: string): string | un
|
||||||
function extractBootstrapFailureReason(text: string): string | null {
|
function extractBootstrapFailureReason(text: string): string | null {
|
||||||
const trimmed = text.trim();
|
const trimmed = text.trim();
|
||||||
if (!trimmed) return null;
|
if (!trimmed) return null;
|
||||||
|
if (isBootstrapInstructionPrompt(trimmed)) return null;
|
||||||
const lower = trimmed.toLowerCase();
|
const lower = trimmed.toLowerCase();
|
||||||
const looksLikeBootstrapFailure =
|
const looksLikeBootstrapFailure =
|
||||||
lower.includes('bootstrap failed') ||
|
lower.includes('bootstrap failed') ||
|
||||||
|
|
@ -2241,6 +2245,17 @@ function extractBootstrapFailureReason(text: string): string | null {
|
||||||
return trimmed.slice(0, 280);
|
return trimmed.slice(0, 280);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function isBootstrapInstructionPrompt(text: string): boolean {
|
||||||
|
const normalized = text.replace(/\s+/g, ' ').trim().toLowerCase();
|
||||||
|
if (!normalized.startsWith('you are bootstrapping into team ')) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return (
|
||||||
|
normalized.includes('your first action is to call the mcp tool') &&
|
||||||
|
(normalized.includes('member_briefing') || normalized.includes('lead_briefing'))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
function isBootstrapTranscriptSuccessText(
|
function isBootstrapTranscriptSuccessText(
|
||||||
text: string,
|
text: string,
|
||||||
teamName: string,
|
teamName: string,
|
||||||
|
|
@ -14942,7 +14957,7 @@ export class TeamProvisioningService {
|
||||||
const expectedMembers = this.getPersistedLaunchMemberNames(snapshot);
|
const expectedMembers = this.getPersistedLaunchMemberNames(snapshot);
|
||||||
for (const expected of expectedMembers) {
|
for (const expected of expectedMembers) {
|
||||||
const current = snapshot.members[expected];
|
const current = snapshot.members[expected];
|
||||||
if (!current || current.bootstrapConfirmed || current.hardFailure) {
|
if (!current || current.bootstrapConfirmed) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
const acceptedAtMs =
|
const acceptedAtMs =
|
||||||
|
|
@ -15128,7 +15143,7 @@ export class TeamProvisioningService {
|
||||||
current.hardFailure = false;
|
current.hardFailure = false;
|
||||||
current.hardFailureReason = undefined;
|
current.hardFailureReason = undefined;
|
||||||
}
|
}
|
||||||
if (!current.bootstrapConfirmed && !current.hardFailure) {
|
if (!current.bootstrapConfirmed) {
|
||||||
const transcriptOutcome = await this.findBootstrapTranscriptOutcome(
|
const transcriptOutcome = await this.findBootstrapTranscriptOutcome(
|
||||||
teamName,
|
teamName,
|
||||||
expected,
|
expected,
|
||||||
|
|
@ -15139,7 +15154,10 @@ export class TeamProvisioningService {
|
||||||
current.lastHeartbeatAt = current.lastHeartbeatAt ?? transcriptOutcome.observedAt;
|
current.lastHeartbeatAt = current.lastHeartbeatAt ?? transcriptOutcome.observedAt;
|
||||||
current.hardFailure = false;
|
current.hardFailure = false;
|
||||||
current.hardFailureReason = undefined;
|
current.hardFailureReason = undefined;
|
||||||
} else if (transcriptOutcome?.kind === 'failure') {
|
if (current.sources) {
|
||||||
|
current.sources.hardFailureSignal = undefined;
|
||||||
|
}
|
||||||
|
} else if (transcriptOutcome?.kind === 'failure' && !current.hardFailure) {
|
||||||
current.hardFailure = true;
|
current.hardFailure = true;
|
||||||
current.hardFailureReason = transcriptOutcome.reason;
|
current.hardFailureReason = transcriptOutcome.reason;
|
||||||
current.sources.hardFailureSignal = true;
|
current.sources.hardFailureSignal = true;
|
||||||
|
|
|
||||||
|
|
@ -390,12 +390,8 @@ function getSkippedSpawnDetails(params: {
|
||||||
.sort((left, right) => left.name.localeCompare(right.name));
|
.sort((left, right) => left.name.localeCompare(right.name));
|
||||||
}
|
}
|
||||||
|
|
||||||
function truncateFailureReason(reason: string, maxLength = 160): string {
|
function normalizeFailureReason(reason: string): string {
|
||||||
const normalized = reason.replace(/\s+/g, ' ').trim();
|
return reason.replace(/\s+/g, ' ').trim();
|
||||||
if (normalized.length <= maxLength) {
|
|
||||||
return normalized;
|
|
||||||
}
|
|
||||||
return `${normalized.slice(0, Math.max(0, maxLength - 3)).trimEnd()}...`;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function buildFailedSpawnPanelMessage(
|
function buildFailedSpawnPanelMessage(
|
||||||
|
|
@ -407,13 +403,13 @@ function buildFailedSpawnPanelMessage(
|
||||||
if (failedSpawnDetails.length === 1) {
|
if (failedSpawnDetails.length === 1) {
|
||||||
const [failed] = failedSpawnDetails;
|
const [failed] = failedSpawnDetails;
|
||||||
return failed.reason
|
return failed.reason
|
||||||
? `${failed.name} failed to start - ${truncateFailureReason(failed.reason, 220)}`
|
? `${failed.name} failed to start - ${normalizeFailureReason(failed.reason)}`
|
||||||
: `${failed.name} failed to start`;
|
: `${failed.name} failed to start`;
|
||||||
}
|
}
|
||||||
const listedFailures = failedSpawnDetails
|
const listedFailures = failedSpawnDetails
|
||||||
.slice(0, 2)
|
.slice(0, 2)
|
||||||
.map((failed) =>
|
.map((failed) =>
|
||||||
failed.reason ? `${failed.name} - ${truncateFailureReason(failed.reason, 120)}` : failed.name
|
failed.reason ? `${failed.name} - ${normalizeFailureReason(failed.reason)}` : failed.name
|
||||||
)
|
)
|
||||||
.join('; ');
|
.join('; ');
|
||||||
const remainingCount = failedSpawnDetails.length - Math.min(failedSpawnDetails.length, 2);
|
const remainingCount = failedSpawnDetails.length - Math.min(failedSpawnDetails.length, 2);
|
||||||
|
|
@ -454,15 +450,13 @@ function buildSkippedSpawnPanelMessage(
|
||||||
if (skippedSpawnDetails.length === 1) {
|
if (skippedSpawnDetails.length === 1) {
|
||||||
const [skipped] = skippedSpawnDetails;
|
const [skipped] = skippedSpawnDetails;
|
||||||
return skipped.reason
|
return skipped.reason
|
||||||
? `${skipped.name} skipped for this launch - ${truncateFailureReason(skipped.reason, 220)}`
|
? `${skipped.name} skipped for this launch - ${normalizeFailureReason(skipped.reason)}`
|
||||||
: `${skipped.name} skipped for this launch`;
|
: `${skipped.name} skipped for this launch`;
|
||||||
}
|
}
|
||||||
const listedSkipped = skippedSpawnDetails
|
const listedSkipped = skippedSpawnDetails
|
||||||
.slice(0, 3)
|
.slice(0, 3)
|
||||||
.map((skipped) =>
|
.map((skipped) =>
|
||||||
skipped.reason
|
skipped.reason ? `${skipped.name} - ${normalizeFailureReason(skipped.reason)}` : skipped.name
|
||||||
? `${skipped.name} - ${truncateFailureReason(skipped.reason, 100)}`
|
|
||||||
: skipped.name
|
|
||||||
)
|
)
|
||||||
.join('; ');
|
.join('; ');
|
||||||
const remainingCount = skippedSpawnDetails.length - Math.min(skippedSpawnDetails.length, 3);
|
const remainingCount = skippedSpawnDetails.length - Math.min(skippedSpawnDetails.length, 3);
|
||||||
|
|
|
||||||
|
|
@ -4747,7 +4747,7 @@ describe('TeamProvisioningService', () => {
|
||||||
(svc as any).runs.set(run.runId, run);
|
(svc as any).runs.set(run.runId, run);
|
||||||
|
|
||||||
vi.mocked(listTmuxPanePidsForCurrentPlatform).mockRejectedValue(
|
vi.mocked(listTmuxPanePidsForCurrentPlatform).mockRejectedValue(
|
||||||
new Error('no server running on /private/tmp/tmux-501/default')
|
new Error('error connecting to /private/tmp/tmux-501/default (No such file or directory)')
|
||||||
);
|
);
|
||||||
|
|
||||||
await svc.restartMember('tmux-team', 'forge');
|
await svc.restartMember('tmux-team', 'forge');
|
||||||
|
|
@ -6558,12 +6558,13 @@ describe('TeamProvisioningService', () => {
|
||||||
firstSpawnAcceptedAt: acceptedAt,
|
firstSpawnAcceptedAt: acceptedAt,
|
||||||
},
|
},
|
||||||
bob: {
|
bob: {
|
||||||
launchState: 'starting',
|
launchState: 'runtime_pending_bootstrap',
|
||||||
agentToolAccepted: false,
|
agentToolAccepted: true,
|
||||||
runtimeAlive: false,
|
runtimeAlive: false,
|
||||||
bootstrapConfirmed: false,
|
bootstrapConfirmed: false,
|
||||||
hardFailure: false,
|
hardFailure: false,
|
||||||
hardFailureReason: undefined,
|
hardFailureReason: undefined,
|
||||||
|
firstSpawnAcceptedAt: acceptedAt,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
@ -6639,6 +6640,124 @@ describe('TeamProvisioningService', () => {
|
||||||
expect(result.statuses.alice?.error).toBeUndefined();
|
expect(result.statuses.alice?.error).toBeUndefined();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('does not classify the bootstrap instruction prompt as a member launch failure', async () => {
|
||||||
|
allowConsoleLogs();
|
||||||
|
const teamName = 'zz-unit-bootstrap-prompt-not-failure';
|
||||||
|
const leadSessionId = 'lead-session';
|
||||||
|
const memberSessionId = 'alice-session';
|
||||||
|
const projectPath = '/Users/test/proj';
|
||||||
|
const projectId = '-Users-test-proj';
|
||||||
|
const acceptedAt = new Date(Date.now() - 5_000).toISOString();
|
||||||
|
|
||||||
|
writeLaunchConfig(teamName, projectPath, leadSessionId, ['alice']);
|
||||||
|
|
||||||
|
const projectRoot = path.join(tempProjectsBase, projectId);
|
||||||
|
fs.mkdirSync(projectRoot, { recursive: true });
|
||||||
|
fs.writeFileSync(
|
||||||
|
path.join(projectRoot, `${memberSessionId}.jsonl`),
|
||||||
|
`${JSON.stringify({
|
||||||
|
timestamp: acceptedAt,
|
||||||
|
teamName,
|
||||||
|
agentName: 'alice',
|
||||||
|
type: 'user',
|
||||||
|
message: {
|
||||||
|
role: 'user',
|
||||||
|
content: `You are bootstrapping into team "${teamName}" as member "alice".\nYour first action is to call the MCP tool member_briefing on the agent-teams server with teamName="${teamName}" and memberName="alice".\nIf member_briefing is still unavailable after that one retry, send exactly one short SendMessage to "team-lead" with the exact error text, then stop this turn and wait.`,
|
||||||
|
},
|
||||||
|
})}\n`,
|
||||||
|
'utf8'
|
||||||
|
);
|
||||||
|
|
||||||
|
const svc = new TeamProvisioningService();
|
||||||
|
const reason = await (svc as any).findBootstrapTranscriptFailureReason(
|
||||||
|
teamName,
|
||||||
|
'alice',
|
||||||
|
Date.parse(acceptedAt) - 1
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(reason).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('clears a stale persisted bootstrap-prompt failure when member_briefing later succeeds', async () => {
|
||||||
|
allowConsoleLogs();
|
||||||
|
const teamName = 'zz-unit-bootstrap-stale-prompt-failure';
|
||||||
|
const leadSessionId = 'lead-session';
|
||||||
|
const memberSessionId = 'alice-session';
|
||||||
|
const projectPath = '/Users/test/proj';
|
||||||
|
const projectId = '-Users-test-proj';
|
||||||
|
const acceptedAt = new Date(Date.now() - 5_000).toISOString();
|
||||||
|
const successAt = new Date(Date.now() - 4_000).toISOString();
|
||||||
|
const staleReason = `You are bootstrapping into team "${teamName}" as member "alice".\nYour first action is to call the MCP tool member_briefing on the agent-teams server with teamName="${teamName}" and memberName="alice".\nIf tool search shows only the prefixed MCP name, use mcp__agent-teams__member_briefing.`;
|
||||||
|
|
||||||
|
writeLaunchConfig(teamName, projectPath, leadSessionId, ['alice', 'bob']);
|
||||||
|
writeLaunchState(teamName, leadSessionId, {
|
||||||
|
alice: {
|
||||||
|
launchState: 'failed_to_start',
|
||||||
|
agentToolAccepted: true,
|
||||||
|
runtimeAlive: false,
|
||||||
|
bootstrapConfirmed: false,
|
||||||
|
hardFailure: true,
|
||||||
|
hardFailureReason: staleReason,
|
||||||
|
firstSpawnAcceptedAt: acceptedAt,
|
||||||
|
},
|
||||||
|
bob: {
|
||||||
|
launchState: 'starting',
|
||||||
|
agentToolAccepted: false,
|
||||||
|
runtimeAlive: false,
|
||||||
|
bootstrapConfirmed: false,
|
||||||
|
hardFailure: false,
|
||||||
|
hardFailureReason: undefined,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const projectRoot = path.join(tempProjectsBase, projectId);
|
||||||
|
fs.mkdirSync(projectRoot, { recursive: true });
|
||||||
|
fs.writeFileSync(
|
||||||
|
path.join(projectRoot, `${memberSessionId}.jsonl`),
|
||||||
|
[
|
||||||
|
JSON.stringify({
|
||||||
|
timestamp: acceptedAt,
|
||||||
|
teamName,
|
||||||
|
agentName: 'alice',
|
||||||
|
type: 'user',
|
||||||
|
message: {
|
||||||
|
role: 'user',
|
||||||
|
content: staleReason,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
JSON.stringify({
|
||||||
|
timestamp: successAt,
|
||||||
|
teamName,
|
||||||
|
agentName: 'alice',
|
||||||
|
type: 'user',
|
||||||
|
message: {
|
||||||
|
role: 'user',
|
||||||
|
content: [
|
||||||
|
{
|
||||||
|
type: 'tool_result',
|
||||||
|
tool_use_id: 'item_1',
|
||||||
|
content: `Member briefing for alice on team "${teamName}" (${teamName}).\nTask briefing for alice:\nNo actionable tasks.`,
|
||||||
|
is_error: false,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
].join('\n') + '\n',
|
||||||
|
'utf8'
|
||||||
|
);
|
||||||
|
|
||||||
|
const svc = new TeamProvisioningService();
|
||||||
|
const result = await svc.getMemberSpawnStatuses(teamName);
|
||||||
|
|
||||||
|
expect(result.statuses.alice).toMatchObject({
|
||||||
|
status: 'online',
|
||||||
|
launchState: 'confirmed_alive',
|
||||||
|
bootstrapConfirmed: true,
|
||||||
|
hardFailure: false,
|
||||||
|
});
|
||||||
|
expect(result.statuses.alice?.hardFailureReason).toBeUndefined();
|
||||||
|
});
|
||||||
|
|
||||||
it('marks an online teammate bootstrap as failed when transcript shows model unavailability', async () => {
|
it('marks an online teammate bootstrap as failed when transcript shows model unavailability', async () => {
|
||||||
allowConsoleLogs();
|
allowConsoleLogs();
|
||||||
const teamName = 'zz-live-bootstrap-model-unavailable';
|
const teamName = 'zz-live-bootstrap-model-unavailable';
|
||||||
|
|
|
||||||
|
|
@ -172,6 +172,7 @@ const REQUIRED_MOCK_AGENT_TEAMS_TOOLS = [
|
||||||
'task_get_comment',
|
'task_get_comment',
|
||||||
'task_link',
|
'task_link',
|
||||||
'task_list',
|
'task_list',
|
||||||
|
'task_restore',
|
||||||
'task_set_clarification',
|
'task_set_clarification',
|
||||||
'task_set_owner',
|
'task_set_owner',
|
||||||
'task_set_status',
|
'task_set_status',
|
||||||
|
|
|
||||||
|
|
@ -91,6 +91,62 @@ describe('buildTeamProvisioningPresentation', () => {
|
||||||
expect(presentation?.defaultLiveOutputOpen).toBe(false);
|
expect(presentation?.defaultLiveOutputOpen).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('does not truncate long failed teammate reasons in the panel message', () => {
|
||||||
|
const reason =
|
||||||
|
'You are bootstrapping into team "relay-works-10" as member "alice". Your first action is to call the MCP tool member_briefing on the agent-teams server with teamName="relay-works-10" and memberName="alice". If tool search shows only the prefixed MCP name, use mcp__agent-teams__member_briefing.';
|
||||||
|
const presentation = buildTeamProvisioningPresentation({
|
||||||
|
progress: {
|
||||||
|
runId: 'run-long-failure',
|
||||||
|
teamName: 'relay-works-10',
|
||||||
|
state: 'ready',
|
||||||
|
startedAt: '2026-04-13T10:00:00.000Z',
|
||||||
|
updatedAt: '2026-04-13T10:00:08.000Z',
|
||||||
|
message: 'Launch completed with teammate errors',
|
||||||
|
messageSeverity: 'warning',
|
||||||
|
pid: 4321,
|
||||||
|
cliLogsTail: '',
|
||||||
|
assistantOutput: '',
|
||||||
|
},
|
||||||
|
members: [
|
||||||
|
{
|
||||||
|
name: 'team-lead',
|
||||||
|
agentType: 'team-lead',
|
||||||
|
status: 'active',
|
||||||
|
currentTaskId: null,
|
||||||
|
taskCount: 0,
|
||||||
|
lastActiveAt: null,
|
||||||
|
messageCount: 0,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'alice',
|
||||||
|
agentType: 'reviewer',
|
||||||
|
status: 'unknown',
|
||||||
|
currentTaskId: null,
|
||||||
|
taskCount: 0,
|
||||||
|
lastActiveAt: null,
|
||||||
|
messageCount: 0,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
memberSpawnStatuses: {
|
||||||
|
alice: {
|
||||||
|
status: 'error',
|
||||||
|
launchState: 'failed_to_start',
|
||||||
|
error: reason,
|
||||||
|
hardFailureReason: reason,
|
||||||
|
updatedAt: '2026-04-13T10:00:03.000Z',
|
||||||
|
runtimeAlive: false,
|
||||||
|
bootstrapConfirmed: false,
|
||||||
|
hardFailure: true,
|
||||||
|
agentToolAccepted: true,
|
||||||
|
firstSpawnAcceptedAt: '2026-04-13T10:00:01.000Z',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
memberSpawnSnapshot: undefined,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(presentation?.panelMessage).toBe(`alice failed to start - ${reason}`);
|
||||||
|
});
|
||||||
|
|
||||||
it('surfaces the failed teammate reason after launch completes with errors', () => {
|
it('surfaces the failed teammate reason after launch completes with errors', () => {
|
||||||
const presentation = buildTeamProvisioningPresentation({
|
const presentation = buildTeamProvisioningPresentation({
|
||||||
progress: {
|
progress: {
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue