fix(team): preserve mixed launch failure diagnostics
This commit is contained in:
parent
b1d27c1382
commit
f57d15c68f
4 changed files with 1005 additions and 22 deletions
|
|
@ -2293,6 +2293,105 @@ function hasRealOpenCodeFailureDiagnostic(text: string): boolean {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const OPEN_CODE_GENERIC_MEMBER_LAUNCH_FAILURE_REASON =
|
||||||
|
'OpenCode bridge reported member launch failure';
|
||||||
|
const OPEN_CODE_SECRET_FLAG_PATTERN =
|
||||||
|
/(--(?:api-key|token|password|secret|authorization|auth-token)(?:=|\s+))("[^"]*"|'[^']*'|\S+)/gi;
|
||||||
|
const OPEN_CODE_BEARER_TOKEN_PATTERN = /\bBearer\s+[A-Za-z0-9._~+/=-]+/gi;
|
||||||
|
const OPEN_CODE_SECRET_KEY_PATTERN = /\bsk-[A-Za-z0-9_-]{16,}\b/g;
|
||||||
|
|
||||||
|
function normalizeOpenCodePersistedFailureReason(value: string | undefined): string | undefined {
|
||||||
|
const trimmed = value?.replace(/\s+/g, ' ').trim();
|
||||||
|
if (!trimmed) {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
return trimmed
|
||||||
|
.replace(OPEN_CODE_SECRET_FLAG_PATTERN, '$1[redacted]')
|
||||||
|
.replace(OPEN_CODE_BEARER_TOKEN_PATTERN, 'Bearer [redacted]')
|
||||||
|
.replace(OPEN_CODE_SECRET_KEY_PATTERN, '[redacted-api-key]');
|
||||||
|
}
|
||||||
|
|
||||||
|
function isGenericOpenCodePersistedFailureReason(value: string | undefined): boolean {
|
||||||
|
const normalized = normalizeOpenCodePersistedFailureReason(value);
|
||||||
|
return (
|
||||||
|
normalized === OPEN_CODE_GENERIC_MEMBER_LAUNCH_FAILURE_REASON ||
|
||||||
|
normalized?.startsWith(`${OPEN_CODE_GENERIC_MEMBER_LAUNCH_FAILURE_REASON}:`) === true ||
|
||||||
|
normalized?.startsWith('OpenCode secondary lane timing:') === true ||
|
||||||
|
normalized?.startsWith(
|
||||||
|
'OpenCode bridge reported ready without all required durable checkpoints:'
|
||||||
|
) === true ||
|
||||||
|
normalized?.startsWith(
|
||||||
|
'OpenCode bridge reported ready before all expected members were confirmed:'
|
||||||
|
) === true ||
|
||||||
|
normalized?.startsWith(
|
||||||
|
'OpenCode bootstrap MCP did not complete required tools before assistant response:'
|
||||||
|
) === true ||
|
||||||
|
normalized?.startsWith('info:opencode_launch_member_timing:') === true ||
|
||||||
|
normalized?.startsWith('info:opencode_launch_total_timing:') === true
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function selectOpenCodePersistedFailureReasonFromDiagnostics(
|
||||||
|
member: PersistedTeamLaunchMemberState
|
||||||
|
): string | undefined {
|
||||||
|
if (!isPersistedOpenCodeSecondaryLaneMember(member)) {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
if (member.launchState !== 'failed_to_start' || member.hardFailure !== true) {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
if (!isGenericOpenCodePersistedFailureReason(member.hardFailureReason)) {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
for (const value of member.diagnostics ?? []) {
|
||||||
|
const normalized = normalizeOpenCodePersistedFailureReason(value);
|
||||||
|
if (!normalized || isGenericOpenCodePersistedFailureReason(normalized)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
return normalized;
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
function promoteOpenCodePersistedFailureReasonsFromDiagnostics(
|
||||||
|
snapshot: PersistedTeamLaunchSnapshot | null
|
||||||
|
): PersistedTeamLaunchSnapshot | null {
|
||||||
|
if (!snapshot) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
let changed = false;
|
||||||
|
const members: Record<string, PersistedTeamLaunchMemberState> = { ...snapshot.members };
|
||||||
|
for (const [memberName, member] of Object.entries(snapshot.members)) {
|
||||||
|
const promotedReason = selectOpenCodePersistedFailureReasonFromDiagnostics(member);
|
||||||
|
if (!promotedReason || promotedReason === member.hardFailureReason) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
members[memberName] = {
|
||||||
|
...member,
|
||||||
|
hardFailureReason: promotedReason,
|
||||||
|
runtimeDiagnostic:
|
||||||
|
member.runtimeDiagnostic &&
|
||||||
|
!isGenericOpenCodePersistedFailureReason(member.runtimeDiagnostic)
|
||||||
|
? member.runtimeDiagnostic
|
||||||
|
: promotedReason,
|
||||||
|
runtimeDiagnosticSeverity: member.runtimeDiagnosticSeverity ?? 'error',
|
||||||
|
};
|
||||||
|
changed = true;
|
||||||
|
}
|
||||||
|
if (!changed) {
|
||||||
|
return snapshot;
|
||||||
|
}
|
||||||
|
return createPersistedLaunchSnapshot({
|
||||||
|
teamName: snapshot.teamName,
|
||||||
|
expectedMembers: snapshot.expectedMembers,
|
||||||
|
bootstrapExpectedMembers: snapshot.bootstrapExpectedMembers,
|
||||||
|
leadSessionId: snapshot.leadSessionId,
|
||||||
|
launchPhase: snapshot.launchPhase,
|
||||||
|
members,
|
||||||
|
updatedAt: nowIso(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
function promoteOpenCodeSecondaryMemberFromCommittedBootstrapEvidence(input: {
|
function promoteOpenCodeSecondaryMemberFromCommittedBootstrapEvidence(input: {
|
||||||
current: PersistedTeamLaunchMemberState;
|
current: PersistedTeamLaunchMemberState;
|
||||||
previous: PersistedTeamLaunchMemberState | null;
|
previous: PersistedTeamLaunchMemberState | null;
|
||||||
|
|
@ -19770,6 +19869,99 @@ export class TeamProvisioningService {
|
||||||
return statuses;
|
return statuses;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async overlayPrimaryBootstrapTruthIntoRunStatusesFromBootstrapState(
|
||||||
|
run: ProvisioningRun
|
||||||
|
): Promise<void> {
|
||||||
|
if (
|
||||||
|
!run.isLaunch ||
|
||||||
|
!Array.isArray(run.mixedSecondaryLanes) ||
|
||||||
|
run.mixedSecondaryLanes.length === 0
|
||||||
|
) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let bootstrapSnapshot: PersistedTeamLaunchSnapshot | null = null;
|
||||||
|
try {
|
||||||
|
bootstrapSnapshot = await readBootstrapLaunchSnapshot(run.teamName);
|
||||||
|
} catch {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!bootstrapSnapshot) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const runStartedAtMs = Date.parse(run.startedAt);
|
||||||
|
const bootstrapUpdatedAtMs = Date.parse(bootstrapSnapshot.updatedAt);
|
||||||
|
if (
|
||||||
|
!Number.isFinite(runStartedAtMs) ||
|
||||||
|
!Number.isFinite(bootstrapUpdatedAtMs) ||
|
||||||
|
bootstrapUpdatedAtMs < runStartedAtMs
|
||||||
|
) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const primaryMemberNames = new Set(
|
||||||
|
(run.effectiveMembers ?? [])
|
||||||
|
.map((member) => member.name?.trim())
|
||||||
|
.filter((name): name is string => Boolean(name))
|
||||||
|
);
|
||||||
|
if (primaryMemberNames.size === 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const updatedAt = nowIso();
|
||||||
|
for (const memberName of primaryMemberNames) {
|
||||||
|
if (this.isOpenCodeSecondaryLaneMemberInRun(run, memberName)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const bootstrapMember = bootstrapSnapshot.members[memberName];
|
||||||
|
if (bootstrapMember?.bootstrapConfirmed !== true) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const current =
|
||||||
|
run.memberSpawnStatuses.get(memberName) ?? createInitialMemberSpawnStatusEntry();
|
||||||
|
if (current.launchState === 'skipped_for_launch' || current.skippedForLaunch === true) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const failureReason = current.hardFailureReason ?? current.error;
|
||||||
|
if (
|
||||||
|
current.launchState === 'failed_to_start' &&
|
||||||
|
!isAutoClearableLaunchFailureReason(failureReason)
|
||||||
|
) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const observedAt =
|
||||||
|
bootstrapMember.lastHeartbeatAt ??
|
||||||
|
bootstrapMember.lastEvaluatedAt ??
|
||||||
|
bootstrapSnapshot.updatedAt ??
|
||||||
|
updatedAt;
|
||||||
|
const next: MemberSpawnStatusEntry = {
|
||||||
|
...current,
|
||||||
|
status: 'online',
|
||||||
|
updatedAt,
|
||||||
|
agentToolAccepted: true,
|
||||||
|
runtimeAlive: true,
|
||||||
|
bootstrapConfirmed: true,
|
||||||
|
hardFailure: false,
|
||||||
|
bootstrapStalled: undefined,
|
||||||
|
error: undefined,
|
||||||
|
hardFailureReason: undefined,
|
||||||
|
livenessSource: current.livenessSource ?? 'heartbeat',
|
||||||
|
firstSpawnAcceptedAt:
|
||||||
|
current.firstSpawnAcceptedAt ?? bootstrapMember.firstSpawnAcceptedAt ?? observedAt,
|
||||||
|
lastHeartbeatAt: isMemberSpawnHeartbeatTimestampNewer(current.lastHeartbeatAt, observedAt)
|
||||||
|
? observedAt
|
||||||
|
: current.lastHeartbeatAt,
|
||||||
|
livenessLastCheckedAt: updatedAt,
|
||||||
|
launchState: 'confirmed_alive',
|
||||||
|
};
|
||||||
|
run.memberSpawnStatuses.set(memberName, next);
|
||||||
|
run.pendingMemberRestarts?.delete(memberName);
|
||||||
|
this.syncMemberLaunchGraceCheck(run, memberName, next);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private syncRunMemberSpawnStatusesFromSnapshot(
|
private syncRunMemberSpawnStatusesFromSnapshot(
|
||||||
run: ProvisioningRun,
|
run: ProvisioningRun,
|
||||||
snapshot: PersistedTeamLaunchSnapshot
|
snapshot: PersistedTeamLaunchSnapshot
|
||||||
|
|
@ -20191,6 +20383,7 @@ export class TeamProvisioningService {
|
||||||
run: ProvisioningRun,
|
run: ProvisioningRun,
|
||||||
launchPhase: 'active' | 'finished' | 'reconciled'
|
launchPhase: 'active' | 'finished' | 'reconciled'
|
||||||
): Promise<PersistedTeamLaunchSnapshot | null> {
|
): Promise<PersistedTeamLaunchSnapshot | null> {
|
||||||
|
await this.overlayPrimaryBootstrapTruthIntoRunStatusesFromBootstrapState(run);
|
||||||
const snapshot = this.buildLiveLaunchSnapshotForRun(run, launchPhase);
|
const snapshot = this.buildLiveLaunchSnapshotForRun(run, launchPhase);
|
||||||
if (!snapshot) {
|
if (!snapshot) {
|
||||||
if (run.isLaunch) {
|
if (run.isLaunch) {
|
||||||
|
|
@ -21209,7 +21402,7 @@ export class TeamProvisioningService {
|
||||||
metaMembers,
|
metaMembers,
|
||||||
})
|
})
|
||||||
: null;
|
: null;
|
||||||
const stableRecoveredMixedSnapshot =
|
const stableRecoveredMixedSnapshotWithCommittedEvidence =
|
||||||
overlaidRecoveredMixedSnapshot &&
|
overlaidRecoveredMixedSnapshot &&
|
||||||
this.hasCommittedOpenCodeSecondaryEvidenceOverlayDelta(
|
this.hasCommittedOpenCodeSecondaryEvidenceOverlayDelta(
|
||||||
overlaidRecoveredMixedSnapshot,
|
overlaidRecoveredMixedSnapshot,
|
||||||
|
|
@ -21217,6 +21410,14 @@ export class TeamProvisioningService {
|
||||||
)
|
)
|
||||||
? await this.writeLaunchStateSnapshot(teamName, overlaidRecoveredMixedSnapshot)
|
? await this.writeLaunchStateSnapshot(teamName, overlaidRecoveredMixedSnapshot)
|
||||||
: overlaidRecoveredMixedSnapshot;
|
: overlaidRecoveredMixedSnapshot;
|
||||||
|
const promotedRecoveredMixedSnapshot = promoteOpenCodePersistedFailureReasonsFromDiagnostics(
|
||||||
|
stableRecoveredMixedSnapshotWithCommittedEvidence
|
||||||
|
);
|
||||||
|
const stableRecoveredMixedSnapshot =
|
||||||
|
promotedRecoveredMixedSnapshot &&
|
||||||
|
promotedRecoveredMixedSnapshot !== stableRecoveredMixedSnapshotWithCommittedEvidence
|
||||||
|
? await this.writeLaunchStateSnapshot(teamName, promotedRecoveredMixedSnapshot)
|
||||||
|
: promotedRecoveredMixedSnapshot;
|
||||||
const filteredBootstrapSnapshot = bootstrapSnapshot
|
const filteredBootstrapSnapshot = bootstrapSnapshot
|
||||||
? this.filterRemovedMembersFromLaunchSnapshot(bootstrapSnapshot, metaMembers)
|
? this.filterRemovedMembersFromLaunchSnapshot(bootstrapSnapshot, metaMembers)
|
||||||
: null;
|
: null;
|
||||||
|
|
@ -21248,15 +21449,46 @@ export class TeamProvisioningService {
|
||||||
: null;
|
: null;
|
||||||
const shouldPersistCommittedEvidenceOverlay =
|
const shouldPersistCommittedEvidenceOverlay =
|
||||||
this.hasCommittedOpenCodeSecondaryEvidenceOverlayDelta(filteredPersisted, persisted);
|
this.hasCommittedOpenCodeSecondaryEvidenceOverlayDelta(filteredPersisted, persisted);
|
||||||
|
const promotedPersisted =
|
||||||
|
promoteOpenCodePersistedFailureReasonsFromDiagnostics(filteredPersisted);
|
||||||
|
const shouldPersistFailureReasonPromotion = promotedPersisted !== filteredPersisted;
|
||||||
const persistedWithCommittedEvidence =
|
const persistedWithCommittedEvidence =
|
||||||
filteredPersisted && shouldPersistCommittedEvidenceOverlay
|
promotedPersisted &&
|
||||||
? await this.writeLaunchStateSnapshot(teamName, filteredPersisted)
|
(shouldPersistCommittedEvidenceOverlay || shouldPersistFailureReasonPromotion)
|
||||||
: filteredPersisted;
|
? await this.writeLaunchStateSnapshot(teamName, promotedPersisted)
|
||||||
|
: promotedPersisted;
|
||||||
const preferredSnapshot = choosePreferredLaunchSnapshot(
|
const preferredSnapshot = choosePreferredLaunchSnapshot(
|
||||||
overlaidBootstrapSnapshot,
|
overlaidBootstrapSnapshot,
|
||||||
persistedWithCommittedEvidence
|
persistedWithCommittedEvidence
|
||||||
);
|
);
|
||||||
if (preferredSnapshot && preferredSnapshot === overlaidBootstrapSnapshot) {
|
const bootstrapSelectionWouldCollapseMixedLaunch =
|
||||||
|
preferredSnapshot &&
|
||||||
|
preferredSnapshot === overlaidBootstrapSnapshot &&
|
||||||
|
preferredSnapshot.teamLaunchState === 'clean_success' &&
|
||||||
|
!this.hasMixedLaunchMetadata(preferredSnapshot) &&
|
||||||
|
this.hasMixedLaunchMetadata(persistedWithCommittedEvidence);
|
||||||
|
if (
|
||||||
|
preferredSnapshot &&
|
||||||
|
preferredSnapshot === overlaidBootstrapSnapshot &&
|
||||||
|
!bootstrapSelectionWouldCollapseMixedLaunch
|
||||||
|
) {
|
||||||
|
if (persistedWithCommittedEvidence) {
|
||||||
|
if (
|
||||||
|
preferredSnapshot.teamLaunchState === 'clean_success' &&
|
||||||
|
!this.hasMixedLaunchMetadata(preferredSnapshot)
|
||||||
|
) {
|
||||||
|
await this.clearPersistedLaunchState(teamName);
|
||||||
|
return {
|
||||||
|
snapshot: preferredSnapshot,
|
||||||
|
statuses: snapshotToMemberSpawnStatuses(preferredSnapshot),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
const writtenSnapshot = await this.writeLaunchStateSnapshot(teamName, preferredSnapshot);
|
||||||
|
return {
|
||||||
|
snapshot: writtenSnapshot,
|
||||||
|
statuses: snapshotToMemberSpawnStatuses(writtenSnapshot),
|
||||||
|
};
|
||||||
|
}
|
||||||
return {
|
return {
|
||||||
snapshot: preferredSnapshot,
|
snapshot: preferredSnapshot,
|
||||||
statuses: snapshotToMemberSpawnStatuses(preferredSnapshot),
|
statuses: snapshotToMemberSpawnStatuses(preferredSnapshot),
|
||||||
|
|
|
||||||
|
|
@ -78,6 +78,11 @@ const REQUIRED_READY_CHECKPOINTS = new Set([
|
||||||
'member_ready',
|
'member_ready',
|
||||||
'run_ready',
|
'run_ready',
|
||||||
]);
|
]);
|
||||||
|
const GENERIC_OPEN_CODE_MEMBER_FAILURE_REASON = 'OpenCode bridge reported member launch failure';
|
||||||
|
const SECRET_FLAG_PATTERN =
|
||||||
|
/(--(?:api-key|token|password|secret|authorization|auth-token)(?:=|\s+))("[^"]*"|'[^']*'|\S+)/gi;
|
||||||
|
const BEARER_TOKEN_PATTERN = /\bBearer\s+[A-Za-z0-9._~+/=-]+/gi;
|
||||||
|
const SECRET_KEY_PATTERN = /\bsk-[A-Za-z0-9_-]{16,}\b/g;
|
||||||
|
|
||||||
export class OpenCodeTeamRuntimeAdapter implements TeamLaunchRuntimeAdapter {
|
export class OpenCodeTeamRuntimeAdapter implements TeamLaunchRuntimeAdapter {
|
||||||
readonly providerId = 'opencode' as const;
|
readonly providerId = 'opencode' as const;
|
||||||
|
|
@ -484,6 +489,23 @@ function mapOpenCodeLaunchDataToRuntimeResult(
|
||||||
: data.teamLaunchState === 'failed'
|
: data.teamLaunchState === 'failed'
|
||||||
? 'failed'
|
? 'failed'
|
||||||
: 'created';
|
: 'created';
|
||||||
|
const checkpointDiagnosticsForMember = [
|
||||||
|
...checkpointDiagnostic,
|
||||||
|
...(missingExpectedMembers.includes(member.name) ? incompleteReadyDiagnostic : []),
|
||||||
|
];
|
||||||
|
const memberDiagnostics = [
|
||||||
|
...(bridgeMember
|
||||||
|
? []
|
||||||
|
: [
|
||||||
|
`OpenCode bridge response did not include ${member.name}; keeping the member pending until lane state materializes.`,
|
||||||
|
]),
|
||||||
|
...(bridgeMember?.diagnostics ?? []),
|
||||||
|
...(bridgeMember?.evidence ?? []).map(
|
||||||
|
(evidence) => `${evidence.kind} at ${evidence.observedAt}`
|
||||||
|
),
|
||||||
|
...memberBridgeDiagnostics,
|
||||||
|
...checkpointDiagnosticsForMember,
|
||||||
|
];
|
||||||
return [
|
return [
|
||||||
member.name,
|
member.name,
|
||||||
mapBridgeMemberToRuntimeEvidence(
|
mapBridgeMemberToRuntimeEvidence(
|
||||||
|
|
@ -493,20 +515,13 @@ function mapOpenCodeLaunchDataToRuntimeResult(
|
||||||
bridgeMember?.runtimePid,
|
bridgeMember?.runtimePid,
|
||||||
bridgeMember?.pendingPermissionRequestIds,
|
bridgeMember?.pendingPermissionRequestIds,
|
||||||
bridgeMember != null,
|
bridgeMember != null,
|
||||||
[
|
memberDiagnostics,
|
||||||
...(bridgeMember
|
selectOpenCodeMemberFailureReason({
|
||||||
? []
|
memberDiagnostics: bridgeMember?.diagnostics ?? [],
|
||||||
: [
|
bridgeDiagnostics: data.diagnostics,
|
||||||
`OpenCode bridge response did not include ${member.name}; keeping the member pending until lane state materializes.`,
|
checkpointDiagnostics: checkpointDiagnosticsForMember,
|
||||||
]),
|
fallback: GENERIC_OPEN_CODE_MEMBER_FAILURE_REASON,
|
||||||
...(bridgeMember?.diagnostics ?? []),
|
})
|
||||||
...(bridgeMember?.evidence ?? []).map(
|
|
||||||
(evidence) => `${evidence.kind} at ${evidence.observedAt}`
|
|
||||||
),
|
|
||||||
...memberBridgeDiagnostics,
|
|
||||||
...checkpointDiagnostic,
|
|
||||||
...(missingExpectedMembers.includes(member.name) ? incompleteReadyDiagnostic : []),
|
|
||||||
]
|
|
||||||
),
|
),
|
||||||
];
|
];
|
||||||
})
|
})
|
||||||
|
|
@ -542,7 +557,8 @@ function mapBridgeMemberToRuntimeEvidence(
|
||||||
runtimePid: number | undefined,
|
runtimePid: number | undefined,
|
||||||
pendingPermissionRequestIds: string[] | undefined,
|
pendingPermissionRequestIds: string[] | undefined,
|
||||||
runtimeMaterialized: boolean,
|
runtimeMaterialized: boolean,
|
||||||
diagnostics: string[]
|
diagnostics: string[],
|
||||||
|
selectedHardFailureReason: string
|
||||||
): TeamRuntimeMemberLaunchEvidence {
|
): TeamRuntimeMemberLaunchEvidence {
|
||||||
const confirmed = launchState === 'confirmed_alive';
|
const confirmed = launchState === 'confirmed_alive';
|
||||||
const failed = launchState === 'failed';
|
const failed = launchState === 'failed';
|
||||||
|
|
@ -590,7 +606,7 @@ function mapBridgeMemberToRuntimeEvidence(
|
||||||
runtimeAlive: confirmed,
|
runtimeAlive: confirmed,
|
||||||
bootstrapConfirmed: confirmed,
|
bootstrapConfirmed: confirmed,
|
||||||
hardFailure: failed,
|
hardFailure: failed,
|
||||||
hardFailureReason: failed ? 'OpenCode bridge reported member launch failure' : undefined,
|
hardFailureReason: failed ? selectedHardFailureReason : undefined,
|
||||||
pendingPermissionRequestIds:
|
pendingPermissionRequestIds:
|
||||||
pendingPermissionRequestIds && pendingPermissionRequestIds.length > 0
|
pendingPermissionRequestIds && pendingPermissionRequestIds.length > 0
|
||||||
? [...new Set(pendingPermissionRequestIds)]
|
? [...new Set(pendingPermissionRequestIds)]
|
||||||
|
|
@ -605,6 +621,83 @@ function mapBridgeMemberToRuntimeEvidence(
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function selectOpenCodeMemberFailureReason(input: {
|
||||||
|
memberDiagnostics: readonly string[];
|
||||||
|
bridgeDiagnostics: readonly {
|
||||||
|
code: string;
|
||||||
|
severity: 'info' | 'warning' | 'error';
|
||||||
|
message: string;
|
||||||
|
}[];
|
||||||
|
checkpointDiagnostics: readonly string[];
|
||||||
|
fallback: string;
|
||||||
|
}): string {
|
||||||
|
return (
|
||||||
|
firstDisplayableOpenCodeFailureMessage(input.memberDiagnostics, { includeGeneric: false }) ??
|
||||||
|
firstDisplayableOpenCodeFailureMessage(
|
||||||
|
input.bridgeDiagnostics
|
||||||
|
.filter((diagnostic) => diagnostic.severity === 'error')
|
||||||
|
.map((diagnostic) => diagnostic.message),
|
||||||
|
{ includeGeneric: false }
|
||||||
|
) ??
|
||||||
|
firstDisplayableOpenCodeFailureMessage(input.memberDiagnostics, { includeGeneric: true }) ??
|
||||||
|
firstDisplayableOpenCodeFailureMessage(input.checkpointDiagnostics, { includeGeneric: true }) ??
|
||||||
|
firstDisplayableOpenCodeFailureMessage(
|
||||||
|
input.bridgeDiagnostics
|
||||||
|
.filter((diagnostic) => diagnostic.severity !== 'info')
|
||||||
|
.map((diagnostic) => diagnostic.message),
|
||||||
|
{ includeGeneric: true }
|
||||||
|
) ??
|
||||||
|
normalizeOpenCodeFailureMessage(input.fallback) ??
|
||||||
|
GENERIC_OPEN_CODE_MEMBER_FAILURE_REASON
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function firstDisplayableOpenCodeFailureMessage(
|
||||||
|
values: readonly string[],
|
||||||
|
options: { includeGeneric: boolean }
|
||||||
|
): string | undefined {
|
||||||
|
for (const value of values) {
|
||||||
|
const normalized = normalizeOpenCodeFailureMessage(value);
|
||||||
|
if (!normalized) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!options.includeGeneric && isGenericOpenCodeFailureMessage(normalized)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
return normalized;
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeOpenCodeFailureMessage(value: string | undefined): string | undefined {
|
||||||
|
const trimmed = value?.replace(/\s+/g, ' ').trim();
|
||||||
|
if (!trimmed) {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
return trimmed
|
||||||
|
.replace(SECRET_FLAG_PATTERN, '$1[redacted]')
|
||||||
|
.replace(BEARER_TOKEN_PATTERN, 'Bearer [redacted]')
|
||||||
|
.replace(SECRET_KEY_PATTERN, '[redacted-api-key]');
|
||||||
|
}
|
||||||
|
|
||||||
|
function isGenericOpenCodeFailureMessage(message: string): boolean {
|
||||||
|
return (
|
||||||
|
message === GENERIC_OPEN_CODE_MEMBER_FAILURE_REASON ||
|
||||||
|
message.startsWith(`${GENERIC_OPEN_CODE_MEMBER_FAILURE_REASON}:`) ||
|
||||||
|
message.startsWith('OpenCode secondary lane timing:') ||
|
||||||
|
message.startsWith(
|
||||||
|
'OpenCode bridge reported ready without all required durable checkpoints:'
|
||||||
|
) ||
|
||||||
|
message.startsWith(
|
||||||
|
'OpenCode bridge reported ready before all expected members were confirmed:'
|
||||||
|
) ||
|
||||||
|
message.startsWith(
|
||||||
|
'OpenCode bootstrap MCP did not complete required tools before assistant response:'
|
||||||
|
) ||
|
||||||
|
isOpenCodeLaunchTimingDiagnostic(message)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
function extractCheckpointNames(data: OpenCodeLaunchTeamCommandData): Set<string> {
|
function extractCheckpointNames(data: OpenCodeLaunchTeamCommandData): Set<string> {
|
||||||
const names = new Set<string>();
|
const names = new Set<string>();
|
||||||
for (const checkpoint of data.durableCheckpoints ?? []) {
|
for (const checkpoint of data.durableCheckpoints ?? []) {
|
||||||
|
|
|
||||||
|
|
@ -156,6 +156,106 @@ describe('OpenCodeTeamRuntimeAdapter', () => {
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('uses concrete member diagnostics as failed OpenCode hard failure reasons', async () => {
|
||||||
|
const concreteReason =
|
||||||
|
'Latest assistant message msg_123 failed with APIError - Insufficient credits. Add more using https://openrouter.ai/settings/credits';
|
||||||
|
const launchOpenCodeTeam = vi.fn<
|
||||||
|
NonNullable<OpenCodeTeamRuntimeBridgePort['launchOpenCodeTeam']>
|
||||||
|
>(
|
||||||
|
async () =>
|
||||||
|
({
|
||||||
|
runId: 'run-1',
|
||||||
|
teamLaunchState: 'failed',
|
||||||
|
members: {
|
||||||
|
alice: {
|
||||||
|
sessionId: 'oc-session-1',
|
||||||
|
launchState: 'failed',
|
||||||
|
model: 'openai/gpt-5.4-mini',
|
||||||
|
diagnostics: ['OpenCode bridge reported member launch failure', concreteReason],
|
||||||
|
evidence: [],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
warnings: [],
|
||||||
|
diagnostics: [],
|
||||||
|
}) satisfies OpenCodeLaunchTeamCommandData
|
||||||
|
);
|
||||||
|
const adapter = new OpenCodeTeamRuntimeAdapter(
|
||||||
|
bridgePort(readiness({ state: 'ready', launchAllowed: true }), { launchOpenCodeTeam })
|
||||||
|
);
|
||||||
|
|
||||||
|
const result = await adapter.launch(launchInput());
|
||||||
|
|
||||||
|
expect(result.members.alice).toMatchObject({
|
||||||
|
launchState: 'failed_to_start',
|
||||||
|
hardFailureReason: concreteReason,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('falls back to bridge error diagnostics when member failure details are generic', async () => {
|
||||||
|
const bridgeError = 'Provider runtime returned a concrete launch error';
|
||||||
|
const launchOpenCodeTeam = vi.fn<
|
||||||
|
NonNullable<OpenCodeTeamRuntimeBridgePort['launchOpenCodeTeam']>
|
||||||
|
>(
|
||||||
|
async () =>
|
||||||
|
({
|
||||||
|
runId: 'run-1',
|
||||||
|
teamLaunchState: 'failed',
|
||||||
|
members: {
|
||||||
|
alice: {
|
||||||
|
sessionId: 'oc-session-1',
|
||||||
|
launchState: 'failed',
|
||||||
|
model: 'openai/gpt-5.4-mini',
|
||||||
|
diagnostics: ['OpenCode bridge reported member launch failure'],
|
||||||
|
evidence: [],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
warnings: [],
|
||||||
|
diagnostics: [{ code: 'provider_error', severity: 'error', message: bridgeError }],
|
||||||
|
}) satisfies OpenCodeLaunchTeamCommandData
|
||||||
|
);
|
||||||
|
const adapter = new OpenCodeTeamRuntimeAdapter(
|
||||||
|
bridgePort(readiness({ state: 'ready', launchAllowed: true }), { launchOpenCodeTeam })
|
||||||
|
);
|
||||||
|
|
||||||
|
const result = await adapter.launch(launchInput());
|
||||||
|
|
||||||
|
expect(result.members.alice?.hardFailureReason).toBe(bridgeError);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('redacts secret-like values in selected OpenCode failure reasons', async () => {
|
||||||
|
const launchOpenCodeTeam = vi.fn<
|
||||||
|
NonNullable<OpenCodeTeamRuntimeBridgePort['launchOpenCodeTeam']>
|
||||||
|
>(
|
||||||
|
async () =>
|
||||||
|
({
|
||||||
|
runId: 'run-1',
|
||||||
|
teamLaunchState: 'failed',
|
||||||
|
members: {
|
||||||
|
alice: {
|
||||||
|
sessionId: 'oc-session-1',
|
||||||
|
launchState: 'failed',
|
||||||
|
model: 'openai/gpt-5.4-mini',
|
||||||
|
diagnostics: [
|
||||||
|
'Provider failed with --api-key sk-openroutersecret000000000000 and Bearer abc.def.ghi',
|
||||||
|
],
|
||||||
|
evidence: [],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
warnings: [],
|
||||||
|
diagnostics: [],
|
||||||
|
}) satisfies OpenCodeLaunchTeamCommandData
|
||||||
|
);
|
||||||
|
const adapter = new OpenCodeTeamRuntimeAdapter(
|
||||||
|
bridgePort(readiness({ state: 'ready', launchAllowed: true }), { launchOpenCodeTeam })
|
||||||
|
);
|
||||||
|
|
||||||
|
const result = await adapter.launch(launchInput());
|
||||||
|
|
||||||
|
expect(result.members.alice?.hardFailureReason).toBe(
|
||||||
|
'Provider failed with --api-key [redacted] and Bearer [redacted]'
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
it('rejects non-OpenCode members before readiness or launch bridge dispatch', async () => {
|
it('rejects non-OpenCode members before readiness or launch bridge dispatch', async () => {
|
||||||
const launchOpenCodeTeam = vi.fn();
|
const launchOpenCodeTeam = vi.fn();
|
||||||
const bridge = bridgePort(readiness({ state: 'ready', launchAllowed: true }), {
|
const bridge = bridgePort(readiness({ state: 'ready', launchAllowed: true }), {
|
||||||
|
|
|
||||||
|
|
@ -133,7 +133,10 @@ import {
|
||||||
createPersistedLaunchSnapshot,
|
createPersistedLaunchSnapshot,
|
||||||
snapshotFromRuntimeMemberStatuses,
|
snapshotFromRuntimeMemberStatuses,
|
||||||
} from '@main/services/team/TeamLaunchStateEvaluator';
|
} from '@main/services/team/TeamLaunchStateEvaluator';
|
||||||
import { getTeamLaunchStatePath } from '@main/services/team/TeamLaunchStateStore';
|
import {
|
||||||
|
getTeamLaunchStatePath,
|
||||||
|
getTeamLaunchSummaryPath,
|
||||||
|
} from '@main/services/team/TeamLaunchStateStore';
|
||||||
import { TeamConfigReader } from '@main/services/team/TeamConfigReader';
|
import { TeamConfigReader } from '@main/services/team/TeamConfigReader';
|
||||||
import {
|
import {
|
||||||
getOpenCodeLaneScopedRuntimeFilePath,
|
getOpenCodeLaneScopedRuntimeFilePath,
|
||||||
|
|
@ -14200,6 +14203,91 @@ describe('TeamProvisioningService', () => {
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('clears stale launch-state and launch-summary when bootstrap truth supersedes a persisted failure', async () => {
|
||||||
|
const teamName = 'bootstrap-supersedes-stale-launch-summary-team';
|
||||||
|
const leadSessionId = 'lead-session';
|
||||||
|
const staleUpdatedAt = '2026-04-16T09:55:00.000Z';
|
||||||
|
const freshObservedAt = '2026-04-16T10:00:00.000Z';
|
||||||
|
writeLaunchConfig(teamName, '/Users/test/proj', leadSessionId, ['bob']);
|
||||||
|
writeLaunchState(
|
||||||
|
teamName,
|
||||||
|
leadSessionId,
|
||||||
|
{
|
||||||
|
bob: {
|
||||||
|
launchState: 'failed_to_start',
|
||||||
|
agentToolAccepted: false,
|
||||||
|
runtimeAlive: false,
|
||||||
|
bootstrapConfirmed: false,
|
||||||
|
hardFailure: true,
|
||||||
|
hardFailureReason: 'Teammate was never spawned during launch.',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{ updatedAt: staleUpdatedAt }
|
||||||
|
);
|
||||||
|
fs.writeFileSync(
|
||||||
|
getTeamLaunchSummaryPath(teamName),
|
||||||
|
`${JSON.stringify(
|
||||||
|
{
|
||||||
|
version: 1,
|
||||||
|
teamName,
|
||||||
|
updatedAt: staleUpdatedAt,
|
||||||
|
launchPhase: 'finished',
|
||||||
|
partialLaunchFailure: true,
|
||||||
|
expectedMemberCount: 1,
|
||||||
|
confirmedMemberCount: 0,
|
||||||
|
missingMembers: ['bob'],
|
||||||
|
teamLaunchState: 'partial_failure',
|
||||||
|
launchUpdatedAt: staleUpdatedAt,
|
||||||
|
confirmedCount: 0,
|
||||||
|
pendingCount: 0,
|
||||||
|
failedCount: 1,
|
||||||
|
skippedCount: 0,
|
||||||
|
runtimeAlivePendingCount: 0,
|
||||||
|
shellOnlyPendingCount: 0,
|
||||||
|
runtimeProcessPendingCount: 0,
|
||||||
|
runtimeCandidatePendingCount: 0,
|
||||||
|
noRuntimePendingCount: 0,
|
||||||
|
permissionPendingCount: 0,
|
||||||
|
},
|
||||||
|
null,
|
||||||
|
2
|
||||||
|
)}\n`,
|
||||||
|
'utf8'
|
||||||
|
);
|
||||||
|
writeBootstrapState(
|
||||||
|
teamName,
|
||||||
|
[
|
||||||
|
{
|
||||||
|
name: 'bob',
|
||||||
|
status: 'bootstrap_confirmed',
|
||||||
|
lastAttemptAt: Date.parse(freshObservedAt),
|
||||||
|
lastObservedAt: Date.parse(freshObservedAt),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
freshObservedAt
|
||||||
|
);
|
||||||
|
|
||||||
|
const svc = new TeamProvisioningService();
|
||||||
|
const result = await svc.getMemberSpawnStatuses(teamName);
|
||||||
|
|
||||||
|
expect(result.statuses.bob).toMatchObject({
|
||||||
|
status: 'online',
|
||||||
|
launchState: 'confirmed_alive',
|
||||||
|
bootstrapConfirmed: true,
|
||||||
|
hardFailure: false,
|
||||||
|
});
|
||||||
|
await expect(fsPromises.readFile(getTeamLaunchStatePath(teamName), 'utf8')).rejects.toMatchObject(
|
||||||
|
{
|
||||||
|
code: 'ENOENT',
|
||||||
|
}
|
||||||
|
);
|
||||||
|
await expect(
|
||||||
|
fsPromises.readFile(getTeamLaunchSummaryPath(teamName), 'utf8')
|
||||||
|
).rejects.toMatchObject({
|
||||||
|
code: 'ENOENT',
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
it('reconciles extra persisted launch members when bootstrap state proves they were registered', async () => {
|
it('reconciles extra persisted launch members when bootstrap state proves they were registered', async () => {
|
||||||
const teamName = 'registered-bootstrap-extra-member-team';
|
const teamName = 'registered-bootstrap-extra-member-team';
|
||||||
const teamDir = path.join(tempTeamsBase, teamName);
|
const teamDir = path.join(tempTeamsBase, teamName);
|
||||||
|
|
@ -15455,6 +15543,476 @@ describe('TeamProvisioningService', () => {
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('keeps primary bootstrap-confirmed members alive when OpenCode secondary lanes fail', async () => {
|
||||||
|
const teamName = 'atlas-hq-source-aware-live';
|
||||||
|
const startedAt = '2026-04-23T10:00:00.000Z';
|
||||||
|
const exactOpenCodeReason =
|
||||||
|
'Latest assistant message msg_alice failed with APIError - Insufficient credits.';
|
||||||
|
writeLaunchConfig(teamName, '/Users/test/proj', 'lead-session', ['bob', 'jack']);
|
||||||
|
writeBootstrapState(
|
||||||
|
teamName,
|
||||||
|
[
|
||||||
|
{
|
||||||
|
name: 'bob',
|
||||||
|
status: 'bootstrap_confirmed',
|
||||||
|
lastObservedAt: Date.parse('2026-04-23T10:01:00.000Z'),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'jack',
|
||||||
|
status: 'bootstrap_confirmed',
|
||||||
|
lastObservedAt: Date.parse('2026-04-23T10:01:00.000Z'),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
'2026-04-23T10:01:00.000Z'
|
||||||
|
);
|
||||||
|
const run = createMemberSpawnRun({
|
||||||
|
teamName,
|
||||||
|
runId: 'run-atlas-hq-source-aware-live',
|
||||||
|
startedAt,
|
||||||
|
expectedMembers: ['bob', 'jack'],
|
||||||
|
memberSpawnStatuses: new Map([
|
||||||
|
[
|
||||||
|
'bob',
|
||||||
|
createMemberSpawnStatusEntry({
|
||||||
|
status: 'error',
|
||||||
|
launchState: 'failed_to_start',
|
||||||
|
agentToolAccepted: false,
|
||||||
|
runtimeAlive: false,
|
||||||
|
bootstrapConfirmed: false,
|
||||||
|
hardFailure: true,
|
||||||
|
error: 'Teammate was never spawned during launch.',
|
||||||
|
hardFailureReason: 'Teammate was never spawned during launch.',
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
[
|
||||||
|
'jack',
|
||||||
|
createMemberSpawnStatusEntry({
|
||||||
|
status: 'error',
|
||||||
|
launchState: 'failed_to_start',
|
||||||
|
agentToolAccepted: false,
|
||||||
|
runtimeAlive: false,
|
||||||
|
bootstrapConfirmed: false,
|
||||||
|
hardFailure: true,
|
||||||
|
error: 'Teammate was never spawned during launch.',
|
||||||
|
hardFailureReason: 'Teammate was never spawned during launch.',
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
]),
|
||||||
|
});
|
||||||
|
run.isLaunch = true;
|
||||||
|
run.request = {
|
||||||
|
teamName,
|
||||||
|
cwd: '/Users/test/proj',
|
||||||
|
providerId: 'codex',
|
||||||
|
providerBackendId: 'codex-native',
|
||||||
|
model: 'gpt-5.4',
|
||||||
|
members: [],
|
||||||
|
};
|
||||||
|
run.effectiveMembers = [
|
||||||
|
{ name: 'bob', providerId: 'codex', model: 'gpt-5.3-codex' },
|
||||||
|
{ name: 'jack', providerId: 'codex', model: 'gpt-5.4' },
|
||||||
|
];
|
||||||
|
run.mixedSecondaryLanes = [
|
||||||
|
{
|
||||||
|
laneId: 'secondary:opencode:alice',
|
||||||
|
providerId: 'opencode',
|
||||||
|
member: {
|
||||||
|
name: 'alice',
|
||||||
|
providerId: 'opencode',
|
||||||
|
model: 'openrouter/z-ai/glm-5.1',
|
||||||
|
},
|
||||||
|
runId: 'lane-run-alice',
|
||||||
|
state: 'finished',
|
||||||
|
result: {
|
||||||
|
runId: 'lane-run-alice',
|
||||||
|
teamName,
|
||||||
|
launchPhase: 'finished',
|
||||||
|
teamLaunchState: 'partial_failure',
|
||||||
|
members: {
|
||||||
|
alice: {
|
||||||
|
memberName: 'alice',
|
||||||
|
providerId: 'opencode',
|
||||||
|
launchState: 'failed_to_start',
|
||||||
|
agentToolAccepted: true,
|
||||||
|
runtimeAlive: false,
|
||||||
|
bootstrapConfirmed: false,
|
||||||
|
hardFailure: true,
|
||||||
|
hardFailureReason: exactOpenCodeReason,
|
||||||
|
diagnostics: [exactOpenCodeReason],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
warnings: [],
|
||||||
|
diagnostics: [exactOpenCodeReason],
|
||||||
|
},
|
||||||
|
warnings: [],
|
||||||
|
diagnostics: [exactOpenCodeReason],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
laneId: 'secondary:opencode:tom',
|
||||||
|
providerId: 'opencode',
|
||||||
|
member: {
|
||||||
|
name: 'tom',
|
||||||
|
providerId: 'opencode',
|
||||||
|
model: 'openrouter/minimax/minimax-m2.5',
|
||||||
|
},
|
||||||
|
runId: 'lane-run-tom',
|
||||||
|
state: 'finished',
|
||||||
|
result: {
|
||||||
|
runId: 'lane-run-tom',
|
||||||
|
teamName,
|
||||||
|
launchPhase: 'finished',
|
||||||
|
teamLaunchState: 'partial_failure',
|
||||||
|
members: {
|
||||||
|
tom: {
|
||||||
|
memberName: 'tom',
|
||||||
|
providerId: 'opencode',
|
||||||
|
launchState: 'failed_to_start',
|
||||||
|
agentToolAccepted: true,
|
||||||
|
runtimeAlive: false,
|
||||||
|
bootstrapConfirmed: false,
|
||||||
|
hardFailure: true,
|
||||||
|
hardFailureReason: 'Tom provider launch failed.',
|
||||||
|
diagnostics: ['Tom provider launch failed.'],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
warnings: [],
|
||||||
|
diagnostics: ['Tom provider launch failed.'],
|
||||||
|
},
|
||||||
|
warnings: [],
|
||||||
|
diagnostics: ['Tom provider launch failed.'],
|
||||||
|
},
|
||||||
|
];
|
||||||
|
run.detectedSessionId = 'lead-session';
|
||||||
|
|
||||||
|
const svc = new TeamProvisioningService();
|
||||||
|
const snapshot = await (svc as any).persistLaunchStateSnapshot(run, 'finished');
|
||||||
|
|
||||||
|
expect(snapshot.members.bob).toMatchObject({
|
||||||
|
launchState: 'confirmed_alive',
|
||||||
|
bootstrapConfirmed: true,
|
||||||
|
hardFailure: false,
|
||||||
|
});
|
||||||
|
expect(snapshot.members.jack).toMatchObject({
|
||||||
|
launchState: 'confirmed_alive',
|
||||||
|
bootstrapConfirmed: true,
|
||||||
|
hardFailure: false,
|
||||||
|
});
|
||||||
|
expect(snapshot.members.alice).toMatchObject({
|
||||||
|
launchState: 'failed_to_start',
|
||||||
|
hardFailureReason: exactOpenCodeReason,
|
||||||
|
});
|
||||||
|
expect(snapshot.members.tom).toMatchObject({
|
||||||
|
launchState: 'failed_to_start',
|
||||||
|
hardFailureReason: 'Tom provider launch failed.',
|
||||||
|
});
|
||||||
|
expect(snapshot.summary.confirmedCount).toBe(2);
|
||||||
|
expect(snapshot.summary.failedCount).toBe(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('reconciles persisted mixed launch-state when primary bootstrap members were marked missing', async () => {
|
||||||
|
const teamName = 'atlas-hq-source-aware-persisted';
|
||||||
|
const exactOpenCodeReason =
|
||||||
|
'Latest assistant message msg_alice failed with APIError - Insufficient credits.';
|
||||||
|
writeTeamMeta(teamName, {
|
||||||
|
providerId: 'codex',
|
||||||
|
providerBackendId: 'codex-native',
|
||||||
|
model: 'gpt-5.4',
|
||||||
|
});
|
||||||
|
writeMembersMeta(teamName, [
|
||||||
|
{ name: 'bob', providerId: 'codex', model: 'gpt-5.3-codex' },
|
||||||
|
{ name: 'jack', providerId: 'codex', model: 'gpt-5.4' },
|
||||||
|
{ name: 'alice', providerId: 'opencode', model: 'openrouter/z-ai/glm-5.1' },
|
||||||
|
{ name: 'tom', providerId: 'opencode', model: 'openrouter/minimax/minimax-m2.5' },
|
||||||
|
]);
|
||||||
|
writeLaunchConfig(teamName, '/Users/test/proj', 'lead-session', ['bob', 'jack']);
|
||||||
|
writeBootstrapState(
|
||||||
|
teamName,
|
||||||
|
[
|
||||||
|
{
|
||||||
|
name: 'bob',
|
||||||
|
status: 'bootstrap_confirmed',
|
||||||
|
lastObservedAt: Date.parse('2026-04-23T10:01:00.000Z'),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'jack',
|
||||||
|
status: 'bootstrap_confirmed',
|
||||||
|
lastObservedAt: Date.parse('2026-04-23T10:01:00.000Z'),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
'2026-04-23T10:01:00.000Z'
|
||||||
|
);
|
||||||
|
fs.writeFileSync(
|
||||||
|
getTeamLaunchStatePath(teamName),
|
||||||
|
`${JSON.stringify(
|
||||||
|
createPersistedLaunchSnapshot({
|
||||||
|
teamName,
|
||||||
|
leadSessionId: 'lead-session',
|
||||||
|
launchPhase: 'finished',
|
||||||
|
expectedMembers: ['bob', 'jack', 'alice', 'tom'],
|
||||||
|
bootstrapExpectedMembers: ['bob', 'jack'],
|
||||||
|
members: {
|
||||||
|
bob: {
|
||||||
|
name: 'bob',
|
||||||
|
providerId: 'codex',
|
||||||
|
laneId: 'primary',
|
||||||
|
laneKind: 'primary',
|
||||||
|
laneOwnerProviderId: 'codex',
|
||||||
|
launchState: 'failed_to_start',
|
||||||
|
agentToolAccepted: false,
|
||||||
|
runtimeAlive: false,
|
||||||
|
bootstrapConfirmed: false,
|
||||||
|
hardFailure: true,
|
||||||
|
hardFailureReason: 'Teammate was never spawned during launch.',
|
||||||
|
lastEvaluatedAt: '2026-04-23T10:02:00.000Z',
|
||||||
|
},
|
||||||
|
jack: {
|
||||||
|
name: 'jack',
|
||||||
|
providerId: 'codex',
|
||||||
|
laneId: 'primary',
|
||||||
|
laneKind: 'primary',
|
||||||
|
laneOwnerProviderId: 'codex',
|
||||||
|
launchState: 'failed_to_start',
|
||||||
|
agentToolAccepted: false,
|
||||||
|
runtimeAlive: false,
|
||||||
|
bootstrapConfirmed: false,
|
||||||
|
hardFailure: true,
|
||||||
|
hardFailureReason: 'Teammate was never spawned during launch.',
|
||||||
|
lastEvaluatedAt: '2026-04-23T10:02:00.000Z',
|
||||||
|
},
|
||||||
|
alice: {
|
||||||
|
name: 'alice',
|
||||||
|
providerId: 'opencode',
|
||||||
|
model: 'openrouter/z-ai/glm-5.1',
|
||||||
|
laneId: 'secondary:opencode:alice',
|
||||||
|
laneKind: 'secondary',
|
||||||
|
laneOwnerProviderId: 'opencode',
|
||||||
|
launchState: 'failed_to_start',
|
||||||
|
agentToolAccepted: true,
|
||||||
|
runtimeAlive: false,
|
||||||
|
bootstrapConfirmed: false,
|
||||||
|
hardFailure: true,
|
||||||
|
hardFailureReason: 'OpenCode bridge reported member launch failure',
|
||||||
|
diagnostics: [exactOpenCodeReason],
|
||||||
|
lastEvaluatedAt: '2026-04-23T10:02:00.000Z',
|
||||||
|
},
|
||||||
|
tom: {
|
||||||
|
name: 'tom',
|
||||||
|
providerId: 'opencode',
|
||||||
|
model: 'openrouter/minimax/minimax-m2.5',
|
||||||
|
laneId: 'secondary:opencode:tom',
|
||||||
|
laneKind: 'secondary',
|
||||||
|
laneOwnerProviderId: 'opencode',
|
||||||
|
launchState: 'failed_to_start',
|
||||||
|
agentToolAccepted: true,
|
||||||
|
runtimeAlive: false,
|
||||||
|
bootstrapConfirmed: false,
|
||||||
|
hardFailure: true,
|
||||||
|
hardFailureReason: 'OpenCode bridge reported member launch failure',
|
||||||
|
diagnostics: ['Tom provider launch failed.'],
|
||||||
|
lastEvaluatedAt: '2026-04-23T10:02:00.000Z',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
updatedAt: '2026-04-23T10:02:00.000Z',
|
||||||
|
}),
|
||||||
|
null,
|
||||||
|
2
|
||||||
|
)}\n`,
|
||||||
|
'utf8'
|
||||||
|
);
|
||||||
|
|
||||||
|
const svc = new TeamProvisioningService();
|
||||||
|
const result = await svc.getMemberSpawnStatuses(teamName);
|
||||||
|
|
||||||
|
expect(result.statuses.bob).toMatchObject({
|
||||||
|
status: 'online',
|
||||||
|
launchState: 'confirmed_alive',
|
||||||
|
bootstrapConfirmed: true,
|
||||||
|
hardFailure: false,
|
||||||
|
});
|
||||||
|
expect(result.statuses.jack).toMatchObject({
|
||||||
|
status: 'online',
|
||||||
|
launchState: 'confirmed_alive',
|
||||||
|
bootstrapConfirmed: true,
|
||||||
|
hardFailure: false,
|
||||||
|
});
|
||||||
|
expect(result.statuses.alice).toMatchObject({
|
||||||
|
status: 'error',
|
||||||
|
launchState: 'failed_to_start',
|
||||||
|
hardFailureReason: exactOpenCodeReason,
|
||||||
|
});
|
||||||
|
expect(result.statuses.tom).toMatchObject({
|
||||||
|
status: 'error',
|
||||||
|
launchState: 'failed_to_start',
|
||||||
|
hardFailureReason: 'Tom provider launch failed.',
|
||||||
|
});
|
||||||
|
const summary = JSON.parse(await fsPromises.readFile(getTeamLaunchSummaryPath(teamName), 'utf8'));
|
||||||
|
expect(summary).toMatchObject({
|
||||||
|
teamLaunchState: 'partial_failure',
|
||||||
|
confirmedCount: 2,
|
||||||
|
failedCount: 2,
|
||||||
|
missingMembers: ['alice', 'tom'],
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does not collapse persisted mixed secondary failures when primary bootstrap snapshot is clean and richer', async () => {
|
||||||
|
const teamName = 'mixed-clean-bootstrap-does-not-collapse-secondary-failure';
|
||||||
|
writeMembersMeta(teamName, [
|
||||||
|
{ name: 'bob', providerId: 'codex', model: 'gpt-5.4' },
|
||||||
|
{ name: 'alice', providerId: 'opencode', model: 'openrouter/z-ai/glm-5.1' },
|
||||||
|
]);
|
||||||
|
writeLaunchConfig(teamName, '/Users/test/proj', 'lead-session', ['bob']);
|
||||||
|
writeBootstrapState(
|
||||||
|
teamName,
|
||||||
|
[
|
||||||
|
{ name: 'bob', status: 'bootstrap_confirmed' },
|
||||||
|
{ name: 'jack', status: 'bootstrap_confirmed' },
|
||||||
|
{ name: 'nova', status: 'bootstrap_confirmed' },
|
||||||
|
{ name: 'sam', status: 'bootstrap_confirmed' },
|
||||||
|
{ name: 'kim', status: 'bootstrap_confirmed' },
|
||||||
|
],
|
||||||
|
'2026-04-23T10:03:00.000Z'
|
||||||
|
);
|
||||||
|
const exactOpenCodeReason =
|
||||||
|
'Latest assistant message msg_alice failed with APIError - Insufficient credits.';
|
||||||
|
fs.writeFileSync(
|
||||||
|
getTeamLaunchStatePath(teamName),
|
||||||
|
`${JSON.stringify(
|
||||||
|
createPersistedLaunchSnapshot({
|
||||||
|
teamName,
|
||||||
|
leadSessionId: 'lead-session',
|
||||||
|
launchPhase: 'finished',
|
||||||
|
expectedMembers: ['bob', 'alice'],
|
||||||
|
bootstrapExpectedMembers: ['bob'],
|
||||||
|
members: {
|
||||||
|
bob: {
|
||||||
|
name: 'bob',
|
||||||
|
providerId: 'codex',
|
||||||
|
laneId: 'primary',
|
||||||
|
laneKind: 'primary',
|
||||||
|
laneOwnerProviderId: 'codex',
|
||||||
|
launchState: 'failed_to_start',
|
||||||
|
agentToolAccepted: false,
|
||||||
|
runtimeAlive: false,
|
||||||
|
bootstrapConfirmed: false,
|
||||||
|
hardFailure: true,
|
||||||
|
hardFailureReason: 'Teammate was never spawned during launch.',
|
||||||
|
lastEvaluatedAt: '2026-04-23T10:02:00.000Z',
|
||||||
|
},
|
||||||
|
alice: {
|
||||||
|
name: 'alice',
|
||||||
|
providerId: 'opencode',
|
||||||
|
laneId: 'secondary:opencode:alice',
|
||||||
|
laneKind: 'secondary',
|
||||||
|
laneOwnerProviderId: 'opencode',
|
||||||
|
launchState: 'failed_to_start',
|
||||||
|
agentToolAccepted: true,
|
||||||
|
runtimeAlive: false,
|
||||||
|
bootstrapConfirmed: false,
|
||||||
|
hardFailure: true,
|
||||||
|
hardFailureReason: 'OpenCode bridge reported member launch failure',
|
||||||
|
diagnostics: [exactOpenCodeReason],
|
||||||
|
lastEvaluatedAt: '2026-04-23T10:02:00.000Z',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
updatedAt: '2026-04-23T10:02:00.000Z',
|
||||||
|
}),
|
||||||
|
null,
|
||||||
|
2
|
||||||
|
)}\n`,
|
||||||
|
'utf8'
|
||||||
|
);
|
||||||
|
|
||||||
|
const svc = new TeamProvisioningService();
|
||||||
|
const result = await svc.getMemberSpawnStatuses(teamName);
|
||||||
|
|
||||||
|
expect(result.teamLaunchState).toBe('partial_failure');
|
||||||
|
expect(result.statuses.bob).toMatchObject({
|
||||||
|
status: 'online',
|
||||||
|
launchState: 'confirmed_alive',
|
||||||
|
bootstrapConfirmed: true,
|
||||||
|
hardFailure: false,
|
||||||
|
});
|
||||||
|
expect(result.statuses.alice).toMatchObject({
|
||||||
|
status: 'error',
|
||||||
|
launchState: 'failed_to_start',
|
||||||
|
hardFailureReason: exactOpenCodeReason,
|
||||||
|
});
|
||||||
|
const persisted = JSON.parse(await fsPromises.readFile(getTeamLaunchStatePath(teamName), 'utf8'));
|
||||||
|
expect(persisted.members.alice).toMatchObject({
|
||||||
|
laneId: 'secondary:opencode:alice',
|
||||||
|
launchState: 'failed_to_start',
|
||||||
|
hardFailureReason: exactOpenCodeReason,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does not revive primary members from stale bootstrap-state during mixed projection', async () => {
|
||||||
|
const teamName = 'atlas-hq-stale-bootstrap-live';
|
||||||
|
writeLaunchConfig(teamName, '/Users/test/proj', 'lead-session', ['bob']);
|
||||||
|
writeBootstrapState(
|
||||||
|
teamName,
|
||||||
|
[
|
||||||
|
{
|
||||||
|
name: 'bob',
|
||||||
|
status: 'bootstrap_confirmed',
|
||||||
|
lastObservedAt: Date.parse('2026-04-23T09:59:00.000Z'),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
'2026-04-23T09:59:00.000Z'
|
||||||
|
);
|
||||||
|
const run = createMemberSpawnRun({
|
||||||
|
teamName,
|
||||||
|
runId: 'run-atlas-hq-stale-bootstrap-live',
|
||||||
|
startedAt: '2026-04-23T10:00:00.000Z',
|
||||||
|
expectedMembers: ['bob'],
|
||||||
|
memberSpawnStatuses: new Map([
|
||||||
|
[
|
||||||
|
'bob',
|
||||||
|
createMemberSpawnStatusEntry({
|
||||||
|
status: 'error',
|
||||||
|
launchState: 'failed_to_start',
|
||||||
|
agentToolAccepted: false,
|
||||||
|
runtimeAlive: false,
|
||||||
|
bootstrapConfirmed: false,
|
||||||
|
hardFailure: true,
|
||||||
|
error: 'Teammate was never spawned during launch.',
|
||||||
|
hardFailureReason: 'Teammate was never spawned during launch.',
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
]),
|
||||||
|
});
|
||||||
|
run.isLaunch = true;
|
||||||
|
run.request = {
|
||||||
|
teamName,
|
||||||
|
cwd: '/Users/test/proj',
|
||||||
|
providerId: 'codex',
|
||||||
|
providerBackendId: 'codex-native',
|
||||||
|
model: 'gpt-5.4',
|
||||||
|
members: [],
|
||||||
|
};
|
||||||
|
run.effectiveMembers = [{ name: 'bob', providerId: 'codex', model: 'gpt-5.3-codex' }];
|
||||||
|
run.mixedSecondaryLanes = [
|
||||||
|
{
|
||||||
|
laneId: 'secondary:opencode:alice',
|
||||||
|
providerId: 'opencode',
|
||||||
|
member: { name: 'alice', providerId: 'opencode', model: 'openrouter/model' },
|
||||||
|
runId: 'lane-run-alice',
|
||||||
|
state: 'finished',
|
||||||
|
result: null,
|
||||||
|
warnings: [],
|
||||||
|
diagnostics: [],
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
const svc = new TeamProvisioningService();
|
||||||
|
const snapshot = await (svc as any).persistLaunchStateSnapshot(run, 'finished');
|
||||||
|
|
||||||
|
expect(snapshot.members.bob).toMatchObject({
|
||||||
|
launchState: 'failed_to_start',
|
||||||
|
bootstrapConfirmed: false,
|
||||||
|
hardFailure: true,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
it('includes queued OpenCode secondary lanes in live spawn statuses during createTeam runs', async () => {
|
it('includes queued OpenCode secondary lanes in live spawn statuses during createTeam runs', async () => {
|
||||||
const svc = new TeamProvisioningService();
|
const svc = new TeamProvisioningService();
|
||||||
vi.spyOn(svc as any, 'refreshMemberSpawnStatusesFromLeadInbox').mockResolvedValue(undefined);
|
vi.spyOn(svc as any, 'refreshMemberSpawnStatusesFromLeadInbox').mockResolvedValue(undefined);
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue