admin-valentin/agent-ecosystem
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(ci): stabilize team launch reconciliation

Browse source
This commit is contained in:
777genius 2026-04-24 01:28:11 +03:00
parent e9f46c5bd4
commit ba4de3775b
4 changed files with 198 additions and 47 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/main/services/team/TeamLaunchStateEvaluator.ts
View file

@ -601,7 +601,7 @@ export function normalizePersistedLaunchSnapshot(
name,
launchState: failed ? 'failed_to_start' : confirmed ? 'confirmed_alive' : 'starting',
agentToolAccepted: true,
runtimeAlive: confirmed,
runtimeAlive: false,
bootstrapConfirmed: confirmed,
hardFailure: failed,
hardFailureReason: failed
@ -620,7 +620,7 @@ export function normalizePersistedLaunchSnapshot(
typeof maybeLegacy.leadSessionId === 'string' && maybeLegacy.leadSessionId.trim().length > 0
? maybeLegacy.leadSessionId.trim()
: undefined,
launchPhase: 'finished',
launchPhase: 'reconciled',
members,
updatedAt,
});

15
src/main/services/team/TeamLaunchStateStore.ts
View file

@ -24,6 +24,18 @@ export function getTeamLaunchSummaryPath(teamName: string): string {
return path.join(getTeamsBasePath(), teamName, TEAM_LAUNCH_SUMMARY_FILE);
}
async function isMissingTeamDirectoryWriteRace(teamName: string, error: unknown): Promise<boolean> {
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
return false;
}
try {
await fs.promises.access(path.dirname(getTeamLaunchStatePath(teamName)));
return false;
} catch {
return true;
}
}
export class TeamLaunchStateStore {
async read(teamName: string): Promise<PersistedTeamLaunchSnapshot | null> {
const targetPath = getTeamLaunchStatePath(teamName);
@ -50,6 +62,9 @@ export class TeamLaunchStateStore {
`${JSON.stringify(createPersistedLaunchSummaryProjection(snapshot), null, 2)}\n`
);
} catch (error) {
if (await isMissingTeamDirectoryWriteRace(teamName, error)) {
return;
}
logger.warn(
`[${teamName}] Failed to persist launch-state: ${
error instanceof Error ? error.message : String(error)

160
src/main/services/team/TeamProvisioningService.ts
View file

@ -1522,6 +1522,10 @@ function matchesTeamMemberIdentity(leftName: string, rightName: string): boolean
);
}
function matchesObservedMemberNameForExpected(observedName: string, expectedName: string): boolean {
return matchesMemberNameOrBase(observedName, expectedName);
}
function matchesExactTeamMemberName(candidateName: string, memberName: string): boolean {
const left = candidateName.trim().toLowerCase();
const right = memberName.trim().toLowerCase();
@ -1534,6 +1538,7 @@ interface MemberSpawnInboxCursor {
}
type LeadInboxMemberSpawnMessage = InboxMessage & { messageId: string };
type LeadInboxLaunchReconcileMessage = Pick<InboxMessage, 'from' | 'text' | 'timestamp'>;
function compareMemberSpawnInboxCursor(
left: MemberSpawnInboxCursor,
@ -4148,7 +4153,9 @@ export class TeamProvisioningService {
private getMixedSecondaryLaunchPhase(run: ProvisioningRun): PersistedTeamLaunchPhase {
return (run.mixedSecondaryLanes ?? []).some(
(lane) => lane.state !== 'finished' || lane.result?.teamLaunchState === 'partial_pending'
(lane) =>
(!lane.result && lane.state !== 'finished') ||
lane.result?.teamLaunchState === 'partial_pending'
)
? 'active'
: 'finished';
@ -5047,7 +5054,7 @@ export class TeamProvisioningService {
}
const matches = expectedMembers.filter((memberName) =>
matchesTeamMemberIdentity(memberName, trimmedCandidate)
matchesObservedMemberNameForExpected(trimmedCandidate, memberName)
);
return matches.length === 1 ? (matches[0] ?? null) : null;
}
@ -6510,7 +6517,9 @@ export class TeamProvisioningService {
launchPhase: run.provisioningComplete ? 'finished' : 'active',
statuses: this.buildRuntimeSpawnStatusRecord(run),
});
const snapshot = liveSnapshot ?? persisted;
const rawSnapshot = liveSnapshot ?? persisted;
const metaMembers = await this.membersMetaStore.getMembers(teamName).catch(() => []);
const snapshot = this.filterRemovedMembersFromLaunchSnapshot(rawSnapshot, metaMembers);
const statuses = await this.attachLiveRuntimeMetadataToStatuses(
teamName,
snapshotToMemberSpawnStatuses(snapshot)
@ -11643,7 +11652,7 @@ export class TeamProvisioningService {
? memberName
: (() => {
const matches = Object.keys(nextStatuses).filter((candidateName) =>
matchesTeamMemberIdentity(candidateName, memberName)
matchesObservedMemberNameForExpected(memberName, candidateName)
);
return matches.length === 1 ? matches[0] : null;
})();
@ -12664,6 +12673,70 @@ export class TeamProvisioningService {
return hasMixedPersistedLaunchMetadata(snapshot);
}
private hasMixedSecondaryLaunchMetadata(snapshot: PersistedTeamLaunchSnapshot | null): boolean {
if (!snapshot) {
return false;
}
return Object.values(snapshot.members).some(
(member) =>
member?.laneKind === 'secondary' ||
(typeof member?.laneId === 'string' && member.laneId.startsWith('secondary:'))
);
}
private hasPrimaryOnlyLaneAwareLaunchMetadata(
snapshot: PersistedTeamLaunchSnapshot | null
): boolean {
if (!snapshot || this.hasMixedSecondaryLaunchMetadata(snapshot)) {
return false;
}
return Object.values(snapshot.members).some(
(member) =>
Boolean(member?.laneId) ||
Boolean(member?.laneKind) ||
Boolean(member?.laneOwnerProviderId) ||
Boolean(member?.launchIdentity)
);
}
private hasLeadInboxLaunchReconcileHeartbeat(
snapshot: PersistedTeamLaunchSnapshot,
messages: readonly LeadInboxLaunchReconcileMessage[]
): boolean {
const expectedMembers = this.getPersistedLaunchMemberNames(snapshot);
if (expectedMembers.length === 0 || messages.length === 0) {
return false;
}
return messages.some((message) => {
if (
typeof message.from !== 'string' ||
typeof message.text !== 'string' ||
typeof message.timestamp !== 'string' ||
!isMeaningfulBootstrapCheckInMessage(message.text)
) {
return false;
}
const expected = this.resolveExpectedLaunchMemberName(expectedMembers, message.from);
if (!expected) {
return false;
}
const current = snapshot.members[expected];
const firstAcceptedAt = current?.firstSpawnAcceptedAt
? Date.parse(current.firstSpawnAcceptedAt)
: NaN;
const messageTs = Date.parse(message.timestamp);
return (
!Number.isFinite(firstAcceptedAt) ||
!Number.isFinite(messageTs) ||
messageTs >= firstAcceptedAt
);
});
}
private shouldRecoverStalePersistedMixedLaunchSnapshot(
snapshot: PersistedTeamLaunchSnapshot
): boolean {
@ -12701,13 +12774,16 @@ export class TeamProvisioningService {
return null;
}
if (snapshot.teamLaunchState === 'clean_success' && launchPhase !== 'active') {
const metaMembers = await this.membersMetaStore.getMembers(run.teamName).catch(() => []);
const filteredSnapshot = this.filterRemovedMembersFromLaunchSnapshot(snapshot, metaMembers);
if (filteredSnapshot.teamLaunchState === 'clean_success' && launchPhase !== 'active') {
await this.clearPersistedLaunchState(run.teamName);
return null;
}
await this.launchStateStore.write(run.teamName, snapshot);
return snapshot;
await this.launchStateStore.write(run.teamName, filteredSnapshot);
return filteredSnapshot;
}
private async launchSingleMixedSecondaryLane(
@ -12742,6 +12818,7 @@ export class TeamProvisioningService {
lane.warnings = [];
lane.diagnostics = [message];
await this.publishMixedSecondaryLaneStatusChange(run, lane);
lane.state = 'finished';
return;
}
@ -12802,7 +12879,6 @@ export class TeamProvisioningService {
this.deleteSecondaryRuntimeRun(run.teamName, lane.laneId);
return;
}
lane.state = 'finished';
lane.result = result;
lane.warnings = [...result.warnings];
lane.diagnostics = [...migration.diagnostics, ...result.diagnostics];
@ -12829,7 +12905,6 @@ export class TeamProvisioningService {
return;
}
const message = error instanceof Error ? error.message : String(error);
lane.state = 'finished';
lane.result = {
runId: lane.runId,
teamName: run.teamName,
@ -12864,6 +12939,7 @@ export class TeamProvisioningService {
}
await this.publishMixedSecondaryLaneStatusChange(run, lane);
lane.state = 'finished';
}
private async stopSingleMixedSecondaryRuntimeLane(
@ -12931,7 +13007,6 @@ export class TeamProvisioningService {
logger.warn(
`[${run.teamName}] OpenCode secondary lane ${lane.laneId} crashed during launch orchestration: ${message}`
);
lane.state = 'finished';
lane.result = createUnexpectedMixedSecondaryLaneFailureResult({
runId: lane.runId ?? randomUUID(),
teamName: run.teamName,
@ -12949,6 +13024,7 @@ export class TeamProvisioningService {
}).catch(() => undefined);
this.deleteSecondaryRuntimeRun(run.teamName, lane.laneId);
await this.publishMixedSecondaryLaneStatusChange(run, lane).catch(() => undefined);
lane.state = 'finished';
}
})();
}
@ -13010,7 +13086,7 @@ export class TeamProvisioningService {
): Promise<PersistedTeamLaunchSnapshot | null> {
if (
persistedSnapshot &&
this.hasMixedLaunchMetadata(persistedSnapshot) &&
this.hasMixedSecondaryLaunchMetadata(persistedSnapshot) &&
!this.shouldRecoverStalePersistedMixedLaunchSnapshot(persistedSnapshot)
) {
return persistedSnapshot;
@ -13245,6 +13321,39 @@ export class TeamProvisioningService {
}
}
private async readLeadInboxMessagesForLaunchReconcile(
teamName: string,
leadName: string
): Promise<LeadInboxLaunchReconcileMessage[]> {
const inboxPath = path.join(getTeamsBasePath(), teamName, 'inboxes', `${leadName}.json`);
try {
const raw = await tryReadRegularFileUtf8(inboxPath, {
timeoutMs: TEAM_JSON_READ_TIMEOUT_MS,
maxBytes: TEAM_INBOX_MAX_BYTES,
});
if (!raw) {
return [];
}
const parsed = JSON.parse(raw) as unknown;
if (!Array.isArray(parsed)) {
return [];
}
return parsed.flatMap((item): LeadInboxLaunchReconcileMessage[] => {
if (!item || typeof item !== 'object') {
return [];
}
const row = item as Partial<InboxMessage>;
return typeof row.from === 'string' &&
typeof row.text === 'string' &&
typeof row.timestamp === 'string'
? [{ from: row.from, text: row.text, timestamp: row.timestamp }]
: [];
});
} catch {
return [];
}
}
private async reconcilePersistedLaunchState(teamName: string): Promise<{
snapshot: ReturnType<typeof createPersistedLaunchSnapshot> | null;
statuses: Record<string, MemberSpawnStatusEntry>;
@ -13310,11 +13419,19 @@ export class TeamProvisioningService {
// best-effort
}
let leadInboxMessages: Awaited<ReturnType<TeamInboxReader['getMessagesFor']>> = [];
try {
leadInboxMessages = await this.inboxReader.getMessagesFor(teamName, leadName);
} catch {
// best-effort
const leadInboxMessages = await this.readLeadInboxMessagesForLaunchReconcile(
teamName,
leadName
);
if (
this.hasPrimaryOnlyLaneAwareLaunchMetadata(filteredPersisted) &&
!this.hasLeadInboxLaunchReconcileHeartbeat(filteredPersisted, leadInboxMessages)
) {
return {
snapshot: filteredPersisted,
statuses: snapshotToMemberSpawnStatuses(filteredPersisted),
};
}
const liveAgentNames = await this.getLiveTeamAgentNames(teamName);
@ -13347,11 +13464,12 @@ export class TeamProvisioningService {
current.lastHeartbeatAt = current.lastHeartbeatAt ?? bootstrapMember.lastHeartbeatAt;
}
const matchedConfigNames = [...configMembers].filter((name) =>
matchesTeamMemberIdentity(name, expected)
matchesObservedMemberNameForExpected(name, expected)
);
const runtimeAlive = [...liveAgentNames].some((name) =>
matchesTeamMemberIdentity(name, expected)
const observedRuntimeAlive = [...liveAgentNames].some((name) =>
matchesObservedMemberNameForExpected(name, expected)
);
const runtimeAlive = current.runtimeAlive === true || observedRuntimeAlive;
const heartbeatMessage = leadInboxMessages.find((message) => {
if (
typeof message.from !== 'string' ||
@ -13384,7 +13502,7 @@ export class TeamProvisioningService {
const acceptedAtMs =
current.firstSpawnAcceptedAt != null ? Date.parse(current.firstSpawnAcceptedAt) : NaN;
current.runtimeAlive = runtimeAlive;
current.lastRuntimeAliveAt = runtimeAlive ? now : current.lastRuntimeAliveAt;
current.lastRuntimeAliveAt = observedRuntimeAlive ? now : current.lastRuntimeAliveAt;
current.sources = {
...(current.sources ?? {}),
processAlive: runtimeAlive || undefined,
@ -13463,7 +13581,7 @@ export class TeamProvisioningService {
teamName,
expectedMembers: persistedMemberNames,
leadSessionId: filteredPersisted.leadSessionId,
launchPhase: filteredPersisted.launchPhase === 'active' ? 'active' : 'reconciled',
launchPhase: filteredPersisted.launchPhase,
members: nextMembers,
updatedAt: now,
});

66
test/main/services/team/TeamProvisioningService.test.ts
View file

@ -2711,18 +2711,16 @@ describe('TeamProvisioningService', () => {
await (svc as any).launchMixedSecondaryLaneIfNeeded(run);
await vi.waitFor(async () => {
expect(adapterLaunch).toHaveBeenCalledTimes(1);
await expect(readOpenCodeRuntimeLaneIndex(tempTeamsBase, teamName)).resolves.toMatchObject(
{
lanes: {
'secondary:opencode:bob': {
state: 'degraded',
diagnostics: expect.arrayContaining([
'OpenCode readiness bridge failed: timeout: OpenCode bridge command timed out',
]),
},
await expect(readOpenCodeRuntimeLaneIndex(tempTeamsBase, teamName)).resolves.toMatchObject({
lanes: {
'secondary:opencode:bob': {
state: 'degraded',
diagnostics: expect.arrayContaining([
'OpenCode readiness bridge failed: timeout: OpenCode bridge command timed out',
]),
},
}
);
},
});
});
});
@ -4429,6 +4427,7 @@ describe('TeamProvisioningService', () => {
};
const membersMetaStore = {
writeMembers: vi.fn(async () => {}),
getMembers: vi.fn(async () => []),
getMeta: vi.fn(async () => null),
};
const teamMetaStore = {
@ -4534,7 +4533,9 @@ describe('TeamProvisioningService', () => {
stdio: ['pipe', 'pipe', 'pipe'],
});
const spawnArgs = spawnCall?.[1] as string[];
expect(spawnArgs).toEqual(expect.arrayContaining(['--model', 'gpt-5.4', '--effort', 'medium']));
expect(spawnArgs).toEqual(
expect.arrayContaining(['--model', 'gpt-5.4', '--effort', 'medium'])
);
const bootstrapSpec = readBootstrapSpecFromSpawnArgs(spawnArgs);
expect(bootstrapSpec).toMatchObject({
@ -4682,11 +4683,18 @@ describe('TeamProvisioningService', () => {
);
const config = JSON.parse(
fs.readFileSync(path.join(tempTeamsBase, 'safe-opencode-only-launch', 'config.json'), 'utf8')
fs.readFileSync(
path.join(tempTeamsBase, 'safe-opencode-only-launch', 'config.json'),
'utf8'
)
) as { members: Array<{ name: string; providerId?: string; model?: string }> };
expect(config.members).toEqual(
expect.arrayContaining([
expect.objectContaining({ name: 'team-lead', providerId: 'opencode', model: 'big-pickle' }),
expect.objectContaining({
name: 'team-lead',
providerId: 'opencode',
model: 'big-pickle',
}),
expect.objectContaining({
name: 'bob',
providerId: 'opencode',
@ -4895,7 +4903,9 @@ describe('TeamProvisioningService', () => {
status: 'online',
launchState: 'confirmed_alive',
});
expect(publicStatuses.expectedMembers).toEqual(expect.arrayContaining(['alice', 'bob', 'tom']));
expect(publicStatuses.expectedMembers).toEqual(
expect.arrayContaining(['alice', 'bob', 'tom'])
);
await svc.cancelProvisioning(runId);
});
@ -7018,15 +7028,23 @@ describe('TeamProvisioningService', () => {
write: vi.fn(async () => {}),
clear: vi.fn(async () => {}),
};
vi.spyOn((svc as any).inboxReader, 'getMessagesFor').mockResolvedValue([
{
from: 'alice-2',
text: 'heartbeat',
timestamp: '2026-04-16T10:00:00.000Z',
messageId: 'msg-suffixed-reconcile',
read: false,
},
]);
fs.mkdirSync(path.join(tempTeamsBase, teamName, 'inboxes'), { recursive: true });
fs.writeFileSync(
path.join(tempTeamsBase, teamName, 'inboxes', 'team-lead.json'),
JSON.stringify(
[
{
from: 'alice-2',
text: 'heartbeat',
timestamp: '2026-04-16T10:00:00.000Z',
messageId: 'msg-suffixed-reconcile',
read: false,
},
],
null,
2
)
);
(svc as any).getLiveTeamAgentNames = vi.fn(async () => new Set<string>());
const result = await (svc as any).reconcilePersistedLaunchState(teamName);