diff --git a/src/main/services/team/TeamLaunchStateEvaluator.ts b/src/main/services/team/TeamLaunchStateEvaluator.ts index 136d4044..6171c037 100644 --- a/src/main/services/team/TeamLaunchStateEvaluator.ts +++ b/src/main/services/team/TeamLaunchStateEvaluator.ts @@ -601,7 +601,7 @@ export function normalizePersistedLaunchSnapshot( name, launchState: failed ? 'failed_to_start' : confirmed ? 'confirmed_alive' : 'starting', agentToolAccepted: true, - runtimeAlive: confirmed, + runtimeAlive: false, bootstrapConfirmed: confirmed, hardFailure: failed, hardFailureReason: failed @@ -620,7 +620,7 @@ export function normalizePersistedLaunchSnapshot( typeof maybeLegacy.leadSessionId === 'string' && maybeLegacy.leadSessionId.trim().length > 0 ? maybeLegacy.leadSessionId.trim() : undefined, - launchPhase: 'finished', + launchPhase: 'reconciled', members, updatedAt, }); diff --git a/src/main/services/team/TeamLaunchStateStore.ts b/src/main/services/team/TeamLaunchStateStore.ts index 2f0dd0cf..fd528c10 100644 --- a/src/main/services/team/TeamLaunchStateStore.ts +++ b/src/main/services/team/TeamLaunchStateStore.ts @@ -24,6 +24,18 @@ export function getTeamLaunchSummaryPath(teamName: string): string { return path.join(getTeamsBasePath(), teamName, TEAM_LAUNCH_SUMMARY_FILE); } +async function isMissingTeamDirectoryWriteRace(teamName: string, error: unknown): Promise { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { + return false; + } + try { + await fs.promises.access(path.dirname(getTeamLaunchStatePath(teamName))); + return false; + } catch { + return true; + } +} + export class TeamLaunchStateStore { async read(teamName: string): Promise { const targetPath = getTeamLaunchStatePath(teamName); @@ -50,6 +62,9 @@ export class TeamLaunchStateStore { `${JSON.stringify(createPersistedLaunchSummaryProjection(snapshot), null, 2)}\n` ); } catch (error) { + if (await isMissingTeamDirectoryWriteRace(teamName, error)) { + return; + } logger.warn( `[${teamName}] Failed to persist launch-state: ${ error instanceof Error ? error.message : String(error) diff --git a/src/main/services/team/TeamProvisioningService.ts b/src/main/services/team/TeamProvisioningService.ts index 30e43f3f..2d17bea9 100644 --- a/src/main/services/team/TeamProvisioningService.ts +++ b/src/main/services/team/TeamProvisioningService.ts @@ -1522,6 +1522,10 @@ function matchesTeamMemberIdentity(leftName: string, rightName: string): boolean ); } +function matchesObservedMemberNameForExpected(observedName: string, expectedName: string): boolean { + return matchesMemberNameOrBase(observedName, expectedName); +} + function matchesExactTeamMemberName(candidateName: string, memberName: string): boolean { const left = candidateName.trim().toLowerCase(); const right = memberName.trim().toLowerCase(); @@ -1534,6 +1538,7 @@ interface MemberSpawnInboxCursor { } type LeadInboxMemberSpawnMessage = InboxMessage & { messageId: string }; +type LeadInboxLaunchReconcileMessage = Pick; function compareMemberSpawnInboxCursor( left: MemberSpawnInboxCursor, @@ -4148,7 +4153,9 @@ export class TeamProvisioningService { private getMixedSecondaryLaunchPhase(run: ProvisioningRun): PersistedTeamLaunchPhase { return (run.mixedSecondaryLanes ?? []).some( - (lane) => lane.state !== 'finished' || lane.result?.teamLaunchState === 'partial_pending' + (lane) => + (!lane.result && lane.state !== 'finished') || + lane.result?.teamLaunchState === 'partial_pending' ) ? 'active' : 'finished'; @@ -5047,7 +5054,7 @@ export class TeamProvisioningService { } const matches = expectedMembers.filter((memberName) => - matchesTeamMemberIdentity(memberName, trimmedCandidate) + matchesObservedMemberNameForExpected(trimmedCandidate, memberName) ); return matches.length === 1 ? (matches[0] ?? null) : null; } @@ -6510,7 +6517,9 @@ export class TeamProvisioningService { launchPhase: run.provisioningComplete ? 'finished' : 'active', statuses: this.buildRuntimeSpawnStatusRecord(run), }); - const snapshot = liveSnapshot ?? persisted; + const rawSnapshot = liveSnapshot ?? persisted; + const metaMembers = await this.membersMetaStore.getMembers(teamName).catch(() => []); + const snapshot = this.filterRemovedMembersFromLaunchSnapshot(rawSnapshot, metaMembers); const statuses = await this.attachLiveRuntimeMetadataToStatuses( teamName, snapshotToMemberSpawnStatuses(snapshot) @@ -11643,7 +11652,7 @@ export class TeamProvisioningService { ? memberName : (() => { const matches = Object.keys(nextStatuses).filter((candidateName) => - matchesTeamMemberIdentity(candidateName, memberName) + matchesObservedMemberNameForExpected(memberName, candidateName) ); return matches.length === 1 ? matches[0] : null; })(); @@ -12664,6 +12673,70 @@ export class TeamProvisioningService { return hasMixedPersistedLaunchMetadata(snapshot); } + private hasMixedSecondaryLaunchMetadata(snapshot: PersistedTeamLaunchSnapshot | null): boolean { + if (!snapshot) { + return false; + } + return Object.values(snapshot.members).some( + (member) => + member?.laneKind === 'secondary' || + (typeof member?.laneId === 'string' && member.laneId.startsWith('secondary:')) + ); + } + + private hasPrimaryOnlyLaneAwareLaunchMetadata( + snapshot: PersistedTeamLaunchSnapshot | null + ): boolean { + if (!snapshot || this.hasMixedSecondaryLaunchMetadata(snapshot)) { + return false; + } + + return Object.values(snapshot.members).some( + (member) => + Boolean(member?.laneId) || + Boolean(member?.laneKind) || + Boolean(member?.laneOwnerProviderId) || + Boolean(member?.launchIdentity) + ); + } + + private hasLeadInboxLaunchReconcileHeartbeat( + snapshot: PersistedTeamLaunchSnapshot, + messages: readonly LeadInboxLaunchReconcileMessage[] + ): boolean { + const expectedMembers = this.getPersistedLaunchMemberNames(snapshot); + if (expectedMembers.length === 0 || messages.length === 0) { + return false; + } + + return messages.some((message) => { + if ( + typeof message.from !== 'string' || + typeof message.text !== 'string' || + typeof message.timestamp !== 'string' || + !isMeaningfulBootstrapCheckInMessage(message.text) + ) { + return false; + } + + const expected = this.resolveExpectedLaunchMemberName(expectedMembers, message.from); + if (!expected) { + return false; + } + + const current = snapshot.members[expected]; + const firstAcceptedAt = current?.firstSpawnAcceptedAt + ? Date.parse(current.firstSpawnAcceptedAt) + : NaN; + const messageTs = Date.parse(message.timestamp); + return ( + !Number.isFinite(firstAcceptedAt) || + !Number.isFinite(messageTs) || + messageTs >= firstAcceptedAt + ); + }); + } + private shouldRecoverStalePersistedMixedLaunchSnapshot( snapshot: PersistedTeamLaunchSnapshot ): boolean { @@ -12701,13 +12774,16 @@ export class TeamProvisioningService { return null; } - if (snapshot.teamLaunchState === 'clean_success' && launchPhase !== 'active') { + const metaMembers = await this.membersMetaStore.getMembers(run.teamName).catch(() => []); + const filteredSnapshot = this.filterRemovedMembersFromLaunchSnapshot(snapshot, metaMembers); + + if (filteredSnapshot.teamLaunchState === 'clean_success' && launchPhase !== 'active') { await this.clearPersistedLaunchState(run.teamName); return null; } - await this.launchStateStore.write(run.teamName, snapshot); - return snapshot; + await this.launchStateStore.write(run.teamName, filteredSnapshot); + return filteredSnapshot; } private async launchSingleMixedSecondaryLane( @@ -12742,6 +12818,7 @@ export class TeamProvisioningService { lane.warnings = []; lane.diagnostics = [message]; await this.publishMixedSecondaryLaneStatusChange(run, lane); + lane.state = 'finished'; return; } @@ -12802,7 +12879,6 @@ export class TeamProvisioningService { this.deleteSecondaryRuntimeRun(run.teamName, lane.laneId); return; } - lane.state = 'finished'; lane.result = result; lane.warnings = [...result.warnings]; lane.diagnostics = [...migration.diagnostics, ...result.diagnostics]; @@ -12829,7 +12905,6 @@ export class TeamProvisioningService { return; } const message = error instanceof Error ? error.message : String(error); - lane.state = 'finished'; lane.result = { runId: lane.runId, teamName: run.teamName, @@ -12864,6 +12939,7 @@ export class TeamProvisioningService { } await this.publishMixedSecondaryLaneStatusChange(run, lane); + lane.state = 'finished'; } private async stopSingleMixedSecondaryRuntimeLane( @@ -12931,7 +13007,6 @@ export class TeamProvisioningService { logger.warn( `[${run.teamName}] OpenCode secondary lane ${lane.laneId} crashed during launch orchestration: ${message}` ); - lane.state = 'finished'; lane.result = createUnexpectedMixedSecondaryLaneFailureResult({ runId: lane.runId ?? randomUUID(), teamName: run.teamName, @@ -12949,6 +13024,7 @@ export class TeamProvisioningService { }).catch(() => undefined); this.deleteSecondaryRuntimeRun(run.teamName, lane.laneId); await this.publishMixedSecondaryLaneStatusChange(run, lane).catch(() => undefined); + lane.state = 'finished'; } })(); } @@ -13010,7 +13086,7 @@ export class TeamProvisioningService { ): Promise { if ( persistedSnapshot && - this.hasMixedLaunchMetadata(persistedSnapshot) && + this.hasMixedSecondaryLaunchMetadata(persistedSnapshot) && !this.shouldRecoverStalePersistedMixedLaunchSnapshot(persistedSnapshot) ) { return persistedSnapshot; @@ -13245,6 +13321,39 @@ export class TeamProvisioningService { } } + private async readLeadInboxMessagesForLaunchReconcile( + teamName: string, + leadName: string + ): Promise { + const inboxPath = path.join(getTeamsBasePath(), teamName, 'inboxes', `${leadName}.json`); + try { + const raw = await tryReadRegularFileUtf8(inboxPath, { + timeoutMs: TEAM_JSON_READ_TIMEOUT_MS, + maxBytes: TEAM_INBOX_MAX_BYTES, + }); + if (!raw) { + return []; + } + const parsed = JSON.parse(raw) as unknown; + if (!Array.isArray(parsed)) { + return []; + } + return parsed.flatMap((item): LeadInboxLaunchReconcileMessage[] => { + if (!item || typeof item !== 'object') { + return []; + } + const row = item as Partial; + return typeof row.from === 'string' && + typeof row.text === 'string' && + typeof row.timestamp === 'string' + ? [{ from: row.from, text: row.text, timestamp: row.timestamp }] + : []; + }); + } catch { + return []; + } + } + private async reconcilePersistedLaunchState(teamName: string): Promise<{ snapshot: ReturnType | null; statuses: Record; @@ -13310,11 +13419,19 @@ export class TeamProvisioningService { // best-effort } - let leadInboxMessages: Awaited> = []; - try { - leadInboxMessages = await this.inboxReader.getMessagesFor(teamName, leadName); - } catch { - // best-effort + const leadInboxMessages = await this.readLeadInboxMessagesForLaunchReconcile( + teamName, + leadName + ); + + if ( + this.hasPrimaryOnlyLaneAwareLaunchMetadata(filteredPersisted) && + !this.hasLeadInboxLaunchReconcileHeartbeat(filteredPersisted, leadInboxMessages) + ) { + return { + snapshot: filteredPersisted, + statuses: snapshotToMemberSpawnStatuses(filteredPersisted), + }; } const liveAgentNames = await this.getLiveTeamAgentNames(teamName); @@ -13347,11 +13464,12 @@ export class TeamProvisioningService { current.lastHeartbeatAt = current.lastHeartbeatAt ?? bootstrapMember.lastHeartbeatAt; } const matchedConfigNames = [...configMembers].filter((name) => - matchesTeamMemberIdentity(name, expected) + matchesObservedMemberNameForExpected(name, expected) ); - const runtimeAlive = [...liveAgentNames].some((name) => - matchesTeamMemberIdentity(name, expected) + const observedRuntimeAlive = [...liveAgentNames].some((name) => + matchesObservedMemberNameForExpected(name, expected) ); + const runtimeAlive = current.runtimeAlive === true || observedRuntimeAlive; const heartbeatMessage = leadInboxMessages.find((message) => { if ( typeof message.from !== 'string' || @@ -13384,7 +13502,7 @@ export class TeamProvisioningService { const acceptedAtMs = current.firstSpawnAcceptedAt != null ? Date.parse(current.firstSpawnAcceptedAt) : NaN; current.runtimeAlive = runtimeAlive; - current.lastRuntimeAliveAt = runtimeAlive ? now : current.lastRuntimeAliveAt; + current.lastRuntimeAliveAt = observedRuntimeAlive ? now : current.lastRuntimeAliveAt; current.sources = { ...(current.sources ?? {}), processAlive: runtimeAlive || undefined, @@ -13463,7 +13581,7 @@ export class TeamProvisioningService { teamName, expectedMembers: persistedMemberNames, leadSessionId: filteredPersisted.leadSessionId, - launchPhase: filteredPersisted.launchPhase === 'active' ? 'active' : 'reconciled', + launchPhase: filteredPersisted.launchPhase, members: nextMembers, updatedAt: now, }); diff --git a/test/main/services/team/TeamProvisioningService.test.ts b/test/main/services/team/TeamProvisioningService.test.ts index 381ee274..2084a6d6 100644 --- a/test/main/services/team/TeamProvisioningService.test.ts +++ b/test/main/services/team/TeamProvisioningService.test.ts @@ -2711,18 +2711,16 @@ describe('TeamProvisioningService', () => { await (svc as any).launchMixedSecondaryLaneIfNeeded(run); await vi.waitFor(async () => { expect(adapterLaunch).toHaveBeenCalledTimes(1); - await expect(readOpenCodeRuntimeLaneIndex(tempTeamsBase, teamName)).resolves.toMatchObject( - { - lanes: { - 'secondary:opencode:bob': { - state: 'degraded', - diagnostics: expect.arrayContaining([ - 'OpenCode readiness bridge failed: timeout: OpenCode bridge command timed out', - ]), - }, + await expect(readOpenCodeRuntimeLaneIndex(tempTeamsBase, teamName)).resolves.toMatchObject({ + lanes: { + 'secondary:opencode:bob': { + state: 'degraded', + diagnostics: expect.arrayContaining([ + 'OpenCode readiness bridge failed: timeout: OpenCode bridge command timed out', + ]), }, - } - ); + }, + }); }); }); @@ -4429,6 +4427,7 @@ describe('TeamProvisioningService', () => { }; const membersMetaStore = { writeMembers: vi.fn(async () => {}), + getMembers: vi.fn(async () => []), getMeta: vi.fn(async () => null), }; const teamMetaStore = { @@ -4534,7 +4533,9 @@ describe('TeamProvisioningService', () => { stdio: ['pipe', 'pipe', 'pipe'], }); const spawnArgs = spawnCall?.[1] as string[]; - expect(spawnArgs).toEqual(expect.arrayContaining(['--model', 'gpt-5.4', '--effort', 'medium'])); + expect(spawnArgs).toEqual( + expect.arrayContaining(['--model', 'gpt-5.4', '--effort', 'medium']) + ); const bootstrapSpec = readBootstrapSpecFromSpawnArgs(spawnArgs); expect(bootstrapSpec).toMatchObject({ @@ -4682,11 +4683,18 @@ describe('TeamProvisioningService', () => { ); const config = JSON.parse( - fs.readFileSync(path.join(tempTeamsBase, 'safe-opencode-only-launch', 'config.json'), 'utf8') + fs.readFileSync( + path.join(tempTeamsBase, 'safe-opencode-only-launch', 'config.json'), + 'utf8' + ) ) as { members: Array<{ name: string; providerId?: string; model?: string }> }; expect(config.members).toEqual( expect.arrayContaining([ - expect.objectContaining({ name: 'team-lead', providerId: 'opencode', model: 'big-pickle' }), + expect.objectContaining({ + name: 'team-lead', + providerId: 'opencode', + model: 'big-pickle', + }), expect.objectContaining({ name: 'bob', providerId: 'opencode', @@ -4895,7 +4903,9 @@ describe('TeamProvisioningService', () => { status: 'online', launchState: 'confirmed_alive', }); - expect(publicStatuses.expectedMembers).toEqual(expect.arrayContaining(['alice', 'bob', 'tom'])); + expect(publicStatuses.expectedMembers).toEqual( + expect.arrayContaining(['alice', 'bob', 'tom']) + ); await svc.cancelProvisioning(runId); }); @@ -7018,15 +7028,23 @@ describe('TeamProvisioningService', () => { write: vi.fn(async () => {}), clear: vi.fn(async () => {}), }; - vi.spyOn((svc as any).inboxReader, 'getMessagesFor').mockResolvedValue([ - { - from: 'alice-2', - text: 'heartbeat', - timestamp: '2026-04-16T10:00:00.000Z', - messageId: 'msg-suffixed-reconcile', - read: false, - }, - ]); + fs.mkdirSync(path.join(tempTeamsBase, teamName, 'inboxes'), { recursive: true }); + fs.writeFileSync( + path.join(tempTeamsBase, teamName, 'inboxes', 'team-lead.json'), + JSON.stringify( + [ + { + from: 'alice-2', + text: 'heartbeat', + timestamp: '2026-04-16T10:00:00.000Z', + messageId: 'msg-suffixed-reconcile', + read: false, + }, + ], + null, + 2 + ) + ); (svc as any).getLiveTeamAgentNames = vi.fn(async () => new Set()); const result = await (svc as any).reconcilePersistedLaunchState(teamName);