From 4d5533585ceb4888962de9542d7e509fc7dd9601 Mon Sep 17 00:00:00 2001 From: 777genius Date: Tue, 28 Apr 2026 18:38:46 +0300 Subject: [PATCH] fix(team): reconcile teammate launch liveness --- .../services/team/TeamProvisioningService.ts | 392 ++++++++++++++++-- .../TeamAgentLaunchMatrix.safe-e2e.test.ts | 186 +++++++++ .../team/TeamProvisioningService.test.ts | 159 ++++++- 3 files changed, 704 insertions(+), 33 deletions(-) diff --git a/src/main/services/team/TeamProvisioningService.ts b/src/main/services/team/TeamProvisioningService.ts index a10ee599..8e087424 100644 --- a/src/main/services/team/TeamProvisioningService.ts +++ b/src/main/services/team/TeamProvisioningService.ts @@ -1714,6 +1714,10 @@ function isOpenCodeBridgeLaunchFailureReason(reason?: string): boolean { return reason?.trim() === 'OpenCode bridge reported member launch failure'; } +function isRegisteredRuntimeMetadataFailureReason(reason?: string): boolean { + return reason?.trim() === 'registered runtime metadata without live process'; +} + function isTmuxNoServerRunningError(error: unknown): boolean { const text = error instanceof Error ? error.message : String(error ?? ''); return ( @@ -1727,6 +1731,7 @@ function isAutoClearableLaunchFailureReason(reason?: string): boolean { isNeverSpawnedDuringLaunchReason(reason) || isLaunchGraceWindowFailureReason(reason) || isConfigRegistrationFailureReason(reason) || + isRegisteredRuntimeMetadataFailureReason(reason) || isOpenCodeBridgeLaunchFailureReason(reason) ); } @@ -4179,6 +4184,7 @@ export class TeamProvisioningService { private inFlightResponses = new Set(); private runtimeAdapterRegistry: TeamRuntimeAdapterRegistry | null = null; private controlApiBaseUrlResolver: (() => Promise) | null = null; + private readonly stoppedTeamOpenCodeRuntimeCleanupInFlight = new Map>(); private crossTeamSender: | ((request: { fromTeam: string; @@ -4728,6 +4734,246 @@ export class TeamProvisioningService { return null; } + private canDeliverToOpenCodeRuntimeForTeam(teamName: string): boolean { + if (this.isTeamAlive(teamName)) { + return true; + } + return this.hasAlivePersistedTeamProcess(teamName); + } + + private hasAlivePersistedTeamProcess(teamName: string): boolean { + const processesPath = path.join(getTeamsBasePath(), teamName, 'processes.json'); + let parsed: unknown; + try { + parsed = JSON.parse(fs.readFileSync(processesPath, 'utf8')) as unknown; + } catch { + return false; + } + if (!Array.isArray(parsed)) { + return false; + } + return parsed.some((row) => { + if (!row || typeof row !== 'object') { + return false; + } + const processRow = row as { pid?: unknown; stoppedAt?: unknown }; + return ( + typeof processRow.pid === 'number' && + Number.isFinite(processRow.pid) && + processRow.stoppedAt == null && + isProcessAlive(processRow.pid) + ); + }); + } + + private cleanupStoppedTeamOpenCodeRuntimeLanesInBackground(teamName: string): void { + void this.stopOpenCodeRuntimeLanesForStoppedTeam(teamName).catch((error) => { + logger.warn( + `[${teamName}] Failed to clean up stopped-team OpenCode runtime lanes: ${ + error instanceof Error ? error.message : String(error) + }` + ); + }); + } + + private stopOpenCodeRuntimeLanesForStoppedTeam(teamName: string): Promise { + const existing = this.stoppedTeamOpenCodeRuntimeCleanupInFlight.get(teamName); + if (existing) { + return existing; + } + let cleanup!: Promise; + cleanup = this.stopOpenCodeRuntimeLanesForStoppedTeamInternal(teamName).finally(() => { + if (this.stoppedTeamOpenCodeRuntimeCleanupInFlight.get(teamName) === cleanup) { + this.stoppedTeamOpenCodeRuntimeCleanupInFlight.delete(teamName); + } + }); + this.stoppedTeamOpenCodeRuntimeCleanupInFlight.set(teamName, cleanup); + return cleanup; + } + + private async stopOpenCodeRuntimeLanesForStoppedTeamInternal(teamName: string): Promise { + if (this.canDeliverToOpenCodeRuntimeForTeam(teamName)) { + return 0; + } + const laneIndex = await readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName).catch( + () => null + ); + const activeLaneIds = Object.entries(laneIndex?.lanes ?? {}) + .filter(([, entry]) => entry.state === 'active') + .map(([laneId]) => laneId) + .sort((left, right) => left.localeCompare(right)); + if (activeLaneIds.length === 0) { + return 0; + } + + const adapter = this.getOpenCodeRuntimeAdapter(); + const previousLaunchState = await this.launchStateStore.read(teamName).catch(() => null); + const [config, metaMembers] = await Promise.all([ + this.configReader.getConfig(teamName).catch(() => null), + this.membersMetaStore.getMembers(teamName).catch(() => []), + ]); + const evidenceReader = new OpenCodeRuntimeManifestEvidenceReader({ + teamsBasePath: getTeamsBasePath(), + }); + let stopped = 0; + for (const laneId of activeLaneIds) { + const evidence = await evidenceReader.read(teamName, laneId).catch(() => null); + const runId = evidence?.activeRunId?.trim() || null; + if (adapter && runId) { + try { + await adapter.stop({ + runId, + laneId, + teamName, + cwd: this.resolveOpenCodeRuntimeLaneCleanupCwd(teamName, laneId, config, metaMembers), + providerId: 'opencode', + reason: 'cleanup', + previousLaunchState, + force: true, + }); + stopped += 1; + } catch (error) { + logger.warn( + `[${teamName}] Failed to stop orphaned OpenCode lane ${laneId}: ${ + error instanceof Error ? error.message : String(error) + }` + ); + continue; + } + } else if (runId) { + logger.warn( + `[${teamName}] OpenCode lane ${laneId} belongs to stopped team, but runtime adapter is unavailable.` + ); + continue; + } else if (!runId) { + const pidStopResult = this.tryStopPersistedOpenCodeRuntimePidForStoppedLane({ + teamName, + laneId, + previousLaunchState, + }); + if (pidStopResult === 'unsafe') { + continue; + } + } + + await clearOpenCodeRuntimeLaneStorage({ + teamsBasePath: getTeamsBasePath(), + teamName, + laneId, + }).catch(() => undefined); + this.deleteSecondaryRuntimeRun(teamName, laneId); + if (laneId === 'primary') { + this.runtimeAdapterRunByTeam.delete(teamName); + this.aliveRunByTeam.delete(teamName); + this.provisioningRunByTeam.delete(teamName); + } + } + return stopped; + } + + private tryStopPersistedOpenCodeRuntimePidForStoppedLane(input: { + teamName: string; + laneId: string; + previousLaunchState: PersistedTeamLaunchSnapshot | null; + }): 'stopped' | 'no_pid' | 'unsafe' { + const persistedMember = Object.values(input.previousLaunchState?.members ?? {}).find( + (member) => member.providerId === 'opencode' && member.laneId === input.laneId + ); + if (!persistedMember) { + return 'no_pid'; + } + const pid = persistedMember.runtimePid; + if (typeof pid !== 'number' || !Number.isFinite(pid) || pid <= 0) { + return 'no_pid'; + } + const command = this.readProcessCommandByPid(pid); + if (!command) { + return 'no_pid'; + } + const persistedProcessCommand = (persistedMember as { processCommand?: unknown }) + .processCommand; + const expectedCommand = + typeof persistedProcessCommand === 'string' ? persistedProcessCommand.trim() : ''; + if (expectedCommand && command !== expectedCommand) { + logger.warn( + `[${input.teamName}] Refusing to stop persisted OpenCode pid ${pid} for lane ${input.laneId}: process command changed.` + ); + return 'unsafe'; + } + if (!this.isOpenCodeServeCommand(command)) { + logger.warn( + `[${input.teamName}] Refusing to stop persisted OpenCode pid ${pid} for lane ${input.laneId}: process is not opencode serve.` + ); + return 'unsafe'; + } + try { + killProcessByPid(pid); + logger.info( + `[${input.teamName}] Killed orphaned OpenCode runtime pid=${pid} for stopped lane ${input.laneId}` + ); + return 'stopped'; + } catch (error) { + logger.warn( + `[${input.teamName}] Failed to kill orphaned OpenCode runtime pid=${pid} for stopped lane ${ + input.laneId + }: ${error instanceof Error ? error.message : String(error)}` + ); + return 'unsafe'; + } + } + + private readProcessCommandByPid(pid: number): string | null { + if (process.platform === 'win32') { + try { + return ( + listWindowsProcessTableSync() + .find((row) => row.pid === pid) + ?.command?.trim() || null + ); + } catch { + return null; + } + } + try { + return execFileSync('ps', ['-p', String(pid), '-o', 'command='], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'ignore'], + }).trim(); + } catch { + return null; + } + } + + private isOpenCodeServeCommand(command: string): boolean { + return /(^|[/\\\s])opencode(?:\.exe)?(\s|$)/i.test(command) && /\sserve(\s|$)/i.test(command); + } + + private resolveOpenCodeRuntimeLaneCleanupCwd( + teamName: string, + laneId: string, + config: TeamConfig | null, + metaMembers: readonly TeamMember[] + ): string | undefined { + const projectPath = config?.projectPath?.trim() || this.readPersistedTeamProjectPath(teamName); + const memberName = this.extractOpenCodeRuntimeLaneMemberName(laneId); + if (!memberName) { + return projectPath || undefined; + } + const normalized = memberName.toLowerCase(); + const configMember = config?.members?.find( + (member) => member.name?.trim().toLowerCase() === normalized + ); + const metaMember = metaMembers.find( + (member) => member.name?.trim().toLowerCase() === normalized + ); + return metaMember?.cwd?.trim() || configMember?.cwd?.trim() || projectPath || undefined; + } + + private extractOpenCodeRuntimeLaneMemberName(laneId: string): string | null { + const match = /^secondary:opencode:(.+)$/i.exec(laneId.trim()); + return match?.[1]?.trim() || null; + } + private getOpenCodeRuntimeAdapter(): TeamLaunchRuntimeAdapter | null { if (!this.runtimeAdapterRegistry?.has('opencode')) { return null; @@ -5332,6 +5578,10 @@ export class TeamProvisioningService { if (!this.isOpenCodePromptDeliveryWatchdogEnabled()) { return 0; } + if (!this.canDeliverToOpenCodeRuntimeForTeam(teamName)) { + await this.stopOpenCodeRuntimeLanesForStoppedTeam(teamName); + return 0; + } const laneIndex = await readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName).catch( () => null ); @@ -5484,6 +5734,10 @@ export class TeamProvisioningService { if (!adapter) { return { delivered: false, reason: 'opencode_runtime_message_bridge_unavailable' }; } + if (!this.canDeliverToOpenCodeRuntimeForTeam(teamName)) { + this.cleanupStoppedTeamOpenCodeRuntimeLanesInBackground(teamName); + return { delivered: false, reason: 'opencode_runtime_not_active' }; + } const [config, teamMeta, metaMembers] = await Promise.all([ this.configReader.getConfig(teamName).catch(() => null), @@ -6472,6 +6726,10 @@ export class TeamProvisioningService { member: TeamMember; projectPath: string | null; }): Promise { + if (!this.canDeliverToOpenCodeRuntimeForTeam(input.teamName)) { + this.cleanupStoppedTeamOpenCodeRuntimeLanesInBackground(input.teamName); + return false; + } const snapshot = await this.launchStateStore.read(input.teamName).catch(() => null); const persistedMember = snapshot?.members?.[input.member.name] ?? @@ -6499,6 +6757,10 @@ export class TeamProvisioningService { private async tryRecoverOpenCodeRuntimeLanesForDeliveryWatchdog( teamName: string ): Promise { + if (!this.canDeliverToOpenCodeRuntimeForTeam(teamName)) { + this.cleanupStoppedTeamOpenCodeRuntimeLanesInBackground(teamName); + return []; + } const snapshot = await this.launchStateStore.read(teamName).catch(() => null); const candidates = Object.values(snapshot?.members ?? {}).filter( isRecoverablePersistedOpenCodeRuntimeCandidate @@ -14114,6 +14376,17 @@ export class TeamProvisioningService { memberName: string, options: OpenCodeMemberInboxRelayOptions = {} ): Promise { + if (!this.canDeliverToOpenCodeRuntimeForTeam(teamName)) { + this.cleanupStoppedTeamOpenCodeRuntimeLanesInBackground(teamName); + return { + relayed: 0, + attempted: 0, + delivered: 0, + failed: 1, + lastDelivery: { delivered: false, reason: 'opencode_runtime_not_active' }, + diagnostics: ['opencode_runtime_not_active'], + }; + } const relayKey = this.getOpenCodeMemberRelayKey(teamName, memberName); const existing = this.openCodeMemberInboxRelayInFlight.get(relayKey); if (existing) { @@ -17874,48 +18147,113 @@ export class TeamProvisioningService { } catch { return []; } - const projectPath = config?.projectPath?.trim(); - if (!projectPath) { - return []; - } - - const projectDir = path.join(getProjectsBasePath(), extractBaseDir(encodePath(projectPath))); - let entries: fs.Dirent[]; - try { - entries = await fs.promises.readdir(projectDir, { withFileTypes: true }); - } catch { - return []; - } - const outcomes: BootstrapTranscriptOutcome[] = []; - const jsonlFiles = entries - .filter((entry) => entry.isFile() && entry.name.endsWith('.jsonl')) - .sort((left, right) => right.name.localeCompare(left.name)); + const projectDirs = await this.collectBootstrapTranscriptProjectDirs( + teamName, + memberName, + config + ); const contextMemberNames = [ memberName, ...((config?.members ?? []) .map((member) => member.name?.trim()) .filter((name): name is string => Boolean(name)) ?? []), ]; - for (const entry of jsonlFiles) { - if (config?.leadSessionId && entry.name === `${config.leadSessionId}.jsonl`) { + for (const projectDir of projectDirs) { + let entries: fs.Dirent[]; + try { + entries = await fs.promises.readdir(projectDir, { withFileTypes: true }); + } catch { continue; } - const outcome = await this.readRecentBootstrapTranscriptOutcome( - path.join(projectDir, entry.name), - sinceMs, - memberName, - teamName, - { contextMemberNames } - ); - if (outcome) { - outcomes.push(outcome); + + const jsonlFiles = entries + .filter((entry) => entry.isFile() && entry.name.endsWith('.jsonl')) + .sort((left, right) => right.name.localeCompare(left.name)); + for (const entry of jsonlFiles) { + if (config?.leadSessionId && entry.name === `${config.leadSessionId}.jsonl`) { + continue; + } + const outcome = await this.readRecentBootstrapTranscriptOutcome( + path.join(projectDir, entry.name), + sinceMs, + memberName, + teamName, + { contextMemberNames } + ); + if (outcome) { + outcomes.push(outcome); + } } } return outcomes; } + private async collectBootstrapTranscriptProjectDirs( + teamName: string, + memberName: string, + config: Awaited> + ): Promise { + const pathCandidates: string[] = []; + const pathSeen = new Set(); + const pushPath = (value: unknown): void => { + if (typeof value !== 'string') { + return; + } + let trimmed = value.trim(); + while (trimmed.endsWith('/') || trimmed.endsWith('\\')) { + trimmed = trimmed.slice(0, -1); + } + if (!trimmed || pathSeen.has(trimmed)) { + return; + } + pathSeen.add(trimmed); + pathCandidates.push(trimmed); + }; + + pushPath(config?.projectPath); + if (Array.isArray(config?.projectPathHistory)) { + for (let index = config.projectPathHistory.length - 1; index >= 0; index -= 1) { + pushPath(config.projectPathHistory[index]); + } + } + + const normalizedMemberName = memberName.trim().toLowerCase(); + const pushMatchingMemberCwd = (member: { name?: unknown; cwd?: unknown }): void => { + const candidateName = typeof member.name === 'string' ? member.name.trim().toLowerCase() : ''; + if (candidateName && matchesTeamMemberIdentity(candidateName, normalizedMemberName)) { + pushPath(member.cwd); + } + }; + for (const member of config?.members ?? []) { + pushMatchingMemberCwd(member); + } + + const metaMembers = await this.membersMetaStore.getMembers(teamName).catch(() => []); + for (const member of metaMembers) { + pushMatchingMemberCwd(member); + } + + const dirs: string[] = []; + const dirSeen = new Set(); + const pushDir = (dir: string): void => { + if (!dir || dirSeen.has(dir)) { + return; + } + dirSeen.add(dir); + dirs.push(dir); + }; + for (const projectPath of pathCandidates) { + const projectId = extractBaseDir(encodePath(projectPath)); + pushDir(path.join(getProjectsBasePath(), projectId)); + if (projectId.includes('_')) { + pushDir(path.join(getProjectsBasePath(), projectId.replace(/_/g, '-'))); + } + } + return dirs; + } + private selectLatestBootstrapTranscriptOutcome( outcomes: readonly BootstrapTranscriptOutcome[] ): BootstrapTranscriptOutcome | null { diff --git a/test/main/services/team/TeamAgentLaunchMatrix.safe-e2e.test.ts b/test/main/services/team/TeamAgentLaunchMatrix.safe-e2e.test.ts index c3f52c90..2ddce750 100644 --- a/test/main/services/team/TeamAgentLaunchMatrix.safe-e2e.test.ts +++ b/test/main/services/team/TeamAgentLaunchMatrix.safe-e2e.test.ts @@ -33,6 +33,7 @@ import { createPersistedLaunchSnapshot } from '../../../../src/main/services/tea import { getOpenCodeRuntimeLaneIndexPath, readOpenCodeRuntimeLaneIndex, + setOpenCodeRuntimeActiveRunManifest, upsertOpenCodeRuntimeLaneIndexEntry, } from '../../../../src/main/services/team/opencode/store/OpenCodeRuntimeManifestEvidenceReader'; @@ -11020,6 +11021,166 @@ describe('Team agent launch matrix safe e2e', () => { expect(adapter.messageInputs).toEqual([]); }); + it('does not scan or deliver orphaned mixed OpenCode lanes after app restart when team is stopped', async () => { + const teamName = 'mixed-opencode-stopped-orphaned-lane-safe-e2e'; + await writeMixedTeamConfig({ teamName, projectPath }); + await writeTeamMeta(teamName, projectPath); + await writeMembersMeta(teamName); + await upsertOpenCodeRuntimeLaneIndexEntry({ + teamsBasePath: getTeamsBasePath(), + teamName, + laneId: 'secondary:opencode:bob', + state: 'active', + diagnostics: ['orphaned lane from previous app session'], + }); + await setOpenCodeRuntimeActiveRunManifest({ + teamsBasePath: getTeamsBasePath(), + teamName, + laneId: 'secondary:opencode:bob', + runId: 'orphaned-opencode-run-bob', + }); + const inboxDir = path.join(getTeamsBasePath(), teamName, 'inboxes'); + await fs.mkdir(inboxDir, { recursive: true }); + await fs.writeFile( + path.join(inboxDir, 'bob.json'), + `${JSON.stringify( + [ + { + from: 'user', + to: 'bob', + text: 'must not be delivered while parent team is stopped', + timestamp: '2026-04-23T10:01:00.000Z', + read: false, + messageId: 'msg-stopped-orphaned-lane-bob', + }, + ], + null, + 2 + )}\n`, + 'utf8' + ); + const adapter = new FakeOpenCodeRuntimeAdapter('clean_success', { + bob: 'confirmed', + }); + const restartedService = new TeamProvisioningService(); + restartedService.setRuntimeAdapterRegistry(new TeamRuntimeAdapterRegistry([adapter])); + + await expect(restartedService.scanOpenCodePromptDeliveryWatchdog(teamName)).resolves.toBe(0); + expect(adapter.stopInputs).toHaveLength(1); + expect(adapter.stopInputs[0]).toMatchObject({ + runId: 'orphaned-opencode-run-bob', + teamName, + laneId: 'secondary:opencode:bob', + providerId: 'opencode', + reason: 'cleanup', + force: true, + }); + await expect(readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName)).resolves.toMatchObject({ + lanes: {}, + }); + await expect( + restartedService.relayOpenCodeMemberInboxMessages(teamName, 'bob') + ).resolves.toMatchObject({ + relayed: 0, + failed: 1, + lastDelivery: { delivered: false, reason: 'opencode_runtime_not_active' }, + }); + await expect( + restartedService.deliverOpenCodeMemberMessage(teamName, { + memberName: 'bob', + text: 'direct message must not reach orphaned lane', + messageId: 'msg-stopped-orphaned-direct-bob', + }) + ).resolves.toEqual({ + delivered: false, + reason: 'opencode_runtime_not_active', + }); + + expect(adapter.reconcileInputs).toEqual([]); + expect(adapter.messageInputs).toEqual([]); + }); + + it('does not recover missing mixed OpenCode lanes from persisted runtime evidence when parent team is stopped', async () => { + const teamName = 'mixed-opencode-stopped-missing-lane-recovery-safe-e2e'; + await writeMixedTeamConfig({ teamName, projectPath }); + await writeTeamMeta(teamName, projectPath); + await writeMembersMeta(teamName); + await fs.writeFile( + path.join(getTeamsBasePath(), teamName, 'launch-state.json'), + `${JSON.stringify( + createPersistedLaunchSnapshot({ + teamName, + expectedMembers: ['alice', 'bob'], + leadSessionId: 'lead-session', + launchPhase: 'reconciled', + members: { + alice: { + name: 'alice', + providerId: 'codex', + laneId: 'primary', + laneKind: 'primary', + laneOwnerProviderId: 'codex', + launchState: 'confirmed_alive', + agentToolAccepted: true, + runtimeAlive: true, + bootstrapConfirmed: true, + hardFailure: false, + lastEvaluatedAt: '2026-04-23T10:00:00.000Z', + }, + bob: { + name: 'bob', + providerId: 'opencode', + model: 'opencode/minimax-m2.5-free', + laneId: 'secondary:opencode:bob', + laneKind: 'secondary', + laneOwnerProviderId: 'opencode', + launchState: 'failed_to_start', + agentToolAccepted: true, + runtimeAlive: false, + bootstrapConfirmed: false, + hardFailure: true, + hardFailureReason: 'OpenCode bridge reported member launch failure', + runtimePid: 7743, + runtimeSessionId: 'ses_bob_materialized', + livenessKind: 'runtime_process_candidate', + pidSource: 'opencode_bridge', + lastEvaluatedAt: '2026-04-23T10:00:00.000Z', + }, + }, + updatedAt: '2026-04-23T10:00:00.000Z', + }), + null, + 2 + )}\n`, + 'utf8' + ); + const adapter = new FakeOpenCodeRuntimeAdapter('partial_pending', { + bob: 'launching', + }); + const restartedService = new TeamProvisioningService(); + restartedService.setRuntimeAdapterRegistry(new TeamRuntimeAdapterRegistry([adapter])); + + await expect( + readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName) + ).resolves.toMatchObject({ lanes: {} }); + await expect( + restartedService.deliverOpenCodeMemberMessage(teamName, { + memberName: 'bob', + text: 'must not recover stopped parent team', + messageId: 'msg-stopped-missing-lane-bob', + }) + ).resolves.toEqual({ + delivered: false, + reason: 'opencode_runtime_not_active', + }); + + expect(adapter.reconcileInputs).toEqual([]); + expect(adapter.messageInputs).toEqual([]); + await expect(readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName)).resolves.toMatchObject({ + lanes: {}, + }); + }); + it('recovers a missing mixed OpenCode lane index from materialized persisted runtime evidence before direct delivery', async () => { const teamName = 'mixed-opencode-direct-message-recovers-missing-lane-safe-e2e'; await writeMixedTeamConfig({ teamName, projectPath }); @@ -11093,6 +11254,7 @@ describe('Team agent launch matrix safe e2e', () => { bob: 'launching', tom: 'failed', }); + await writeAliveProcessRegistry(teamName); const restartedService = new TeamProvisioningService(); restartedService.setRuntimeAdapterRegistry(new TeamRuntimeAdapterRegistry([adapter])); @@ -11192,6 +11354,7 @@ describe('Team agent launch matrix safe e2e', () => { const adapter = new FakeOpenCodeRuntimeAdapter('clean_success', { bob: 'confirmed', }); + await writeAliveProcessRegistry(teamName); const restartedService = new TeamProvisioningService(); restartedService.setRuntimeAdapterRegistry(new TeamRuntimeAdapterRegistry([adapter])); @@ -11326,6 +11489,7 @@ describe('Team agent launch matrix safe e2e', () => { bob: 'launching', tom: 'failed', }); + await writeAliveProcessRegistry(teamName); const restartedService = new TeamProvisioningService(); restartedService.setRuntimeAdapterRegistry(new TeamRuntimeAdapterRegistry([adapter])); const scheduledWatchdogJobs: unknown[] = []; @@ -11466,6 +11630,7 @@ describe('Team agent launch matrix safe e2e', () => { bob: 'launching', tom: 'confirmed', }); + await writeAliveProcessRegistry(teamName); const restartedService = new TeamProvisioningService(); restartedService.setRuntimeAdapterRegistry(new TeamRuntimeAdapterRegistry([adapter])); const scheduledWatchdogJobs: unknown[] = []; @@ -17231,6 +17396,27 @@ function trackLiveRun(svc: TeamProvisioningService, run: any): void { (svc as any).aliveRunByTeam.set(run.teamName, run.runId); } +async function writeAliveProcessRegistry(teamName: string): Promise { + const teamDir = path.join(getTeamsBasePath(), teamName); + await fs.mkdir(teamDir, { recursive: true }); + await fs.writeFile( + path.join(teamDir, 'processes.json'), + `${JSON.stringify( + [ + { + id: 'lead-process', + label: 'Team Lead', + pid: process.pid, + registeredAt: '2026-04-23T10:00:00.000Z', + }, + ], + null, + 2 + )}\n`, + 'utf8' + ); +} + function expectDirectChildKillCount(actual: number, expected: number): void { // Windows uses taskkill.exe for process-tree termination, so fake child.kill is not called. expect(actual).toBe(process.platform === 'win32' ? 0 : expected); diff --git a/test/main/services/team/TeamProvisioningService.test.ts b/test/main/services/team/TeamProvisioningService.test.ts index 6553b523..af8e478b 100644 --- a/test/main/services/team/TeamProvisioningService.test.ts +++ b/test/main/services/team/TeamProvisioningService.test.ts @@ -275,6 +275,27 @@ function writeBootstrapState( ); } +function writeAliveProcessRegistry(teamName: string): void { + const teamDir = path.join(tempTeamsBase, teamName); + fs.mkdirSync(teamDir, { recursive: true }); + fs.writeFileSync( + path.join(teamDir, 'processes.json'), + `${JSON.stringify( + [ + { + id: 'lead-process', + label: 'Team Lead', + pid: process.pid, + registeredAt: '2026-04-23T10:00:00.000Z', + }, + ], + null, + 2 + )}\n`, + 'utf8' + ); +} + function writeTeamMeta(teamName: string, overrides: Record = {}): void { const teamDir = path.join(tempTeamsBase, teamName); fs.mkdirSync(teamDir, { recursive: true }); @@ -434,6 +455,7 @@ describe('TeamProvisioningService', () => { fs.mkdirSync(tempTeamsBase, { recursive: true }); fs.mkdirSync(tempTasksBase, { recursive: true }); fs.mkdirSync(tempProjectsBase, { recursive: true }); + writeAliveProcessRegistry('team-a'); }); afterEach(() => { @@ -3134,6 +3156,7 @@ describe('TeamProvisioningService', () => { ); (svc as any).getTrackedRunId = vi.fn(() => null); + (svc as any).canDeliverToOpenCodeRuntimeForTeam = vi.fn(() => true); (svc as any).resolveCurrentOpenCodeRuntimeRunId = vi.fn(async () => 'opencode-run-bob'); (svc as any).isOpenCodeRuntimeLaneIndexActive = vi.fn(async () => true); (svc as any).configReader = { @@ -4416,9 +4439,11 @@ describe('TeamProvisioningService', () => { sessionId: 'oc-session-bob', prePromptCursor: 'cursor-before', responseObservation: { - state: sendMessageToMember.mock.calls.length === 1 ? 'responded_non_visible_tool' : 'pending', + state: + sendMessageToMember.mock.calls.length === 1 ? 'responded_non_visible_tool' : 'pending', deliveredUserMessageId: 'oc-user-ask', - assistantMessageId: sendMessageToMember.mock.calls.length === 1 ? 'oc-assistant-read' : null, + assistantMessageId: + sendMessageToMember.mock.calls.length === 1 ? 'oc-assistant-read' : null, toolCallNames: sendMessageToMember.mock.calls.length === 1 ? ['read'] : [], visibleMessageToolCallId: null, visibleReplyMessageId: null, @@ -7909,9 +7934,7 @@ describe('TeamProvisioningService', () => { } ); - const config = JSON.parse( - fs.readFileSync(path.join(teamDir, 'config.json'), 'utf8') - ) as { + const config = JSON.parse(fs.readFileSync(path.join(teamDir, 'config.json'), 'utf8')) as { leadSessionId?: string; projectPath?: string; members: Array<{ @@ -7996,7 +8019,9 @@ describe('TeamProvisioningService', () => { }; }); - const { svc, membersMetaStore } = createSafeLaunchService({ memberWorktreeManager: worktreeManager }); + const { svc, membersMetaStore } = createSafeLaunchService({ + memberWorktreeManager: worktreeManager, + }); svc.setRuntimeAdapterRegistry( new TeamRuntimeAdapterRegistry([ { @@ -9934,6 +9959,49 @@ describe('TeamProvisioningService', () => { }); }); + it('clears registered-only stale failure when a verified runtime process appears later', async () => { + const svc = new TeamProvisioningService(); + (svc as any).getLiveTeamAgentRuntimeMetadata = vi.fn( + async () => + new Map([ + [ + 'tom', + { + alive: true, + model: 'gpt-5.4', + livenessKind: 'runtime_process', + runtimeDiagnostic: 'verified runtime process detected', + }, + ], + ]) + ); + + const result = await (svc as any).attachLiveRuntimeMetadataToStatuses('forge-labs-10', { + tom: createMemberSpawnStatusEntry({ + status: 'error', + launchState: 'failed_to_start', + error: 'registered runtime metadata without live process', + hardFailure: true, + hardFailureReason: 'registered runtime metadata without live process', + livenessKind: 'registered_only', + runtimeDiagnostic: 'registered runtime metadata without live process', + }), + }); + + expect(result.tom).toMatchObject({ + status: 'online', + launchState: 'runtime_pending_bootstrap', + runtimeAlive: true, + hardFailure: false, + hardFailureReason: undefined, + error: undefined, + runtimeModel: 'gpt-5.4', + livenessKind: 'runtime_process', + runtimeDiagnostic: 'verified runtime process detected', + livenessSource: 'process', + }); + }); + it('does not clear OpenCode bridge launch failure from process-only liveness', async () => { const svc = new TeamProvisioningService(); (svc as any).getLiveTeamAgentRuntimeMetadata = vi.fn( @@ -10622,6 +10690,85 @@ describe('TeamProvisioningService', () => { }); }); + it('confirms a teammate from bootstrap transcript stored under its worktree cwd', async () => { + const teamName = 'worktree-bootstrap-transcript-team'; + const leadSessionId = 'lead-session'; + const projectPath = '/Users/test/proj'; + const worktreePath = `${projectPath}/.claude/worktrees/team-${teamName}-tom-12345678`; + const acceptedAt = new Date(Date.now() - 90_000).toISOString(); + const observedAt = new Date(Date.now() - 30_000).toISOString(); + const teamDir = path.join(tempTeamsBase, teamName); + fs.mkdirSync(teamDir, { recursive: true }); + fs.writeFileSync( + path.join(teamDir, 'config.json'), + JSON.stringify({ + name: teamName, + projectPath, + leadSessionId, + members: [ + { name: 'team-lead', agentType: 'team-lead', cwd: projectPath }, + { name: 'tom', providerId: 'codex', model: 'gpt-5.4', cwd: worktreePath }, + { name: 'bob', providerId: 'codex', model: 'gpt-5.4-mini', cwd: projectPath }, + ], + }), + 'utf8' + ); + writeLaunchState(teamName, leadSessionId, { + tom: { + providerId: 'codex', + laneId: 'primary', + laneKind: 'primary', + laneOwnerProviderId: 'codex', + launchState: 'failed_to_start', + agentToolAccepted: true, + runtimeAlive: false, + bootstrapConfirmed: false, + hardFailure: true, + hardFailureReason: 'registered runtime metadata without live process', + firstSpawnAcceptedAt: acceptedAt, + lastEvaluatedAt: acceptedAt, + }, + }); + const worktreeProjectDir = path.join(tempProjectsBase, encodePath(worktreePath)); + fs.mkdirSync(worktreeProjectDir, { recursive: true }); + fs.writeFileSync( + path.join(worktreeProjectDir, 'tom-session.jsonl'), + `${JSON.stringify({ + type: 'user', + teamName, + agentName: 'tom', + timestamp: observedAt, + cwd: worktreePath, + message: { + role: 'user', + content: [ + { + type: 'tool_result', + tool_use_id: 'item_0', + content: `Member briefing for tom on team "${teamName}" (${teamName}).\nRole: developer.`, + }, + ], + }, + })}\n`, + 'utf8' + ); + + const svc = new TeamProvisioningService(); + (svc as any).getLiveTeamAgentRuntimeMetadata = vi.fn(async () => new Map()); + const result = await svc.getMemberSpawnStatuses(teamName); + + expect(result.statuses.tom).toMatchObject({ + status: 'online', + launchState: 'confirmed_alive', + agentToolAccepted: true, + runtimeAlive: false, + bootstrapConfirmed: true, + hardFailure: false, + hardFailureReason: undefined, + lastHeartbeatAt: observedAt, + }); + }); + it('treats suffixed persisted heartbeat senders as the expected member during reconcile', async () => { const teamName = 'suffixed-heartbeat-reconcile-team'; const svc = new TeamProvisioningService();