diff --git a/src/main/services/team/TeamProvisioningService.ts b/src/main/services/team/TeamProvisioningService.ts index 4466bfbc..49b368a4 100644 --- a/src/main/services/team/TeamProvisioningService.ts +++ b/src/main/services/team/TeamProvisioningService.ts @@ -1595,6 +1595,84 @@ function isDefinitiveOpenCodePreLaunchFailure( ); } +function hasOpenCodeRuntimeHandle( + value: + | Pick + | Pick + | undefined +): boolean { + if (!value) { + return false; + } + const runtimePid = + typeof value.runtimePid === 'number' && + Number.isFinite(value.runtimePid) && + value.runtimePid > 0; + const runtimeSessionId = (value as { runtimeSessionId?: unknown }).runtimeSessionId; + const runtimeEvidenceSessionId = (value as { sessionId?: unknown }).sessionId; + const sessionId = + (typeof runtimeSessionId === 'string' && runtimeSessionId.trim().length > 0) || + (typeof runtimeEvidenceSessionId === 'string' && runtimeEvidenceSessionId.trim().length > 0); + return runtimePid || sessionId; +} + +function hasOpenCodeRuntimeLivenessMarker( + value: Pick | undefined +): boolean { + return ( + value?.livenessKind === 'runtime_process' || + value?.livenessKind === 'runtime_process_candidate' || + value?.livenessKind === 'permission_blocked' + ); +} + +function isRecoverablePersistedOpenCodeRuntimeCandidate( + member: PersistedTeamLaunchMemberState | undefined | null +): boolean { + if (!member || member.skippedForLaunch) { + return false; + } + if ( + member.providerId !== 'opencode' || + member.laneKind !== 'secondary' || + member.laneOwnerProviderId !== 'opencode' || + typeof member.laneId !== 'string' || + member.laneId.trim().length === 0 + ) { + return false; + } + const hasPendingPermission = (member.pendingPermissionRequestIds?.length ?? 0) > 0; + return ( + member.agentToolAccepted === true && (hasOpenCodeRuntimeHandle(member) || hasPendingPermission) + ); +} + +function isRecoverablePersistedOpenCodeTerminalRuntimeCandidate( + member: PersistedTeamLaunchMemberState | undefined | null +): boolean { + return ( + isRecoverablePersistedOpenCodeRuntimeCandidate(member) && + member?.launchState === 'failed_to_start' && + member.hardFailure === true && + hasOpenCodeRuntimeHandle(member) + ); +} + +function isRecoverableOpenCodeRuntimeEvidence( + evidence: TeamRuntimeMemberLaunchEvidence | undefined | null +): evidence is TeamRuntimeMemberLaunchEvidence { + if (!evidence) { + return false; + } + return ( + evidence.runtimeAlive === true || + evidence.bootstrapConfirmed === true || + (evidence.pendingPermissionRequestIds?.length ?? 0) > 0 || + hasOpenCodeRuntimeHandle(evidence) || + (evidence.agentToolAccepted === true && hasOpenCodeRuntimeLivenessMarker(evidence)) + ); +} + function isLaunchGraceWindowFailureReason(reason?: string): boolean { return reason?.trim() === 'Teammate did not join within the launch grace window.'; } @@ -5161,12 +5239,16 @@ export class TeamProvisioningService { if (!laneIndex) { return 0; } - return await this.scanOpenCodePromptDeliveryWatchdogForActiveLanes( - teamName, - Object.values(laneIndex.lanes) - .filter((lane) => lane.state === 'active') - .map((lane) => lane.laneId) - ); + let activeLaneIds = Object.values(laneIndex.lanes) + .filter((lane) => lane.state === 'active') + .map((lane) => lane.laneId); + activeLaneIds = [ + ...new Set([ + ...activeLaneIds, + ...(await this.tryRecoverOpenCodeRuntimeLanesForDeliveryWatchdog(teamName)), + ]), + ]; + return await this.scanOpenCodePromptDeliveryWatchdogForActiveLanes(teamName, activeLaneIds); } private async scanOpenCodePromptDeliveryWatchdogForActiveLanes( @@ -5392,7 +5474,7 @@ export class TeamProvisioningService { return { delivered: false, reason: 'opencode_runtime_not_active' }; } } - const runtimeRunId = + let runtimeRunId = laneIdentity.laneKind === 'secondary' && laneIdentity.laneOwnerProviderId === 'opencode' ? (liveSecondaryLaneRunId ?? (await this.resolveCurrentOpenCodeRuntimeRunId(teamName, laneIdentity.laneId))) @@ -5411,6 +5493,33 @@ export class TeamProvisioningService { } runtimeActive = await this.isOpenCodeRuntimeLaneIndexActive(teamName, laneIdentity.laneId); } + if ( + !runtimeActive && + laneIdentity.laneKind === 'secondary' && + laneIdentity.laneOwnerProviderId === 'opencode' + ) { + const recovered = await this.tryRecoverOpenCodeRuntimeLaneBeforeDelivery({ + teamName, + laneId: laneIdentity.laneId, + member: { + ...(configMember ?? {}), + ...(metaMember ?? {}), + name: canonicalMemberName, + providerId: 'opencode', + model: metaMember?.model ?? configMember?.model, + role: metaMember?.role ?? configMember?.role, + workflow: metaMember?.workflow ?? configMember?.workflow, + effort: metaMember?.effort ?? configMember?.effort, + cwd: memberRuntimeCwd || undefined, + isolation: metaMember?.isolation ?? configMember?.isolation, + }, + projectPath: config?.projectPath?.trim() || this.readPersistedTeamProjectPath(teamName), + }); + if (recovered) { + runtimeRunId = await this.resolveCurrentOpenCodeRuntimeRunId(teamName, laneIdentity.laneId); + runtimeActive = true; + } + } if (!runtimeActive) { return { delivered: false, reason: 'opencode_runtime_not_active' }; } @@ -6258,6 +6367,108 @@ export class TeamProvisioningService { return laneIndex?.lanes[laneId]?.state === 'active'; } + private async tryRecoverOpenCodeRuntimeLaneBeforeDelivery(input: { + teamName: string; + laneId: string; + member: TeamMember; + projectPath: string | null; + }): Promise { + const snapshot = await this.launchStateStore.read(input.teamName).catch(() => null); + const persistedMember = + snapshot?.members?.[input.member.name] ?? + Object.values(snapshot?.members ?? {}).find((member) => member.laneId === input.laneId); + if (!persistedMember || !isRecoverablePersistedOpenCodeRuntimeCandidate(persistedMember)) { + return false; + } + const runtimeEvidence = await this.tryRecoverMissingOpenCodeSecondaryLaneFromRuntime({ + teamName: input.teamName, + laneId: input.laneId, + member: input.member, + projectPath: input.projectPath, + previousLaunchState: snapshot, + persistedMember, + }); + if (!runtimeEvidence) { + return false; + } + logger.info( + `[${input.teamName}] Recovered OpenCode lane ${input.laneId} before message delivery.` + ); + return true; + } + + private async tryRecoverOpenCodeRuntimeLanesForDeliveryWatchdog( + teamName: string + ): Promise { + const snapshot = await this.launchStateStore.read(teamName).catch(() => null); + const candidates = Object.values(snapshot?.members ?? {}).filter( + isRecoverablePersistedOpenCodeRuntimeCandidate + ); + if (candidates.length === 0) { + return []; + } + + const [config, teamMeta, metaMembers, currentLaneIndex] = await Promise.all([ + this.configReader.getConfig(teamName).catch(() => null), + this.teamMetaStore.getMeta(teamName).catch(() => null), + this.membersMetaStore.getMembers(teamName).catch(() => []), + readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName).catch(() => null), + ]); + const projectPath = config?.projectPath?.trim() || this.readPersistedTeamProjectPath(teamName); + const leadMember = config?.members?.find((member) => isLeadMember(member)); + const leadProviderId = + normalizeOptionalTeamProviderId(teamMeta?.launchIdentity?.providerId) ?? + normalizeOptionalTeamProviderId(teamMeta?.providerId) ?? + normalizeOptionalTeamProviderId(leadMember?.providerId); + const recoveredLaneIds: string[] = []; + for (const persistedMember of candidates) { + const memberName = persistedMember.name.trim(); + const configMember = config?.members?.find( + (member) => member.name?.trim().toLowerCase() === memberName.toLowerCase() + ); + const metaMember = metaMembers.find( + (member) => member.name?.trim().toLowerCase() === memberName.toLowerCase() + ); + if (metaMember?.removedAt != null || configMember?.removedAt != null) { + continue; + } + const laneIdentity = buildPlannedMemberLaneIdentity({ + leadProviderId, + member: { + name: memberName, + providerId: 'opencode', + }, + }); + if (laneIdentity.laneId !== persistedMember.laneId) { + continue; + } + if (currentLaneIndex?.lanes[laneIdentity.laneId]) { + continue; + } + const recovered = await this.tryRecoverOpenCodeRuntimeLaneBeforeDelivery({ + teamName, + laneId: laneIdentity.laneId, + member: { + ...(configMember ?? {}), + ...(metaMember ?? {}), + name: memberName, + providerId: 'opencode', + model: metaMember?.model ?? configMember?.model ?? persistedMember.model, + role: metaMember?.role ?? configMember?.role, + workflow: metaMember?.workflow ?? configMember?.workflow, + effort: metaMember?.effort ?? configMember?.effort ?? persistedMember.effort, + cwd: metaMember?.cwd ?? configMember?.cwd ?? persistedMember.cwd, + isolation: metaMember?.isolation ?? configMember?.isolation, + }, + projectPath, + }); + if (recovered) { + recoveredLaneIds.push(laneIdentity.laneId); + } + } + return [...new Set(recoveredLaneIds)]; + } + private async resolveOpenCodeRuntimeLaneId(params: { teamName: string; runId: string; @@ -16240,6 +16451,13 @@ export class TeamProvisioningService { private shouldRecoverStalePersistedMixedLaunchSnapshot( snapshot: PersistedTeamLaunchSnapshot ): boolean { + const hasRecoverableOpenCodeRuntimeCandidate = Object.values(snapshot.members).some((member) => + isRecoverablePersistedOpenCodeTerminalRuntimeCandidate(member) + ); + if (hasRecoverableOpenCodeRuntimeCandidate) { + return true; + } + if (snapshot.teamLaunchState !== 'partial_pending') { return false; } @@ -16692,6 +16910,49 @@ export class TeamProvisioningService { } let laneEntry = laneIndex.lanes[laneIdentity.laneId]; + const persistedMember = + persistedSnapshot?.members?.[member.name] ?? bootstrapSnapshot?.members?.[member.name]; + if ( + !laneEntry && + persistedMember && + isRecoverablePersistedOpenCodeRuntimeCandidate(persistedMember) && + persistedMember.laneId === laneIdentity.laneId + ) { + const runtimeEvidence = await this.tryRecoverMissingOpenCodeSecondaryLaneFromRuntime({ + teamName, + laneId: laneIdentity.laneId, + member, + projectPath, + previousLaunchState: persistedSnapshot ?? bootstrapSnapshot, + persistedMember, + }); + if (runtimeEvidence) { + recoveredAny = true; + secondaryMembers.push({ + laneId: laneIdentity.laneId, + member, + leadDefaults, + evidence: { + launchState: runtimeEvidence.launchState, + agentToolAccepted: runtimeEvidence.agentToolAccepted, + runtimeAlive: runtimeEvidence.runtimeAlive, + bootstrapConfirmed: runtimeEvidence.bootstrapConfirmed, + hardFailure: runtimeEvidence.hardFailure, + hardFailureReason: runtimeEvidence.hardFailureReason, + pendingPermissionRequestIds: runtimeEvidence.pendingPermissionRequestIds, + runtimePid: runtimeEvidence.runtimePid, + sessionId: runtimeEvidence.sessionId, + runtimeSessionId: runtimeEvidence.sessionId, + livenessKind: runtimeEvidence.livenessKind, + pidSource: runtimeEvidence.pidSource, + runtimeDiagnostic: runtimeEvidence.runtimeDiagnostic, + runtimeDiagnosticSeverity: runtimeEvidence.runtimeDiagnosticSeverity, + diagnostics: runtimeEvidence.diagnostics, + }, + }); + continue; + } + } if (laneEntry?.state === 'active') { const runtimeEvidence = await this.tryRecoverActiveOpenCodeSecondaryLaneFromRuntime({ teamName, @@ -16844,6 +17105,61 @@ export class TeamProvisioningService { } } + private async tryRecoverMissingOpenCodeSecondaryLaneFromRuntime(params: { + teamName: string; + laneId: string; + member: TeamMember; + projectPath: string | null; + previousLaunchState: PersistedTeamLaunchSnapshot | null; + persistedMember: PersistedTeamLaunchMemberState; + }): Promise { + const currentLaneIndex = await readOpenCodeRuntimeLaneIndex( + getTeamsBasePath(), + params.teamName + ).catch(() => null); + const currentEntry = currentLaneIndex?.lanes[params.laneId]; + if (currentEntry?.state === 'degraded' || currentEntry?.state === 'stopped') { + return null; + } + if (!isRecoverablePersistedOpenCodeRuntimeCandidate(params.persistedMember)) { + return null; + } + + const runtimeEvidence = await this.tryRecoverActiveOpenCodeSecondaryLaneFromRuntime({ + teamName: params.teamName, + laneId: params.laneId, + member: params.member, + projectPath: params.projectPath, + previousLaunchState: params.previousLaunchState, + }); + if (!isRecoverableOpenCodeRuntimeEvidence(runtimeEvidence)) { + return null; + } + + const diagnostics = Array.from( + new Set([ + 'Recovered missing OpenCode runtime lane index from persisted runtime evidence.', + ...(runtimeEvidence.diagnostics ?? []), + ]) + ); + await upsertOpenCodeRuntimeLaneIndexEntry({ + teamsBasePath: getTeamsBasePath(), + teamName: params.teamName, + laneId: params.laneId, + state: 'active', + diagnostics, + }).catch((error: unknown) => { + logger.warn( + `[${params.teamName}] Failed to recover missing OpenCode lane index ${params.laneId}: ${getErrorMessage(error)}` + ); + }); + + return { + ...runtimeEvidence, + diagnostics, + }; + } + private async readLeadInboxMessagesForLaunchReconcile( teamName: string, leadName: string diff --git a/src/main/services/team/fileLock.ts b/src/main/services/team/fileLock.ts index 9f95fdb1..31bcf28b 100644 --- a/src/main/services/team/fileLock.ts +++ b/src/main/services/team/fileLock.ts @@ -5,18 +5,63 @@ const STALE_TIMEOUT_MS = 30_000; const ACQUIRE_TIMEOUT_MS = 5_000; const RETRY_INTERVAL_MS = 20; -function readLockAge(lockPath: string): number | null { +export interface FileLockOptions { + acquireTimeoutMs?: number; + staleTimeoutMs?: number; + retryIntervalMs?: number; +} + +interface LockInfo { + pid: number | null; + ageMs: number | null; +} + +function readLockInfo(lockPath: string): LockInfo { + let pid: number | null = null; + let ageMs: number | null = null; try { const content = fs.readFileSync(lockPath, 'utf8'); - const ts = parseInt(content.split('\n')[1] ?? '', 10); - if (Number.isFinite(ts)) return Date.now() - ts; + const lines = content.split('\n'); + const parsedPid = parseInt(lines[0] ?? '', 10); + if (Number.isFinite(parsedPid) && parsedPid > 0) { + pid = parsedPid; + } + const ts = parseInt(lines[1] ?? '', 10); + if (Number.isFinite(ts)) { + ageMs = Date.now() - ts; + } } catch { /* lock may have been released concurrently */ } - return null; + if (ageMs === null) { + try { + ageMs = Date.now() - fs.statSync(lockPath).mtimeMs; + } catch { + /* lock may have been released concurrently */ + } + } + return { pid, ageMs }; } -function tryAcquire(lockPath: string): boolean { +function isProcessAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + return code !== 'ESRCH'; + } +} + +function shouldBreakExistingLock(lockPath: string, staleTimeoutMs: number): boolean { + const info = readLockInfo(lockPath); + if (info.pid !== null && !isProcessAlive(info.pid)) { + return true; + } + return info.ageMs !== null && info.ageMs > staleTimeoutMs; +} + +function tryAcquire(lockPath: string, options: Required): boolean { try { const dir = path.dirname(lockPath); if (!fs.existsSync(dir)) { @@ -28,8 +73,7 @@ function tryAcquire(lockPath: string): boolean { return true; } catch (err) { if ((err as NodeJS.ErrnoException).code === 'EEXIST') { - const age = readLockAge(lockPath); - if (age !== null && age > STALE_TIMEOUT_MS) { + if (shouldBreakExistingLock(lockPath, options.staleTimeoutMs)) { try { fs.unlinkSync(lockPath); } catch { @@ -50,15 +94,24 @@ function releaseLock(lockPath: string): void { } } -export async function withFileLock(filePath: string, fn: () => Promise): Promise { +export async function withFileLock( + filePath: string, + fn: () => Promise, + options: FileLockOptions = {} +): Promise { + const resolvedOptions = { + acquireTimeoutMs: options.acquireTimeoutMs ?? ACQUIRE_TIMEOUT_MS, + staleTimeoutMs: options.staleTimeoutMs ?? STALE_TIMEOUT_MS, + retryIntervalMs: options.retryIntervalMs ?? RETRY_INTERVAL_MS, + }; const lockPath = `${filePath}.lock`; - const deadline = Date.now() + ACQUIRE_TIMEOUT_MS; + const deadline = Date.now() + resolvedOptions.acquireTimeoutMs; - while (!tryAcquire(lockPath)) { + while (!tryAcquire(lockPath, resolvedOptions)) { if (Date.now() >= deadline) { throw new Error(`File lock timeout: ${filePath}`); } - await new Promise((resolve) => setTimeout(resolve, RETRY_INTERVAL_MS)); + await new Promise((resolve) => setTimeout(resolve, resolvedOptions.retryIntervalMs)); } try { diff --git a/src/main/services/team/opencode/store/OpenCodeRuntimeManifestEvidenceReader.ts b/src/main/services/team/opencode/store/OpenCodeRuntimeManifestEvidenceReader.ts index c5c28e51..8939a180 100644 --- a/src/main/services/team/opencode/store/OpenCodeRuntimeManifestEvidenceReader.ts +++ b/src/main/services/team/opencode/store/OpenCodeRuntimeManifestEvidenceReader.ts @@ -28,6 +28,11 @@ const OPENCODE_TEAM_RUNTIME_LANES_DIR = 'lanes'; const OPENCODE_TEAM_RUNTIME_LANES_INDEX_FILE = 'lanes.json'; const OPENCODE_RUNTIME_MANIFEST_FILE = 'manifest.json'; const OPENCODE_RUNTIME_RUN_TOMBSTONES_FILE = 'opencode-run-tombstones.json'; +const OPENCODE_LANE_INDEX_LOCK_OPTIONS = { + acquireTimeoutMs: 30_000, + staleTimeoutMs: 25_000, + retryIntervalMs: 25, +} as const; export interface OpenCodeRuntimeLaneIndexEntry { laneId: string; @@ -362,9 +367,13 @@ export async function writeOpenCodeRuntimeLaneIndex( index: OpenCodeRuntimeLaneIndex ): Promise { const filePath = getOpenCodeRuntimeLaneIndexPath(teamsBasePath, teamName); - await withFileLock(filePath, async () => { - await writeOpenCodeRuntimeLaneIndexUnlocked(teamsBasePath, teamName, index); - }); + await withFileLock( + filePath, + async () => { + await writeOpenCodeRuntimeLaneIndexUnlocked(teamsBasePath, teamName, index); + }, + OPENCODE_LANE_INDEX_LOCK_OPTIONS + ); } export async function upsertOpenCodeRuntimeLaneIndexEntry(params: { @@ -375,17 +384,24 @@ export async function upsertOpenCodeRuntimeLaneIndexEntry(params: { diagnostics?: string[]; }): Promise { const filePath = getOpenCodeRuntimeLaneIndexPath(params.teamsBasePath, params.teamName); - await withFileLock(filePath, async () => { - const index = await readOpenCodeRuntimeLaneIndexUnlocked(params.teamsBasePath, params.teamName); - index.updatedAt = new Date().toISOString(); - index.lanes[params.laneId] = { - laneId: params.laneId, - state: params.state, - updatedAt: index.updatedAt, - diagnostics: params.diagnostics?.length ? [...params.diagnostics] : undefined, - }; - await writeOpenCodeRuntimeLaneIndexUnlocked(params.teamsBasePath, params.teamName, index); - }); + await withFileLock( + filePath, + async () => { + const index = await readOpenCodeRuntimeLaneIndexUnlocked( + params.teamsBasePath, + params.teamName + ); + index.updatedAt = new Date().toISOString(); + index.lanes[params.laneId] = { + laneId: params.laneId, + state: params.state, + updatedAt: index.updatedAt, + diagnostics: params.diagnostics?.length ? [...params.diagnostics] : undefined, + }; + await writeOpenCodeRuntimeLaneIndexUnlocked(params.teamsBasePath, params.teamName, index); + }, + OPENCODE_LANE_INDEX_LOCK_OPTIONS + ); } export async function setOpenCodeRuntimeActiveRunManifest(params: { @@ -463,15 +479,22 @@ export async function removeOpenCodeRuntimeLaneIndexEntry(params: { laneId: string; }): Promise { const filePath = getOpenCodeRuntimeLaneIndexPath(params.teamsBasePath, params.teamName); - await withFileLock(filePath, async () => { - const index = await readOpenCodeRuntimeLaneIndexUnlocked(params.teamsBasePath, params.teamName); - if (!index.lanes[params.laneId]) { - return; - } - delete index.lanes[params.laneId]; - index.updatedAt = new Date().toISOString(); - await writeOpenCodeRuntimeLaneIndexUnlocked(params.teamsBasePath, params.teamName, index); - }); + await withFileLock( + filePath, + async () => { + const index = await readOpenCodeRuntimeLaneIndexUnlocked( + params.teamsBasePath, + params.teamName + ); + if (!index.lanes[params.laneId]) { + return; + } + delete index.lanes[params.laneId]; + index.updatedAt = new Date().toISOString(); + await writeOpenCodeRuntimeLaneIndexUnlocked(params.teamsBasePath, params.teamName, index); + }, + OPENCODE_LANE_INDEX_LOCK_OPTIONS + ); } export async function clearOpenCodeRuntimeLaneStorage(params: { diff --git a/test/main/services/team/TeamAgentLaunchMatrix.safe-e2e.test.ts b/test/main/services/team/TeamAgentLaunchMatrix.safe-e2e.test.ts index 9d5e3348..ee913d69 100644 --- a/test/main/services/team/TeamAgentLaunchMatrix.safe-e2e.test.ts +++ b/test/main/services/team/TeamAgentLaunchMatrix.safe-e2e.test.ts @@ -2,7 +2,7 @@ import { promises as fs } from 'fs'; import * as os from 'os'; import * as path from 'path'; -import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { TeamProvisioningService } from '../../../../src/main/services/team/TeamProvisioningService'; import type { @@ -31,6 +31,7 @@ import { } from '../../../../src/main/utils/pathDecoder'; import { createPersistedLaunchSnapshot } from '../../../../src/main/services/team/TeamLaunchStateEvaluator'; import { + getOpenCodeRuntimeLaneIndexPath, readOpenCodeRuntimeLaneIndex, upsertOpenCodeRuntimeLaneIndexEntry, } from '../../../../src/main/services/team/opencode/store/OpenCodeRuntimeManifestEvidenceReader'; @@ -2004,6 +2005,57 @@ describe('Team agent launch matrix safe e2e', () => { ]); }); + it('launches mixed OpenCode lanes after a fresh abandoned lane index lock', async () => { + const teamName = 'mixed-opencode-abandoned-lane-lock-safe-e2e'; + await writeMixedTeamConfig({ teamName, projectPath }); + await writeTeamMeta(teamName, projectPath); + await writeMembersMeta(teamName); + const lockPath = `${getOpenCodeRuntimeLaneIndexPath(getTeamsBasePath(), teamName)}.lock`; + const abandonedPid = 424_242; + await fs.mkdir(path.dirname(lockPath), { recursive: true }); + await fs.writeFile(lockPath, `${abandonedPid}\n${Date.now()}\n`, 'utf8'); + const killSpy = vi.spyOn(process, 'kill').mockImplementation(((pid: number | string) => { + if (pid === abandonedPid) { + const error = new Error('process is gone') as NodeJS.ErrnoException; + error.code = 'ESRCH'; + throw error; + } + return true; + }) as typeof process.kill); + const adapter = new FakeOpenCodeRuntimeAdapter('clean_success', { + bob: 'confirmed', + tom: 'confirmed', + }); + const svc = new TeamProvisioningService(); + svc.setRuntimeAdapterRegistry(new TeamRuntimeAdapterRegistry([adapter])); + const run = createMixedLiveRun({ teamName, projectPath }); + run.child = { kill: () => undefined }; + trackLiveRun(svc, run); + + try { + await (svc as any).launchMixedSecondaryLaneIfNeeded(run); + } finally { + killSpy.mockRestore(); + } + + await waitForCondition(() => adapter.launchInputs.length === 2); + await expect(readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName)).resolves.toMatchObject( + { + lanes: { + 'secondary:opencode:bob': { state: 'active' }, + 'secondary:opencode:tom': { state: 'active' }, + }, + } + ); + await waitForCondition(() => + run.mixedSecondaryLanes.every((lane: { state: string }) => lane.state === 'finished') + ); + expect(run.mixedSecondaryLanes.map((lane: { state: string }) => lane.state)).toEqual([ + 'finished', + 'finished', + ]); + }); + it('stopAllTeams stops in-flight mixed OpenCode secondary lanes without late failure degrading launch state', async () => { const teamName = 'mixed-opencode-stop-all-inflight-safe-e2e'; await writeMixedTeamConfig({ teamName, projectPath }); @@ -10434,6 +10486,7 @@ describe('Team agent launch matrix safe e2e', () => { text: 'message recovered pure opencode lane', messageId: 'msg-recovered-pure-opencode', }); + expect(messageAdapter.messageInputs[0]?.runId).toBe(launchAdapter.launchInputs[0]?.runId); }); it('delivers direct OpenCode member messages to recovered pure OpenCode lanes despite stale terminal provisioning state', async () => { @@ -10967,6 +11020,565 @@ describe('Team agent launch matrix safe e2e', () => { expect(adapter.messageInputs).toEqual([]); }); + it('recovers a missing mixed OpenCode lane index from materialized persisted runtime evidence before direct delivery', async () => { + const teamName = 'mixed-opencode-direct-message-recovers-missing-lane-safe-e2e'; + await writeMixedTeamConfig({ teamName, projectPath }); + await writeTeamMeta(teamName, projectPath); + await writeMembersMeta(teamName); + await fs.writeFile( + path.join(getTeamsBasePath(), teamName, 'launch-state.json'), + `${JSON.stringify( + createPersistedLaunchSnapshot({ + teamName, + expectedMembers: ['alice', 'bob', 'tom'], + leadSessionId: 'lead-session', + launchPhase: 'reconciled', + members: { + alice: { + name: 'alice', + providerId: 'codex', + laneId: 'primary', + laneKind: 'primary', + laneOwnerProviderId: 'codex', + launchState: 'confirmed_alive', + agentToolAccepted: true, + runtimeAlive: true, + bootstrapConfirmed: true, + hardFailure: false, + lastEvaluatedAt: '2026-04-23T10:00:00.000Z', + }, + bob: { + name: 'bob', + providerId: 'opencode', + model: 'opencode/minimax-m2.5-free', + laneId: 'secondary:opencode:bob', + laneKind: 'secondary', + laneOwnerProviderId: 'opencode', + launchState: 'failed_to_start', + agentToolAccepted: true, + runtimeAlive: false, + bootstrapConfirmed: false, + hardFailure: true, + hardFailureReason: 'OpenCode bridge reported member launch failure', + runtimePid: 7743, + runtimeSessionId: 'ses_bob_materialized', + livenessKind: 'runtime_process_candidate', + pidSource: 'opencode_bridge', + lastEvaluatedAt: '2026-04-23T10:00:00.000Z', + }, + tom: { + name: 'tom', + providerId: 'opencode', + model: 'opencode/nemotron-3-super-free', + laneId: 'secondary:opencode:tom', + laneKind: 'secondary', + laneOwnerProviderId: 'opencode', + launchState: 'failed_to_start', + agentToolAccepted: false, + runtimeAlive: false, + bootstrapConfirmed: false, + hardFailure: true, + hardFailureReason: 'OpenCode bridge reported member launch failure', + lastEvaluatedAt: '2026-04-23T10:00:00.000Z', + }, + }, + updatedAt: '2026-04-23T10:00:00.000Z', + }), + null, + 2 + )}\n`, + 'utf8' + ); + const adapter = new FakeOpenCodeRuntimeAdapter('partial_pending', { + bob: 'launching', + tom: 'failed', + }); + const restartedService = new TeamProvisioningService(); + restartedService.setRuntimeAdapterRegistry(new TeamRuntimeAdapterRegistry([adapter])); + + await expect( + readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName) + ).resolves.toMatchObject({ lanes: {} }); + await expect( + restartedService.deliverOpenCodeMemberMessage(teamName, { + memberName: 'bob', + text: 'recovered bob receives direct message', + messageId: 'msg-recovered-missing-lane-bob', + }) + ).resolves.toEqual({ + delivered: true, + diagnostics: [], + }); + + expect(adapter.reconcileInputs).toHaveLength(1); + expect(adapter.reconcileInputs[0]).toMatchObject({ + teamName, + laneId: 'secondary:opencode:bob', + reason: 'startup_recovery', + }); + await expect(readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName)).resolves.toMatchObject( + { + lanes: { + 'secondary:opencode:bob': { + state: 'active', + }, + }, + } + ); + expect(adapter.messageInputs).toHaveLength(1); + expect(adapter.messageInputs[0]).toMatchObject({ + teamName, + laneId: 'secondary:opencode:bob', + memberName: 'bob', + cwd: projectPath, + text: 'recovered bob receives direct message', + messageId: 'msg-recovered-missing-lane-bob', + }); + }); + + it('recovers a missing mixed OpenCode lane index from confirmed-alive persisted runtime evidence before direct delivery', async () => { + const teamName = 'mixed-opencode-direct-message-recovers-confirmed-missing-lane-safe-e2e'; + await writeMixedTeamConfig({ teamName, projectPath }); + await writeTeamMeta(teamName, projectPath); + await writeMembersMeta(teamName); + await fs.writeFile( + path.join(getTeamsBasePath(), teamName, 'launch-state.json'), + `${JSON.stringify( + createPersistedLaunchSnapshot({ + teamName, + expectedMembers: ['alice', 'bob'], + leadSessionId: 'lead-session', + launchPhase: 'reconciled', + members: { + alice: { + name: 'alice', + providerId: 'codex', + laneId: 'primary', + laneKind: 'primary', + laneOwnerProviderId: 'codex', + launchState: 'confirmed_alive', + agentToolAccepted: true, + runtimeAlive: true, + bootstrapConfirmed: true, + hardFailure: false, + lastEvaluatedAt: '2026-04-23T10:00:00.000Z', + }, + bob: { + name: 'bob', + providerId: 'opencode', + model: 'opencode/minimax-m2.5-free', + laneId: 'secondary:opencode:bob', + laneKind: 'secondary', + laneOwnerProviderId: 'opencode', + launchState: 'confirmed_alive', + agentToolAccepted: true, + runtimeAlive: true, + bootstrapConfirmed: true, + hardFailure: false, + runtimePid: 7743, + runtimeSessionId: 'ses_bob_confirmed_materialized', + livenessKind: 'runtime_process', + pidSource: 'opencode_bridge', + lastEvaluatedAt: '2026-04-23T10:00:00.000Z', + }, + }, + updatedAt: '2026-04-23T10:00:00.000Z', + }), + null, + 2 + )}\n`, + 'utf8' + ); + const adapter = new FakeOpenCodeRuntimeAdapter('clean_success', { + bob: 'confirmed', + }); + const restartedService = new TeamProvisioningService(); + restartedService.setRuntimeAdapterRegistry(new TeamRuntimeAdapterRegistry([adapter])); + + await expect( + readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName) + ).resolves.toMatchObject({ lanes: {} }); + await expect( + restartedService.deliverOpenCodeMemberMessage(teamName, { + memberName: 'bob', + text: 'confirmed alive missing lane recovers', + messageId: 'msg-recovered-confirmed-missing-lane-bob', + }) + ).resolves.toEqual({ + delivered: true, + diagnostics: [], + }); + + expect(adapter.reconcileInputs).toHaveLength(1); + expect(adapter.reconcileInputs[0]).toMatchObject({ + teamName, + laneId: 'secondary:opencode:bob', + reason: 'startup_recovery', + }); + await expect(readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName)).resolves.toMatchObject( + { + lanes: { + 'secondary:opencode:bob': { + state: 'active', + }, + }, + } + ); + expect(adapter.messageInputs[0]).toMatchObject({ + teamName, + laneId: 'secondary:opencode:bob', + memberName: 'bob', + text: 'confirmed alive missing lane recovers', + messageId: 'msg-recovered-confirmed-missing-lane-bob', + }); + }); + + it('recovers a missing mixed OpenCode lane index before watchdog scans unread OpenCode inbox', async () => { + const teamName = 'mixed-opencode-watchdog-recovers-missing-lane-safe-e2e'; + await writeMixedTeamConfig({ teamName, projectPath }); + await writeTeamMeta(teamName, projectPath); + await writeMembersMeta(teamName); + await fs.writeFile( + path.join(getTeamsBasePath(), teamName, 'launch-state.json'), + `${JSON.stringify( + createPersistedLaunchSnapshot({ + teamName, + expectedMembers: ['alice', 'bob', 'tom'], + leadSessionId: 'lead-session', + launchPhase: 'reconciled', + members: { + alice: { + name: 'alice', + providerId: 'codex', + laneId: 'primary', + laneKind: 'primary', + laneOwnerProviderId: 'codex', + launchState: 'confirmed_alive', + agentToolAccepted: true, + runtimeAlive: true, + bootstrapConfirmed: true, + hardFailure: false, + lastEvaluatedAt: '2026-04-23T10:00:00.000Z', + }, + bob: { + name: 'bob', + providerId: 'opencode', + model: 'opencode/minimax-m2.5-free', + laneId: 'secondary:opencode:bob', + laneKind: 'secondary', + laneOwnerProviderId: 'opencode', + launchState: 'failed_to_start', + agentToolAccepted: true, + runtimeAlive: false, + bootstrapConfirmed: false, + hardFailure: true, + hardFailureReason: 'OpenCode bridge reported member launch failure', + runtimePid: 7743, + runtimeSessionId: 'ses_bob_materialized', + livenessKind: 'runtime_process_candidate', + pidSource: 'opencode_bridge', + lastEvaluatedAt: '2026-04-23T10:00:00.000Z', + }, + tom: { + name: 'tom', + providerId: 'opencode', + model: 'opencode/nemotron-3-super-free', + laneId: 'secondary:opencode:tom', + laneKind: 'secondary', + laneOwnerProviderId: 'opencode', + launchState: 'failed_to_start', + agentToolAccepted: false, + runtimeAlive: false, + bootstrapConfirmed: false, + hardFailure: true, + hardFailureReason: 'OpenCode bridge reported member launch failure', + lastEvaluatedAt: '2026-04-23T10:00:00.000Z', + }, + }, + updatedAt: '2026-04-23T10:00:00.000Z', + }), + null, + 2 + )}\n`, + 'utf8' + ); + const inboxDir = path.join(getTeamsBasePath(), teamName, 'inboxes'); + await fs.mkdir(inboxDir, { recursive: true }); + await fs.writeFile( + path.join(inboxDir, 'bob.json'), + `${JSON.stringify( + [ + { + from: 'user', + to: 'bob', + text: 'recover this unread OpenCode message', + timestamp: '2026-04-23T10:01:00.000Z', + read: false, + messageId: 'msg-watchdog-recovers-missing-lane-bob', + }, + ], + null, + 2 + )}\n`, + 'utf8' + ); + const adapter = new FakeOpenCodeRuntimeAdapter('partial_pending', { + bob: 'launching', + tom: 'failed', + }); + const restartedService = new TeamProvisioningService(); + restartedService.setRuntimeAdapterRegistry(new TeamRuntimeAdapterRegistry([adapter])); + const scheduledWatchdogJobs: unknown[] = []; + (restartedService as any).scheduleOpenCodePromptDeliveryWatchdog = (input: unknown): void => { + scheduledWatchdogJobs.push(input); + }; + + await expect( + readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName) + ).resolves.toMatchObject({ lanes: {} }); + + await expect(restartedService.scanOpenCodePromptDeliveryWatchdog(teamName)).resolves.toBe(1); + + expect(adapter.reconcileInputs).toHaveLength(1); + expect(adapter.reconcileInputs[0]).toMatchObject({ + teamName, + laneId: 'secondary:opencode:bob', + reason: 'startup_recovery', + }); + await expect(readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName)).resolves.toMatchObject( + { + lanes: { + 'secondary:opencode:bob': { + state: 'active', + }, + }, + } + ); + expect(scheduledWatchdogJobs).toEqual([ + expect.objectContaining({ + teamName, + memberName: 'bob', + messageId: 'msg-watchdog-recovers-missing-lane-bob', + delayMs: 500, + }), + ]); + }); + + it('recovers one missing mixed OpenCode lane before watchdog scans while sibling lane is active', async () => { + const teamName = 'mixed-opencode-watchdog-recovers-one-missing-lane-safe-e2e'; + await writeMixedTeamConfig({ teamName, projectPath }); + await writeTeamMeta(teamName, projectPath); + await writeMembersMeta(teamName); + await fs.writeFile( + path.join(getTeamsBasePath(), teamName, 'launch-state.json'), + `${JSON.stringify( + createPersistedLaunchSnapshot({ + teamName, + expectedMembers: ['alice', 'bob', 'tom'], + leadSessionId: 'lead-session', + launchPhase: 'reconciled', + members: { + alice: { + name: 'alice', + providerId: 'codex', + laneId: 'primary', + laneKind: 'primary', + laneOwnerProviderId: 'codex', + launchState: 'confirmed_alive', + agentToolAccepted: true, + runtimeAlive: true, + bootstrapConfirmed: true, + hardFailure: false, + lastEvaluatedAt: '2026-04-23T10:00:00.000Z', + }, + bob: { + name: 'bob', + providerId: 'opencode', + model: 'opencode/minimax-m2.5-free', + laneId: 'secondary:opencode:bob', + laneKind: 'secondary', + laneOwnerProviderId: 'opencode', + launchState: 'failed_to_start', + agentToolAccepted: true, + runtimeAlive: false, + bootstrapConfirmed: false, + hardFailure: true, + hardFailureReason: 'OpenCode bridge reported member launch failure', + runtimePid: 7743, + runtimeSessionId: 'ses_bob_materialized', + livenessKind: 'runtime_process_candidate', + pidSource: 'opencode_bridge', + lastEvaluatedAt: '2026-04-23T10:00:00.000Z', + }, + tom: { + name: 'tom', + providerId: 'opencode', + model: 'opencode/nemotron-3-super-free', + laneId: 'secondary:opencode:tom', + laneKind: 'secondary', + laneOwnerProviderId: 'opencode', + launchState: 'confirmed_alive', + agentToolAccepted: true, + runtimeAlive: true, + bootstrapConfirmed: true, + hardFailure: false, + runtimePid: 7750, + runtimeSessionId: 'ses_tom_active', + livenessKind: 'runtime_process', + pidSource: 'opencode_bridge', + lastEvaluatedAt: '2026-04-23T10:00:00.000Z', + }, + }, + updatedAt: '2026-04-23T10:00:00.000Z', + }), + null, + 2 + )}\n`, + 'utf8' + ); + await upsertOpenCodeRuntimeLaneIndexEntry({ + teamsBasePath: getTeamsBasePath(), + teamName, + laneId: 'secondary:opencode:tom', + state: 'active', + }); + const inboxDir = path.join(getTeamsBasePath(), teamName, 'inboxes'); + await fs.mkdir(inboxDir, { recursive: true }); + await fs.writeFile( + path.join(inboxDir, 'bob.json'), + `${JSON.stringify( + [ + { + from: 'user', + to: 'bob', + text: 'recover only bob while tom stays active', + timestamp: '2026-04-23T10:01:00.000Z', + read: false, + messageId: 'msg-watchdog-recovers-one-missing-lane-bob', + }, + ], + null, + 2 + )}\n`, + 'utf8' + ); + const adapter = new FakeOpenCodeRuntimeAdapter('partial_pending', { + bob: 'launching', + tom: 'confirmed', + }); + const restartedService = new TeamProvisioningService(); + restartedService.setRuntimeAdapterRegistry(new TeamRuntimeAdapterRegistry([adapter])); + const scheduledWatchdogJobs: unknown[] = []; + (restartedService as any).scheduleOpenCodePromptDeliveryWatchdog = (input: unknown): void => { + scheduledWatchdogJobs.push(input); + }; + + await expect(restartedService.scanOpenCodePromptDeliveryWatchdog(teamName)).resolves.toBe(1); + + expect(adapter.reconcileInputs).toHaveLength(1); + expect(adapter.reconcileInputs[0]).toMatchObject({ + teamName, + laneId: 'secondary:opencode:bob', + reason: 'startup_recovery', + }); + await expect(readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName)).resolves.toMatchObject( + { + lanes: { + 'secondary:opencode:bob': { + state: 'active', + }, + 'secondary:opencode:tom': { + state: 'active', + }, + }, + } + ); + expect(scheduledWatchdogJobs).toEqual([ + expect.objectContaining({ + teamName, + memberName: 'bob', + messageId: 'msg-watchdog-recovers-one-missing-lane-bob', + delayMs: 500, + }), + ]); + }); + + it('does not recover a missing mixed OpenCode lane index from liveness-only persisted metadata', async () => { + const teamName = 'mixed-opencode-direct-message-liveness-only-missing-lane-safe-e2e'; + await writeMixedTeamConfig({ teamName, projectPath }); + await writeTeamMeta(teamName, projectPath); + await writeMembersMeta(teamName); + await fs.writeFile( + path.join(getTeamsBasePath(), teamName, 'launch-state.json'), + `${JSON.stringify( + createPersistedLaunchSnapshot({ + teamName, + expectedMembers: ['alice', 'bob'], + leadSessionId: 'lead-session', + launchPhase: 'reconciled', + members: { + alice: { + name: 'alice', + providerId: 'codex', + laneId: 'primary', + laneKind: 'primary', + laneOwnerProviderId: 'codex', + launchState: 'confirmed_alive', + agentToolAccepted: true, + runtimeAlive: true, + bootstrapConfirmed: true, + hardFailure: false, + lastEvaluatedAt: '2026-04-23T10:00:00.000Z', + }, + bob: { + name: 'bob', + providerId: 'opencode', + model: 'opencode/minimax-m2.5-free', + laneId: 'secondary:opencode:bob', + laneKind: 'secondary', + laneOwnerProviderId: 'opencode', + launchState: 'failed_to_start', + agentToolAccepted: true, + runtimeAlive: false, + bootstrapConfirmed: false, + hardFailure: true, + hardFailureReason: 'OpenCode bridge reported member launch failure', + livenessKind: 'runtime_process_candidate', + pidSource: 'opencode_bridge', + lastEvaluatedAt: '2026-04-23T10:00:00.000Z', + }, + }, + updatedAt: '2026-04-23T10:00:00.000Z', + }), + null, + 2 + )}\n`, + 'utf8' + ); + const adapter = new FakeOpenCodeRuntimeAdapter('partial_pending', { + bob: 'launching', + }); + const restartedService = new TeamProvisioningService(); + restartedService.setRuntimeAdapterRegistry(new TeamRuntimeAdapterRegistry([adapter])); + + await expect( + restartedService.deliverOpenCodeMemberMessage(teamName, { + memberName: 'bob', + text: 'must not recover from liveness-only stale metadata', + messageId: 'msg-liveness-only-missing-lane-bob', + }) + ).resolves.toEqual({ + delivered: false, + reason: 'opencode_runtime_not_active', + }); + + expect(adapter.reconcileInputs).toEqual([]); + expect(adapter.messageInputs).toEqual([]); + await expect(readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName)).resolves.toMatchObject( + { lanes: {} } + ); + }); + it('does not deliver direct OpenCode member messages to one detached mixed lane while its sibling lane stays live', async () => { const teamName = 'mixed-opencode-direct-message-one-detached-lane-safe-e2e'; await writeMixedTeamConfig({ teamName, projectPath }); diff --git a/test/main/services/team/TeamProvisioningService.test.ts b/test/main/services/team/TeamProvisioningService.test.ts index 3aaafc4d..dc32c302 100644 --- a/test/main/services/team/TeamProvisioningService.test.ts +++ b/test/main/services/team/TeamProvisioningService.test.ts @@ -10955,6 +10955,152 @@ describe('TeamProvisioningService', () => { }); }); + it('recovers missing mixed secondary lane index from materialized OpenCode runtime evidence', async () => { + const teamName = 'relay-works-missing-lane-recovery'; + writeTeamMeta(teamName, { + providerId: 'codex', + providerBackendId: 'codex-native', + model: 'gpt-5.4', + }); + writeMembersMeta(teamName, [ + { + name: 'atlas', + providerId: 'opencode', + model: 'opencode/nemotron-3-super-free', + }, + { + name: 'bob', + providerId: 'codex', + model: 'gpt-5.4', + }, + ]); + writeLaunchConfig(teamName, '/Users/test/proj', 'lead-session', ['bob']); + writeBootstrapState(teamName, [{ name: 'bob', status: 'registered' }]); + fs.writeFileSync( + getTeamLaunchStatePath(teamName), + `${JSON.stringify( + { + version: 2, + teamName, + updatedAt: '2026-04-23T10:00:00.000Z', + expectedMembers: ['atlas', 'bob'], + bootstrapExpectedMembers: ['bob'], + leadSessionId: 'lead-session', + launchPhase: 'reconciled', + members: { + atlas: { + name: 'atlas', + providerId: 'opencode', + model: 'opencode/nemotron-3-super-free', + laneId: 'secondary:opencode:atlas', + laneKind: 'secondary', + laneOwnerProviderId: 'opencode', + launchState: 'failed_to_start', + agentToolAccepted: true, + runtimeAlive: false, + bootstrapConfirmed: false, + hardFailure: true, + hardFailureReason: 'OpenCode bridge reported member launch failure', + runtimePid: 44123, + runtimeSessionId: 'ses_atlas_materialized', + livenessKind: 'runtime_process_candidate', + pidSource: 'opencode_bridge', + lastEvaluatedAt: '2026-04-23T10:00:00.000Z', + }, + bob: { + name: 'bob', + providerId: 'codex', + laneId: 'primary', + laneKind: 'primary', + laneOwnerProviderId: 'codex', + launchState: 'confirmed_alive', + agentToolAccepted: true, + runtimeAlive: true, + bootstrapConfirmed: true, + hardFailure: false, + lastEvaluatedAt: '2026-04-23T10:00:00.000Z', + }, + }, + summary: { + confirmedCount: 1, + pendingCount: 0, + failedCount: 1, + runtimeAlivePendingCount: 0, + }, + teamLaunchState: 'partial_failure', + }, + null, + 2 + )}\n`, + 'utf8' + ); + + const adapterReconcile = vi.fn(async (input: Record) => { + const member = (input.expectedMembers as Array<{ name: string }>)[0]?.name; + return { + runId: String(input.runId), + teamName, + launchPhase: 'reconciled', + teamLaunchState: 'partial_pending', + members: member + ? { + [member]: { + memberName: member, + providerId: 'opencode', + launchState: 'runtime_pending_bootstrap', + agentToolAccepted: true, + runtimeAlive: false, + bootstrapConfirmed: false, + hardFailure: false, + runtimePid: 44123, + sessionId: 'ses_atlas_materialized', + livenessKind: 'runtime_process_candidate', + diagnostics: ['runtime process candidate recovered'], + }, + } + : {}, + snapshot: null, + warnings: [], + diagnostics: ['fake reconcile recovered materialized runtime'], + }; + }); + const svc = new TeamProvisioningService(); + svc.setRuntimeAdapterRegistry( + new TeamRuntimeAdapterRegistry([ + { + providerId: 'opencode', + prepare: vi.fn(), + launch: vi.fn(), + reconcile: adapterReconcile, + stop: vi.fn(), + } as any, + ]) + ); + + const result = await svc.getMemberSpawnStatuses(teamName); + + expect(adapterReconcile).toHaveBeenCalledTimes(1); + expect(adapterReconcile).toHaveBeenCalledWith( + expect.objectContaining({ + teamName, + laneId: 'secondary:opencode:atlas', + reason: 'startup_recovery', + }) + ); + expect(result.expectedMembers).toEqual(expect.arrayContaining(['atlas', 'bob'])); + expect(result.statuses.atlas).toMatchObject({ + status: 'waiting', + launchState: 'runtime_pending_bootstrap', + }); + await expect(readOpenCodeRuntimeLaneIndex(tempTeamsBase, teamName)).resolves.toMatchObject({ + lanes: { + 'secondary:opencode:atlas': { + state: 'active', + }, + }, + }); + }); + it('reconciles stale persisted mixed pending OpenCode lanes instead of keeping them pending forever', async () => { const teamName = 'signal-ops-7'; writeTeamMeta(teamName, { diff --git a/test/main/services/team/fileLock.test.ts b/test/main/services/team/fileLock.test.ts index 9896e0b3..1e745ab8 100644 --- a/test/main/services/team/fileLock.test.ts +++ b/test/main/services/team/fileLock.test.ts @@ -1,7 +1,7 @@ import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; -import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { withFileLock } from '@main/services/team/fileLock'; @@ -67,6 +67,29 @@ describe('withFileLock', () => { expect(result).toBe('ok'); }); + it('removes a fresh abandoned lock when the owner process is gone', async () => { + const lockPath = `${testFile}.lock`; + const abandonedPid = 424_242; + fs.writeFileSync(lockPath, `${abandonedPid}\n${Date.now()}\n`, 'utf8'); + const killSpy = vi.spyOn(process, 'kill').mockImplementation(((pid: number | string) => { + if (pid === abandonedPid) { + const error = new Error('process is gone') as NodeJS.ErrnoException; + error.code = 'ESRCH'; + throw error; + } + return true; + }) as typeof process.kill); + + try { + const result = await withFileLock(testFile, async () => 'ok'); + + expect(result).toBe('ok'); + expect(fs.existsSync(lockPath)).toBe(false); + } finally { + killSpy.mockRestore(); + } + }); + it('creates parent directories for lock file', async () => { const nested = path.join(tmpDir, 'a', 'b', 'deep.json');