From fa8bbcbb3809a987c4f58691016dde2e492bf670 Mon Sep 17 00:00:00 2001 From: 777genius Date: Thu, 23 Apr 2026 13:25:38 +0300 Subject: [PATCH] test(team): add opencode mixed recovery smoke --- package.json | 2 + scripts/prove-opencode-mixed-recovery.mjs | 57 ++ .../services/team/TeamProvisioningService.ts | 318 ++++++++++- .../team/OpenCodeMixedRecovery.live.test.ts | 453 +++++++++++++++ .../OpenCodeTeamProvisioning.live.test.ts | 251 +++++++++ .../team/TeamProvisioningService.test.ts | 516 ++++++++++++++++++ 6 files changed, 1568 insertions(+), 29 deletions(-) create mode 100644 scripts/prove-opencode-mixed-recovery.mjs create mode 100644 test/main/services/team/OpenCodeMixedRecovery.live.test.ts create mode 100644 test/main/services/team/OpenCodeTeamProvisioning.live.test.ts diff --git a/package.json b/package.json index b1d34e6a..9212bc9d 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,8 @@ "dev:web": "node ./scripts/dev-web.mjs", "dev:kill": "node bin/kill-dev.js", "opencode:prove-production": "node ./scripts/prove-opencode-production.mjs", + "opencode:prove-mixed-recovery": "node ./scripts/prove-opencode-mixed-recovery.mjs", + "opencode:prove-team-provisioning": "OPENCODE_E2E=1 OPENCODE_E2E_TEAM_PROVISIONING=1 pnpm exec vitest run --maxWorkers 1 --minWorkers 1 test/main/services/team/OpenCodeTeamProvisioning.live.test.ts", "prebuild": "tsx scripts/fetch-pricing-data.ts && pnpm --filter agent-teams-controller build && pnpm --filter agent-teams-mcp build", "build": "electron-vite build", "dist": "electron-builder --mac --win --linux", diff --git a/scripts/prove-opencode-mixed-recovery.mjs b/scripts/prove-opencode-mixed-recovery.mjs new file mode 100644 index 00000000..c05689fc --- /dev/null +++ b/scripts/prove-opencode-mixed-recovery.mjs @@ -0,0 +1,57 @@ +#!/usr/bin/env node + +import { spawnSync } from 'node:child_process'; +import path from 'node:path'; +import process from 'node:process'; +import { fileURLToPath } from 'node:url'; + +const scriptDir = path.dirname(fileURLToPath(import.meta.url)); +const repoRoot = path.resolve(scriptDir, '..'); +const orchestratorRoot = process.env.CLAUDE_DEV_RUNTIME_ROOT?.trim(); +const siblingOrchestrator = path.resolve(repoRoot, '..', 'agent_teams_orchestrator'); + +const env = { + ...process.env, + OPENCODE_E2E: '1', + OPENCODE_E2E_MIXED_RECOVERY: '1', + OPENCODE_E2E_PROJECT_PATH: process.env.OPENCODE_E2E_PROJECT_PATH?.trim() || repoRoot, + OPENCODE_E2E_MODEL: process.env.OPENCODE_E2E_MODEL?.trim() || 'opencode/big-pickle', + OPENCODE_DISABLE_AUTOUPDATE: process.env.OPENCODE_DISABLE_AUTOUPDATE ?? '1', +}; + +if (!env.CLAUDE_AGENT_TEAMS_ORCHESTRATOR_CLI_PATH?.trim()) { + const runtimeRoot = orchestratorRoot ? path.resolve(orchestratorRoot) : siblingOrchestrator; + env.CLAUDE_AGENT_TEAMS_ORCHESTRATOR_CLI_PATH = path.join(runtimeRoot, 'cli'); +} + +console.log('Running OpenCode mixed recovery live smoke'); +console.log(`Model: ${env.OPENCODE_E2E_MODEL}`); +console.log(`Project: ${env.OPENCODE_E2E_PROJECT_PATH}`); +console.log(`Orchestrator CLI: ${env.CLAUDE_AGENT_TEAMS_ORCHESTRATOR_CLI_PATH}`); + +const result = spawnSync( + 'pnpm', + [ + 'exec', + 'vitest', + 'run', + '--maxWorkers', + '1', + '--minWorkers', + '1', + 'test/main/services/team/OpenCodeMixedRecovery.live.test.ts', + ], + { + cwd: repoRoot, + env, + stdio: 'inherit', + shell: process.platform === 'win32', + } +); + +if (result.error) { + console.error(`Failed to run OpenCode mixed recovery smoke: ${result.error.message}`); + process.exit(1); +} + +process.exit(result.status ?? 1); diff --git a/src/main/services/team/TeamProvisioningService.ts b/src/main/services/team/TeamProvisioningService.ts index 360839d8..2296d984 100644 --- a/src/main/services/team/TeamProvisioningService.ts +++ b/src/main/services/team/TeamProvisioningService.ts @@ -1924,6 +1924,39 @@ function extractBootstrapFailureReason(text: string): string | null { return trimmed.slice(0, 280); } +function isBootstrapTranscriptSuccessText( + text: string, + teamName: string, + memberName: string +): boolean { + const normalizedText = text.replace(/\s+/g, ' ').trim().toLowerCase(); + if (!normalizedText) { + return false; + } + + const normalizedTeamName = teamName.trim().toLowerCase(); + const normalizedMemberName = memberName.trim().toLowerCase(); + if (!normalizedTeamName || !normalizedMemberName) { + return false; + } + + if ( + normalizedText.startsWith( + `member briefing for ${normalizedMemberName} on team "${normalizedTeamName}" (${normalizedTeamName}).` + ) || + normalizedText.startsWith( + `member briefing for ${normalizedMemberName} on team '${normalizedTeamName}' (${normalizedTeamName}).` + ) + ) { + return true; + } + + return ( + normalizedText.includes(`bootstrap выполнен для \`${normalizedMemberName}\``) && + normalizedText.includes(`команде \`${normalizedTeamName}\``) + ); +} + function extractTranscriptTextContent(value: unknown): string[] { if (typeof value === 'string') { const trimmed = value.trim(); @@ -6134,6 +6167,58 @@ export class TeamProvisioningService { } } + private confirmMemberSpawnStatusFromTranscript( + run: ProvisioningRun, + memberName: string, + observedAt: string + ): void { + const prev = run.memberSpawnStatuses.get(memberName) ?? createInitialMemberSpawnStatusEntry(); + const updatedAt = nowIso(); + const next: MemberSpawnStatusEntry = { + ...prev, + status: 'online', + updatedAt, + agentToolAccepted: true, + runtimeAlive: prev.runtimeAlive === true, + bootstrapConfirmed: true, + hardFailure: false, + error: undefined, + hardFailureReason: undefined, + livenessSource: prev.livenessSource ?? 'process', + firstSpawnAcceptedAt: prev.firstSpawnAcceptedAt ?? observedAt, + lastHeartbeatAt: isMemberSpawnHeartbeatTimestampNewer(prev.lastHeartbeatAt, observedAt) + ? observedAt + : prev.lastHeartbeatAt, + }; + next.launchState = deriveMemberLaunchState(next); + + if ( + prev.status === next.status && + prev.launchState === next.launchState && + prev.error === next.error && + prev.hardFailureReason === next.hardFailureReason && + prev.livenessSource === next.livenessSource && + prev.agentToolAccepted === next.agentToolAccepted && + prev.runtimeAlive === next.runtimeAlive && + prev.bootstrapConfirmed === next.bootstrapConfirmed && + prev.hardFailure === next.hardFailure && + prev.firstSpawnAcceptedAt === next.firstSpawnAcceptedAt && + prev.lastHeartbeatAt === next.lastHeartbeatAt + ) { + return; + } + + run.memberSpawnStatuses.set(memberName, next); + run.pendingMemberRestarts?.delete(memberName); + this.syncMemberLaunchGraceCheck(run, memberName, next); + this.appendMemberBootstrapDiagnostic(run, memberName, 'bootstrap confirmed via transcript'); + if (!this.isCurrentTrackedRun(run)) return; + this.emitMemberSpawnChange(run, memberName); + if (run.isLaunch) { + void this.persistLaunchStateSnapshot(run, run.provisioningComplete ? 'finished' : 'active'); + } + } + /** * Get current member spawn statuses for a team. * Returns a map of memberName → MemberSpawnStatusEntry. @@ -6349,6 +6434,11 @@ export class TeamProvisioningService { launchMember?.model?.trim() ?? member.model?.trim() ?? undefined; + const launchSnapshotAlive = + this.isTeamAlive(teamName) && + (launchMember?.runtimeAlive === true || + launchMember?.bootstrapConfirmed === true || + launchMember?.launchState === 'confirmed_alive'); let rssBytes = rssPid ? rssBytesByPid.get(rssPid) : undefined; if (rssBytes == null && isSharedOpenCodeHost && typeof rssPid === 'number' && rssPid > 0) { try { @@ -6364,7 +6454,7 @@ export class TeamProvisioningService { snapshotMembers[memberName] = { memberName, - alive: liveRuntimeMember?.alive ?? launchMember?.runtimeAlive ?? false, + alive: liveRuntimeMember?.alive === true || launchSnapshotAlive, restartable, ...(backendType ? { backendType } : {}), ...(launchMember?.providerId ? { providerId: launchMember.providerId } : {}), @@ -6892,6 +6982,7 @@ export class TeamProvisioningService { } run.lastMemberSpawnAuditAt = now; await this.auditMemberSpawnStatuses(run); + await this.reconcileBootstrapTranscriptSuccesses(run); } private async reconcileBootstrapTranscriptFailures(run: ProvisioningRun): Promise { @@ -6920,6 +7011,32 @@ export class TeamProvisioningService { } } + private async reconcileBootstrapTranscriptSuccesses(run: ProvisioningRun): Promise { + for (const memberName of run.expectedMembers ?? []) { + const current = run.memberSpawnStatuses.get(memberName); + if ( + !current || + current.launchState === 'failed_to_start' || + current.launchState === 'confirmed_alive' || + current.bootstrapConfirmed === true || + current.agentToolAccepted !== true + ) { + continue; + } + const acceptedAtMs = + current.firstSpawnAcceptedAt != null ? Date.parse(current.firstSpawnAcceptedAt) : NaN; + const transcriptOutcome = await this.findBootstrapTranscriptOutcome( + run.teamName, + memberName, + Number.isFinite(acceptedAtMs) ? acceptedAtMs : null + ); + if (transcriptOutcome?.kind !== 'success') { + continue; + } + this.confirmMemberSpawnStatusFromTranscript(run, memberName, transcriptOutcome.observedAt); + } + } + private static readonly CONTEXT_EMIT_THROTTLE_MS = 2000; private static readonly LEAD_TEXT_EMIT_THROTTLE_MS = 2000; @@ -12147,6 +12264,8 @@ export class TeamProvisioningService { primaryStatuses: this.buildRuntimeSpawnStatusRecord(run), secondaryMembers: mixedSecondaryLanes.map((secondaryLane) => { const evidenceEntry = secondaryLane.result?.members[secondaryLane.member.name]; + const finishedWithoutRuntimeEvidence = + secondaryLane.state === 'finished' && !secondaryLane.result; return { laneId: secondaryLane.laneId, member: secondaryLane.member, @@ -12173,7 +12292,21 @@ export class TeamProvisioningService { pendingPermissionRequestIds: evidenceEntry.pendingPermissionRequestIds, diagnostics: evidenceEntry.diagnostics, } - : null, + : finishedWithoutRuntimeEvidence + ? { + launchState: 'runtime_pending_bootstrap', + agentToolAccepted: false, + runtimeAlive: false, + bootstrapConfirmed: false, + hardFailure: false, + diagnostics: + secondaryLane.diagnostics.length > 0 + ? [...secondaryLane.diagnostics] + : [ + 'OpenCode secondary lane finished without runtime evidence. Waiting for runtime reconciliation.', + ], + } + : null, pendingReason: secondaryLane.result || secondaryLane.state === 'finished' ? undefined @@ -12498,6 +12631,7 @@ export class TeamProvisioningService { if (activeMembers.length === 0) { return null; } + const projectPath = this.readPersistedTeamProjectPath(teamName); const laneIndex = await readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName).catch( () => ({ @@ -12541,6 +12675,7 @@ export class TeamProvisioningService { bootstrapConfirmed?: boolean; hardFailure?: boolean; hardFailureReason?: string; + pendingPermissionRequestIds?: string[]; diagnostics?: string[]; }; pendingReason?: string; @@ -12566,6 +12701,32 @@ export class TeamProvisioningService { let laneEntry = laneIndex.lanes[laneIdentity.laneId]; if (laneEntry?.state === 'active') { + const runtimeEvidence = await this.tryRecoverActiveOpenCodeSecondaryLaneFromRuntime({ + teamName, + laneId: laneIdentity.laneId, + member, + projectPath, + previousLaunchState: persistedSnapshot ?? bootstrapSnapshot, + }); + if (runtimeEvidence) { + recoveredAny = true; + secondaryMembers.push({ + laneId: laneIdentity.laneId, + member, + leadDefaults, + evidence: { + launchState: runtimeEvidence.launchState, + agentToolAccepted: runtimeEvidence.agentToolAccepted, + runtimeAlive: runtimeEvidence.runtimeAlive, + bootstrapConfirmed: runtimeEvidence.bootstrapConfirmed, + hardFailure: runtimeEvidence.hardFailure, + hardFailureReason: runtimeEvidence.hardFailureReason, + pendingPermissionRequestIds: runtimeEvidence.pendingPermissionRequestIds, + diagnostics: runtimeEvidence.diagnostics, + }, + }); + continue; + } const recovery = await recoverStaleOpenCodeRuntimeLaneIndexEntry({ teamsBasePath: getTeamsBasePath(), teamName, @@ -12640,6 +12801,50 @@ export class TeamProvisioningService { return recoveredSnapshot; } + private async tryRecoverActiveOpenCodeSecondaryLaneFromRuntime(params: { + teamName: string; + laneId: string; + member: TeamMember; + projectPath: string | null; + previousLaunchState: PersistedTeamLaunchSnapshot | null; + }): Promise { + const adapter = this.getOpenCodeRuntimeAdapter(); + if (!adapter || !params.projectPath) { + return null; + } + + try { + const reconcileResult = await adapter.reconcile({ + runId: randomUUID(), + laneId: params.laneId, + teamName: params.teamName, + providerId: 'opencode', + expectedMembers: [ + { + name: params.member.name, + role: params.member.role, + workflow: params.member.workflow, + isolation: params.member.isolation === 'worktree' ? ('worktree' as const) : undefined, + providerId: 'opencode', + model: params.member.model, + effort: params.member.effort, + cwd: params.projectPath, + }, + ], + previousLaunchState: params.previousLaunchState, + reason: 'startup_recovery', + }); + return reconcileResult.members[params.member.name] ?? null; + } catch (error) { + logger.warn( + `[${params.teamName}] Failed to recover stale OpenCode lane ${params.laneId} from runtime bridge: ${ + error instanceof Error ? error.message : String(error) + }` + ); + return null; + } + } + private async reconcilePersistedLaunchState(teamName: string): Promise<{ snapshot: ReturnType | null; statuses: Record; @@ -12802,14 +13007,19 @@ export class TeamProvisioningService { current.hardFailureReason = undefined; } if (!current.bootstrapConfirmed && !current.hardFailure) { - const transcriptFailureReason = await this.findBootstrapTranscriptFailureReason( + const transcriptOutcome = await this.findBootstrapTranscriptOutcome( teamName, expected, Number.isFinite(acceptedAtMs) ? acceptedAtMs : null ); - if (transcriptFailureReason) { + if (transcriptOutcome?.kind === 'success') { + current.bootstrapConfirmed = true; + current.lastHeartbeatAt = current.lastHeartbeatAt ?? transcriptOutcome.observedAt; + current.hardFailure = false; + current.hardFailureReason = undefined; + } else if (transcriptOutcome?.kind === 'failure') { current.hardFailure = true; - current.hardFailureReason = transcriptFailureReason; + current.hardFailureReason = transcriptOutcome.reason; current.sources.hardFailureSignal = true; } } @@ -12864,6 +13074,26 @@ export class TeamProvisioningService { memberName: string, sinceMs: number | null ): Promise { + const outcome = await this.findBootstrapTranscriptOutcome(teamName, memberName, sinceMs); + return outcome?.kind === 'failure' ? outcome.reason : null; + } + + private async findBootstrapTranscriptOutcome( + teamName: string, + memberName: string, + sinceMs: number | null + ): Promise< + | { + kind: 'success'; + observedAt: string; + } + | { + kind: 'failure'; + observedAt: string; + reason: string; + } + | null + > { let summaries: Awaited>; try { summaries = await this.memberLogsFinder.findMemberLogs(teamName, memberName, sinceMs); @@ -12873,26 +13103,39 @@ export class TeamProvisioningService { for (const summary of summaries) { if (!summary.filePath) continue; - const reason = await this.readRecentBootstrapFailureReason( + const outcome = await this.readRecentBootstrapTranscriptOutcome( summary.filePath, sinceMs, - memberName + memberName, + teamName ); - if (reason) { - return reason; + if (outcome) { + return outcome; } } - return this.findBootstrapFailureReasonInProjectRoot(teamName, memberName, sinceMs); + return this.findBootstrapTranscriptOutcomeInProjectRoot(teamName, memberName, sinceMs); } - private async readRecentBootstrapFailureReason( + private async readRecentBootstrapTranscriptOutcome( filePath: string, sinceMs: number | null, - memberName?: string - ): Promise { + memberName: string, + teamName: string + ): Promise< + | { + kind: 'success'; + observedAt: string; + } + | { + kind: 'failure'; + observedAt: string; + reason: string; + } + | null + > { let handle: fs.promises.FileHandle | null = null; - const normalizedMemberName = memberName?.trim().toLowerCase() || null; + const normalizedMemberName = memberName.trim().toLowerCase(); try { handle = await fs.promises.open(filePath, 'r'); const stat = await handle.stat(); @@ -12923,20 +13166,25 @@ export class TeamProvisioningService { if (sinceMs != null && Number.isFinite(timestampMs) && timestampMs < sinceMs) { continue; } - if (normalizedMemberName) { - const parsedAgentName = - typeof (parsed as { agentName?: unknown }).agentName === 'string' - ? (parsed as { agentName?: string }).agentName?.trim().toLowerCase() || null - : null; - if (parsedAgentName && parsedAgentName !== normalizedMemberName) { - continue; - } + const parsedAgentName = + typeof (parsed as { agentName?: unknown }).agentName === 'string' + ? (parsed as { agentName?: string }).agentName?.trim().toLowerCase() || null + : null; + if (parsedAgentName && parsedAgentName !== normalizedMemberName) { + continue; } const text = extractTranscriptMessageText(parsed); if (!text) continue; + const observedAt = + typeof parsed.timestamp === 'string' && parsed.timestamp.trim().length > 0 + ? parsed.timestamp.trim() + : new Date().toISOString(); const reason = extractBootstrapFailureReason(text); if (reason) { - return reason; + return { kind: 'failure', observedAt, reason }; + } + if (isBootstrapTranscriptSuccessText(text, teamName, memberName)) { + return { kind: 'success', observedAt }; } } } catch { @@ -12948,11 +13196,22 @@ export class TeamProvisioningService { return null; } - private async findBootstrapFailureReasonInProjectRoot( + private async findBootstrapTranscriptOutcomeInProjectRoot( teamName: string, memberName: string, sinceMs: number | null - ): Promise { + ): Promise< + | { + kind: 'success'; + observedAt: string; + } + | { + kind: 'failure'; + observedAt: string; + reason: string; + } + | null + > { let config: Awaited>; try { config = await this.configReader.getConfig(teamName); @@ -12979,13 +13238,14 @@ export class TeamProvisioningService { if (config?.leadSessionId && entry.name === `${config.leadSessionId}.jsonl`) { continue; } - const reason = await this.readRecentBootstrapFailureReason( + const outcome = await this.readRecentBootstrapTranscriptOutcome( path.join(projectDir, entry.name), sinceMs, - memberName + memberName, + teamName ); - if (reason) { - return reason; + if (outcome) { + return outcome; } } diff --git a/test/main/services/team/OpenCodeMixedRecovery.live.test.ts b/test/main/services/team/OpenCodeMixedRecovery.live.test.ts new file mode 100644 index 00000000..89941f3e --- /dev/null +++ b/test/main/services/team/OpenCodeMixedRecovery.live.test.ts @@ -0,0 +1,453 @@ +import { constants as fsConstants, promises as fs } from 'fs'; +import * as os from 'os'; +import * as path from 'path'; + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { OpenCodeBridgeCommandClient } from '../../../../src/main/services/team/opencode/bridge/OpenCodeBridgeCommandClient'; +import { + createOpenCodeBridgeCommandLeaseStore, + createOpenCodeBridgeCommandLedgerStore, +} from '../../../../src/main/services/team/opencode/bridge/OpenCodeBridgeCommandLedgerStore'; +import { + createOpenCodeBridgeClientIdentity, + OpenCodeBridgeCommandHandshakePort, +} from '../../../../src/main/services/team/opencode/bridge/OpenCodeBridgeHandshakeClient'; +import { OpenCodeReadinessBridge } from '../../../../src/main/services/team/opencode/bridge/OpenCodeReadinessBridge'; +import { OpenCodeStateChangingBridgeCommandService } from '../../../../src/main/services/team/opencode/bridge/OpenCodeStateChangingBridgeCommandService'; +import { + getTeamBootstrapStatePath, +} from '../../../../src/main/services/team/TeamBootstrapStateReader'; +import { TeamMembersMetaStore } from '../../../../src/main/services/team/TeamMembersMetaStore'; +import { TeamMetaStore } from '../../../../src/main/services/team/TeamMetaStore'; +import { TeamProvisioningService } from '../../../../src/main/services/team/TeamProvisioningService'; +import { TeamRuntimeAdapterRegistry } from '../../../../src/main/services/team/runtime/TeamRuntimeAdapter'; +import { OpenCodeTeamRuntimeAdapter } from '../../../../src/main/services/team/runtime/OpenCodeTeamRuntimeAdapter'; +import { resolveAgentTeamsMcpLaunchSpec } from '../../../../src/main/services/team/TeamMcpConfigBuilder'; +import { + readOpenCodeRuntimeLaneIndex, + upsertOpenCodeRuntimeLaneIndexEntry, +} from '../../../../src/main/services/team/opencode/store/OpenCodeRuntimeManifestEvidenceReader'; +import { applyOpenCodeAutoUpdatePolicy } from '../../../../src/main/services/runtime/openCodeAutoUpdatePolicy'; +import { + getTeamsBasePath, + setClaudeBasePathOverride, +} from '../../../../src/main/utils/pathDecoder'; + +import type { + TeamRuntimeLaunchInput, + TeamRuntimeStopInput, +} from '../../../../src/main/services/team/runtime/TeamRuntimeAdapter'; +import type { RuntimeStoreManifestEvidence } from '../../../../src/main/services/team/opencode/bridge/OpenCodeBridgeCommandContract'; +import type { RuntimeStoreManifestReader } from '../../../../src/main/services/team/opencode/bridge/OpenCodeStateChangingBridgeCommandService'; +import type { OpenCodeBridgeCommandExecutor } from '../../../../src/main/services/team/opencode/bridge/OpenCodeStateChangingBridgeCommandService'; + +const liveDescribe = + process.env.OPENCODE_E2E === '1' && process.env.OPENCODE_E2E_MIXED_RECOVERY === '1' + ? describe + : describe.skip; + +const PROJECT_PATH = process.env.OPENCODE_E2E_PROJECT_PATH?.trim() || process.cwd(); +const DEFAULT_ORCHESTRATOR_CLI = '/Users/belief/dev/projects/claude/agent_teams_orchestrator/cli'; +const DEFAULT_MODEL = 'opencode/big-pickle'; + +liveDescribe('OpenCode mixed recovery live e2e', () => { + let tempDir: string; + let tempClaudeRoot: string; + + beforeEach(async () => { + tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'opencode-mixed-recovery-e2e-')); + tempClaudeRoot = path.join(tempDir, '.claude'); + await fs.mkdir(tempClaudeRoot, { recursive: true }); + setClaudeBasePathOverride(tempClaudeRoot); + }); + + afterEach(async () => { + setClaudeBasePathOverride(null); + await fs.rm(tempDir, { recursive: true, force: true }); + }); + + it( + 'recovers active mixed OpenCode side lanes from live runtime reconcile instead of marking them never spawned', + async () => { + const selectedModel = process.env.OPENCODE_E2E_MODEL?.trim() || DEFAULT_MODEL; + const orchestratorCli = + process.env.CLAUDE_AGENT_TEAMS_ORCHESTRATOR_CLI_PATH?.trim() || DEFAULT_ORCHESTRATOR_CLI; + await assertExecutable(orchestratorCli); + + const mcpLaunchSpec = await resolveAgentTeamsMcpLaunchSpec(); + const bridgeEnv = { + ...createStableBridgeEnv(), + PATH: withBunOnPath(process.env.PATH ?? ''), + CLAUDE_MULTIMODEL_AGENT_TEAMS_MCP_COMMAND: mcpLaunchSpec.command, + CLAUDE_MULTIMODEL_AGENT_TEAMS_MCP_ENTRY: mcpLaunchSpec.args[0] ?? '', + }; + const bridgeClient = new OpenCodeBridgeCommandClient({ + binaryPath: orchestratorCli, + tempDirectory: path.join(tempDir, 'bridge-input'), + env: bridgeEnv, + }); + const stateChangingCommands = createStateChangingCommands({ + bridge: bridgeClient, + controlDir: path.join(tempDir, 'control'), + }); + const readinessBridge = new OpenCodeReadinessBridge(bridgeClient, { + stateChangingCommands, + timeoutMs: 180_000, + launchTimeoutMs: 180_000, + reconcileTimeoutMs: 90_000, + stopTimeoutMs: 90_000, + }); + const adapter = new OpenCodeTeamRuntimeAdapter(readinessBridge, { + launchMode: 'dogfood', + }); + const svc = new TeamProvisioningService(); + svc.setRuntimeAdapterRegistry(new TeamRuntimeAdapterRegistry([adapter])); + + const teamName = `mixed-opencode-recovery-${Date.now()}`; + const launchedLanes: TeamRuntimeLaunchInput[] = []; + + await writeMixedRecoveryFixtures({ + teamName, + projectPath: PROJECT_PATH, + secondaryMembers: ['bob'], + }); + + try { + const launchInput = createSecondaryLaneLaunchInput({ + teamName, + laneId: 'secondary:opencode:bob', + memberName: 'bob', + selectedModel, + }); + launchedLanes.push(launchInput); + const launchResult = await adapter.launch(launchInput); + expect(launchResult.teamLaunchState).toBe('clean_success'); + expect(launchResult.members.bob).toMatchObject({ + launchState: 'confirmed_alive', + runtimeAlive: true, + bootstrapConfirmed: true, + }); + + await upsertOpenCodeRuntimeLaneIndexEntry({ + teamsBasePath: getTeamsBasePath(), + teamName, + laneId: launchInput.laneId ?? 'secondary:opencode:bob', + state: 'active', + }); + + const result = await svc.getMemberSpawnStatuses(teamName); + + expect(result.expectedMembers).toEqual(expect.arrayContaining(['alice', 'bob'])); + expect(result.statuses.bob).toMatchObject({ + status: 'online', + launchState: 'confirmed_alive', + }); + expect(result.statuses.bob.error).toBeUndefined(); + await expect(readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName)).resolves.toMatchObject( + { + lanes: { + [launchInput.laneId ?? 'secondary:opencode:bob']: { + state: 'active', + }, + }, + } + ); + } finally { + for (const launchInput of launchedLanes) { + await adapter + .stop({ + runId: launchInput.runId, + laneId: launchInput.laneId, + teamName, + cwd: PROJECT_PATH, + providerId: 'opencode', + reason: 'cleanup', + previousLaunchState: null, + force: true, + } satisfies TeamRuntimeStopInput) + .catch(() => undefined); + } + } + }, + 240_000 + ); + + it( + 'recovers multiple active mixed OpenCode side lanes from live runtime reconcile', + async () => { + const selectedModel = process.env.OPENCODE_E2E_MODEL?.trim() || DEFAULT_MODEL; + const orchestratorCli = + process.env.CLAUDE_AGENT_TEAMS_ORCHESTRATOR_CLI_PATH?.trim() || DEFAULT_ORCHESTRATOR_CLI; + await assertExecutable(orchestratorCli); + + const mcpLaunchSpec = await resolveAgentTeamsMcpLaunchSpec(); + const bridgeEnv = { + ...createStableBridgeEnv(), + PATH: withBunOnPath(process.env.PATH ?? ''), + CLAUDE_MULTIMODEL_AGENT_TEAMS_MCP_COMMAND: mcpLaunchSpec.command, + CLAUDE_MULTIMODEL_AGENT_TEAMS_MCP_ENTRY: mcpLaunchSpec.args[0] ?? '', + }; + const bridgeClient = new OpenCodeBridgeCommandClient({ + binaryPath: orchestratorCli, + tempDirectory: path.join(tempDir, 'bridge-input-multi'), + env: bridgeEnv, + }); + const stateChangingCommands = createStateChangingCommands({ + bridge: bridgeClient, + controlDir: path.join(tempDir, 'control-multi'), + }); + const readinessBridge = new OpenCodeReadinessBridge(bridgeClient, { + stateChangingCommands, + timeoutMs: 180_000, + launchTimeoutMs: 180_000, + reconcileTimeoutMs: 90_000, + stopTimeoutMs: 90_000, + }); + const adapter = new OpenCodeTeamRuntimeAdapter(readinessBridge, { + launchMode: 'dogfood', + }); + const svc = new TeamProvisioningService(); + svc.setRuntimeAdapterRegistry(new TeamRuntimeAdapterRegistry([adapter])); + + const teamName = `mixed-opencode-recovery-multi-${Date.now()}`; + const sideMembers = ['bob', 'jack', 'tom'] as const; + const launchedLanes: TeamRuntimeLaunchInput[] = []; + + await writeMixedRecoveryFixtures({ + teamName, + projectPath: PROJECT_PATH, + secondaryMembers: [...sideMembers], + }); + + try { + for (const memberName of sideMembers) { + const launchInput = createSecondaryLaneLaunchInput({ + teamName, + laneId: `secondary:opencode:${memberName}`, + memberName, + selectedModel, + }); + launchedLanes.push(launchInput); + const launchResult = await adapter.launch(launchInput); + expect(launchResult.teamLaunchState).toBe('clean_success'); + expect(launchResult.members[memberName]).toMatchObject({ + launchState: 'confirmed_alive', + runtimeAlive: true, + bootstrapConfirmed: true, + }); + await upsertOpenCodeRuntimeLaneIndexEntry({ + teamsBasePath: getTeamsBasePath(), + teamName, + laneId: launchInput.laneId ?? `secondary:opencode:${memberName}`, + state: 'active', + }); + } + + const result = await svc.getMemberSpawnStatuses(teamName); + + expect(result.expectedMembers).toEqual( + expect.arrayContaining(['alice', 'bob', 'jack', 'tom']) + ); + for (const memberName of sideMembers) { + expect(result.statuses[memberName]).toMatchObject({ + status: 'online', + launchState: 'confirmed_alive', + }); + expect(result.statuses[memberName]?.error).toBeUndefined(); + } + await expect(readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName)).resolves.toMatchObject( + { + lanes: Object.fromEntries( + sideMembers.map((memberName) => [ + `secondary:opencode:${memberName}`, + { state: 'active' }, + ]) + ), + } + ); + } finally { + for (const launchInput of launchedLanes) { + await adapter + .stop({ + runId: launchInput.runId, + laneId: launchInput.laneId, + teamName, + cwd: PROJECT_PATH, + providerId: 'opencode', + reason: 'cleanup', + previousLaunchState: null, + force: true, + } satisfies TeamRuntimeStopInput) + .catch(() => undefined); + } + } + }, + 420_000 + ); +}); + +function createSecondaryLaneLaunchInput(input: { + teamName: string; + laneId: string; + memberName: string; + selectedModel: string; +}): TeamRuntimeLaunchInput { + return { + runId: `mixed-opencode-recovery-${Date.now()}`, + laneId: input.laneId, + teamName: input.teamName, + cwd: PROJECT_PATH, + prompt: 'Mixed OpenCode recovery live e2e', + providerId: 'opencode', + model: input.selectedModel, + skipPermissions: true, + expectedMembers: [ + { + name: input.memberName, + role: 'Developer', + providerId: 'opencode', + model: input.selectedModel, + cwd: PROJECT_PATH, + }, + ], + previousLaunchState: null, + }; +} + +async function writeMixedRecoveryFixtures(input: { + teamName: string; + projectPath: string; + secondaryMembers: string[]; +}): Promise { + const teamDir = path.join(getTeamsBasePath(), input.teamName); + await fs.mkdir(teamDir, { recursive: true }); + + await new TeamMetaStore().writeMeta(input.teamName, { + cwd: input.projectPath, + providerId: 'codex', + providerBackendId: 'codex-native', + model: 'gpt-5.4', + createdAt: Date.now(), + }); + await new TeamMembersMetaStore().writeMembers( + input.teamName, + [ + { + name: 'alice', + role: 'Reviewer', + providerId: 'codex', + model: 'gpt-5.4-mini', + }, + ...input.secondaryMembers.map((memberName) => ({ + name: memberName, + role: 'Developer', + providerId: 'opencode' as const, + model: process.env.OPENCODE_E2E_MODEL?.trim() || DEFAULT_MODEL, + })), + ], + { + providerBackendId: 'codex-native', + } + ); + await fs.writeFile( + path.join(teamDir, 'config.json'), + `${JSON.stringify( + { + name: input.teamName, + projectPath: input.projectPath, + leadSessionId: 'lead-session', + members: [ + { name: 'team-lead', agentType: 'team-lead' }, + { name: 'alice' }, + ], + }, + null, + 2 + )}\n`, + 'utf8' + ); + await fs.writeFile( + getTeamBootstrapStatePath(input.teamName), + `${JSON.stringify( + { + version: 1, + teamName: input.teamName, + updatedAt: new Date().toISOString(), + phase: 'completed', + members: [ + { + name: 'alice', + status: 'registered', + lastAttemptAt: Date.now(), + lastObservedAt: Date.now(), + }, + ], + terminal: { + status: 'completed', + }, + }, + null, + 2 + )}\n`, + 'utf8' + ); +} + +function createStateChangingCommands(input: { + bridge: OpenCodeBridgeCommandExecutor; + controlDir: string; +}): OpenCodeStateChangingBridgeCommandService { + const clientIdentity = createOpenCodeBridgeClientIdentity({ + appVersion: '1.3.0-e2e', + gitSha: null, + buildId: 'opencode-mixed-recovery-e2e', + }); + + return new OpenCodeStateChangingBridgeCommandService({ + expectedClientIdentity: clientIdentity, + handshakePort: new OpenCodeBridgeCommandHandshakePort({ + bridge: input.bridge, + clientIdentity, + }), + leaseStore: createOpenCodeBridgeCommandLeaseStore({ + filePath: path.join(input.controlDir, 'leases.json'), + }), + ledger: createOpenCodeBridgeCommandLedgerStore({ + filePath: path.join(input.controlDir, 'ledger.json'), + }), + bridge: input.bridge, + manifestReader: new StaticManifestReader(), + }); +} + +class StaticManifestReader implements RuntimeStoreManifestReader { + async read(): Promise { + return { + highWatermark: 0, + activeRunId: null, + capabilitySnapshotId: null, + }; + } +} + +async function assertExecutable(filePath: string): Promise { + await fs.access(filePath, fsConstants.X_OK); +} + +function withBunOnPath(pathValue: string): string { + const bunDir = '/Users/belief/.bun/bin'; + return pathValue.split(path.delimiter).includes(bunDir) + ? pathValue + : `${bunDir}${path.delimiter}${pathValue}`; +} + +function createStableBridgeEnv(): NodeJS.ProcessEnv { + const realHome = os.userInfo().homedir; + const env = applyOpenCodeAutoUpdatePolicy({ ...process.env }); + return { + ...env, + HOME: realHome, + USERPROFILE: realHome, + }; +} diff --git a/test/main/services/team/OpenCodeTeamProvisioning.live.test.ts b/test/main/services/team/OpenCodeTeamProvisioning.live.test.ts new file mode 100644 index 00000000..5964c0d9 --- /dev/null +++ b/test/main/services/team/OpenCodeTeamProvisioning.live.test.ts @@ -0,0 +1,251 @@ +import { constants as fsConstants, promises as fs } from 'fs'; +import * as os from 'os'; +import * as path from 'path'; + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { OpenCodeBridgeCommandClient } from '../../../../src/main/services/team/opencode/bridge/OpenCodeBridgeCommandClient'; +import { + createOpenCodeBridgeCommandLeaseStore, + createOpenCodeBridgeCommandLedgerStore, +} from '../../../../src/main/services/team/opencode/bridge/OpenCodeBridgeCommandLedgerStore'; +import { + createOpenCodeBridgeClientIdentity, + OpenCodeBridgeCommandHandshakePort, +} from '../../../../src/main/services/team/opencode/bridge/OpenCodeBridgeHandshakeClient'; +import { OpenCodeReadinessBridge } from '../../../../src/main/services/team/opencode/bridge/OpenCodeReadinessBridge'; +import { OpenCodeStateChangingBridgeCommandService } from '../../../../src/main/services/team/opencode/bridge/OpenCodeStateChangingBridgeCommandService'; +import { TeamProvisioningService } from '../../../../src/main/services/team/TeamProvisioningService'; +import { TeamRuntimeAdapterRegistry } from '../../../../src/main/services/team/runtime/TeamRuntimeAdapter'; +import { OpenCodeTeamRuntimeAdapter } from '../../../../src/main/services/team/runtime/OpenCodeTeamRuntimeAdapter'; +import { resolveAgentTeamsMcpLaunchSpec } from '../../../../src/main/services/team/TeamMcpConfigBuilder'; +import { readOpenCodeRuntimeLaneIndex } from '../../../../src/main/services/team/opencode/store/OpenCodeRuntimeManifestEvidenceReader'; +import { applyOpenCodeAutoUpdatePolicy } from '../../../../src/main/services/runtime/openCodeAutoUpdatePolicy'; +import { + getTeamsBasePath, + setClaudeBasePathOverride, +} from '../../../../src/main/utils/pathDecoder'; + +import type { RuntimeStoreManifestEvidence } from '../../../../src/main/services/team/opencode/bridge/OpenCodeBridgeCommandContract'; +import type { RuntimeStoreManifestReader } from '../../../../src/main/services/team/opencode/bridge/OpenCodeStateChangingBridgeCommandService'; +import type { OpenCodeBridgeCommandExecutor } from '../../../../src/main/services/team/opencode/bridge/OpenCodeStateChangingBridgeCommandService'; +import type { TeamProvisioningProgress } from '../../../../src/shared/types'; + +const liveDescribe = + process.env.OPENCODE_E2E === '1' && process.env.OPENCODE_E2E_TEAM_PROVISIONING === '1' + ? describe + : describe.skip; + +const PROJECT_PATH = process.env.OPENCODE_E2E_PROJECT_PATH?.trim() || process.cwd(); +const DEFAULT_ORCHESTRATOR_CLI = '/Users/belief/dev/projects/claude/agent_teams_orchestrator/cli'; +const DEFAULT_MODEL = 'opencode/big-pickle'; + +liveDescribe('OpenCode team provisioning live e2e', () => { + let tempDir: string; + let tempClaudeRoot: string; + + beforeEach(async () => { + tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'opencode-team-provisioning-e2e-')); + tempClaudeRoot = path.join(tempDir, '.claude'); + await fs.mkdir(tempClaudeRoot, { recursive: true }); + setClaudeBasePathOverride(tempClaudeRoot); + }); + + afterEach(async () => { + setClaudeBasePathOverride(null); + await fs.rm(tempDir, { recursive: true, force: true }); + }); + + it( + 'creates and stops a pure OpenCode team through TeamProvisioningService using the live runtime adapter', + async () => { + const selectedModel = process.env.OPENCODE_E2E_MODEL?.trim() || DEFAULT_MODEL; + const orchestratorCli = + process.env.CLAUDE_AGENT_TEAMS_ORCHESTRATOR_CLI_PATH?.trim() || DEFAULT_ORCHESTRATOR_CLI; + await assertExecutable(orchestratorCli); + + const mcpLaunchSpec = await resolveAgentTeamsMcpLaunchSpec(); + const bridgeEnv = { + ...createStableBridgeEnv(), + PATH: withBunOnPath(process.env.PATH ?? ''), + CLAUDE_MULTIMODEL_AGENT_TEAMS_MCP_COMMAND: mcpLaunchSpec.command, + CLAUDE_MULTIMODEL_AGENT_TEAMS_MCP_ENTRY: mcpLaunchSpec.args[0] ?? '', + }; + const bridgeClient = new OpenCodeBridgeCommandClient({ + binaryPath: orchestratorCli, + tempDirectory: path.join(tempDir, 'bridge-input'), + env: bridgeEnv, + }); + const stateChangingCommands = createStateChangingCommands({ + bridge: bridgeClient, + controlDir: path.join(tempDir, 'control'), + }); + const readinessBridge = new OpenCodeReadinessBridge(bridgeClient, { + stateChangingCommands, + timeoutMs: 180_000, + launchTimeoutMs: 180_000, + reconcileTimeoutMs: 90_000, + stopTimeoutMs: 90_000, + }); + const adapter = new OpenCodeTeamRuntimeAdapter(readinessBridge, { + launchMode: 'dogfood', + }); + const svc = new TeamProvisioningService(); + svc.setRuntimeAdapterRegistry(new TeamRuntimeAdapterRegistry([adapter])); + + const teamName = `opencode-team-provisioning-${Date.now()}`; + const progressEvents: TeamProvisioningProgress[] = []; + + try { + const { runId } = await svc.createTeam( + { + teamName, + cwd: PROJECT_PATH, + providerId: 'opencode', + model: selectedModel, + skipPermissions: true, + members: [ + { + name: 'alice', + role: 'Developer', + providerId: 'opencode', + model: selectedModel, + }, + { + name: 'bob', + role: 'Reviewer', + providerId: 'opencode', + model: selectedModel, + }, + ], + }, + (progress) => { + progressEvents.push(progress); + } + ); + + expect(runId).toBeTruthy(); + const progressDump = progressEvents + .map((progress) => + [ + progress.state, + progress.message, + progress.messageSeverity, + progress.error, + progress.cliLogsTail, + ] + .filter(Boolean) + .join(' | ') + ) + .join('\n'); + expect( + progressEvents.some((progress) => + progress.message.includes('OpenCode team launch is ready') + ), + progressDump + ).toBe(true); + + const runtimeSnapshot = await svc.getTeamAgentRuntimeSnapshot(teamName); + expect(runtimeSnapshot.members.alice).toMatchObject({ + alive: true, + runtimeModel: selectedModel, + }); + expect(runtimeSnapshot.members.bob).toMatchObject({ + alive: true, + runtimeModel: selectedModel, + }); + await expect(readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName)).resolves.toMatchObject( + { + lanes: { + primary: { + state: 'active', + }, + }, + } + ); + + svc.stopTeam(teamName); + await waitUntil(async () => { + const laneIndex = await readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName); + return Object.keys(laneIndex.lanes).length === 0; + }, 90_000); + } finally { + svc.stopTeam(teamName); + } + }, + 300_000 + ); +}); + +function createStateChangingCommands(input: { + bridge: OpenCodeBridgeCommandExecutor; + controlDir: string; +}): OpenCodeStateChangingBridgeCommandService { + const clientIdentity = createOpenCodeBridgeClientIdentity({ + appVersion: '1.3.0-e2e', + gitSha: null, + buildId: 'opencode-team-provisioning-e2e', + }); + + return new OpenCodeStateChangingBridgeCommandService({ + expectedClientIdentity: clientIdentity, + handshakePort: new OpenCodeBridgeCommandHandshakePort({ + bridge: input.bridge, + clientIdentity, + }), + leaseStore: createOpenCodeBridgeCommandLeaseStore({ + filePath: path.join(input.controlDir, 'leases.json'), + }), + ledger: createOpenCodeBridgeCommandLedgerStore({ + filePath: path.join(input.controlDir, 'ledger.json'), + }), + bridge: input.bridge, + manifestReader: new StaticManifestReader(), + }); +} + +class StaticManifestReader implements RuntimeStoreManifestReader { + async read(): Promise { + return { + highWatermark: 0, + activeRunId: null, + capabilitySnapshotId: null, + }; + } +} + +async function waitUntil( + predicate: () => Promise, + timeoutMs: number, + pollMs = 500 +): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (await predicate()) { + return; + } + await new Promise((resolve) => setTimeout(resolve, pollMs)); + } + throw new Error(`Timed out after ${timeoutMs}ms waiting for condition`); +} + +async function assertExecutable(filePath: string): Promise { + await fs.access(filePath, fsConstants.X_OK); +} + +function withBunOnPath(pathValue: string): string { + const bunDir = '/Users/belief/.bun/bin'; + return pathValue.split(path.delimiter).includes(bunDir) + ? pathValue + : `${bunDir}${path.delimiter}${pathValue}`; +} + +function createStableBridgeEnv(): NodeJS.ProcessEnv { + const realHome = os.userInfo().homedir; + const env = applyOpenCodeAutoUpdatePolicy({ ...process.env }); + return { + ...env, + HOME: realHome, + USERPROFILE: realHome, + }; +} diff --git a/test/main/services/team/TeamProvisioningService.test.ts b/test/main/services/team/TeamProvisioningService.test.ts index 08040af6..f81d0483 100644 --- a/test/main/services/team/TeamProvisioningService.test.ts +++ b/test/main/services/team/TeamProvisioningService.test.ts @@ -805,6 +805,40 @@ describe('TeamProvisioningService', () => { expect(snapshot.members.alice).toBeUndefined(); }); + it('keeps pure OpenCode launch members alive from confirmed launch snapshot while runtime adapter is tracked', async () => { + const teamName = 'pure-opencode-runtime-team'; + const projectPath = '/Users/test/project'; + writeLaunchConfig(teamName, projectPath, 'lead-session', ['alice']); + writeLaunchState(teamName, 'lead-session', { + alice: { + providerId: 'opencode', + model: 'opencode/big-pickle', + launchState: 'confirmed_alive', + agentToolAccepted: true, + runtimeAlive: true, + bootstrapConfirmed: true, + hardFailure: false, + hardFailureReason: undefined, + }, + }); + + const svc = new TeamProvisioningService(); + (svc as any).runtimeAdapterRunByTeam.set(teamName, { + runId: 'opencode-runtime-run', + providerId: 'opencode', + cwd: projectPath, + }); + (svc as any).aliveRunByTeam.set(teamName, 'opencode-runtime-run'); + + const snapshot = await svc.getTeamAgentRuntimeSnapshot(teamName); + + expect(snapshot.members.alice).toMatchObject({ + alive: true, + providerId: 'opencode', + runtimeModel: 'opencode/big-pickle', + }); + }); + it('excludes removed meta members from live runtime metadata resolution', async () => { const svc = new TeamProvisioningService(); (svc as any).configReader = { @@ -4842,6 +4876,97 @@ describe('TeamProvisioningService', () => { expect(result.teamLaunchState).toBe('partial_failure'); }); + it('marks persisted bootstrap as confirmed when member transcript shows successful member_briefing', async () => { + allowConsoleLogs(); + const teamName = 'zz-unit-bootstrap-transcript-success'; + const leadSessionId = 'lead-session'; + const memberSessionId = 'alice-session'; + const projectPath = '/Users/test/proj'; + const projectId = '-Users-test-proj'; + const acceptedAt = new Date(Date.now() - 5_000).toISOString(); + const successAt = new Date(Date.now() - 4_000).toISOString(); + + writeLaunchConfig(teamName, projectPath, leadSessionId, ['alice', 'bob']); + writeLaunchState(teamName, leadSessionId, { + alice: { + launchState: 'runtime_pending_bootstrap', + agentToolAccepted: true, + runtimeAlive: true, + bootstrapConfirmed: false, + hardFailure: false, + hardFailureReason: undefined, + firstSpawnAcceptedAt: acceptedAt, + }, + bob: { + launchState: 'starting', + agentToolAccepted: false, + runtimeAlive: false, + bootstrapConfirmed: false, + hardFailure: false, + hardFailureReason: undefined, + }, + }); + + const projectRoot = path.join(tempProjectsBase, projectId); + fs.mkdirSync(projectRoot, { recursive: true }); + fs.writeFileSync( + path.join(projectRoot, `${leadSessionId}.jsonl`), + `${JSON.stringify({ + timestamp: new Date(Date.now() - 10_000).toISOString(), + teamName, + type: 'user', + message: { role: 'user', content: 'Lead bootstrap context' }, + })}\n`, + 'utf8' + ); + fs.writeFileSync( + path.join(projectRoot, `${memberSessionId}.jsonl`), + [ + JSON.stringify({ + timestamp: acceptedAt, + teamName, + agentName: 'alice', + type: 'user', + message: { + role: 'user', + content: `You are bootstrapping into team "${teamName}" as member "alice".`, + }, + }), + JSON.stringify({ + timestamp: successAt, + teamName, + agentName: 'alice', + type: 'user', + message: { + role: 'user', + content: [ + { + type: 'tool_result', + tool_use_id: 'item_1', + content: `Member briefing for alice on team "${teamName}" (${teamName}).\nTask briefing for alice:\nNo actionable tasks.`, + is_error: false, + }, + ], + }, + }), + ].join('\n') + '\n', + 'utf8' + ); + + const svc = new TeamProvisioningService(); + (svc as any).getLiveTeamAgentNames = vi.fn(() => new Set(['alice'])); + + const result = await svc.getMemberSpawnStatuses(teamName); + + expect(result.statuses.alice).toMatchObject({ + status: 'online', + launchState: 'confirmed_alive', + bootstrapConfirmed: true, + runtimeAlive: true, + }); + expect(result.statuses.alice?.error).toBeUndefined(); + }); + it('marks an online teammate bootstrap as failed when transcript shows model unavailability', async () => { allowConsoleLogs(); const teamName = 'zz-live-bootstrap-model-unavailable'; @@ -4937,6 +5062,184 @@ describe('TeamProvisioningService', () => { expect(run.provisioningOutputParts.join('\n')).toContain('requested model is not available'); }); + it('marks a live teammate bootstrap as confirmed when transcript shows successful member_briefing', async () => { + allowConsoleLogs(); + const teamName = 'zz-live-bootstrap-transcript-success'; + const leadSessionId = 'lead-session'; + const memberSessionId = 'alice-session'; + const projectPath = '/Users/test/proj'; + const projectId = '-Users-test-proj'; + const acceptedAt = new Date(Date.now() - 5_000).toISOString(); + const successAt = new Date(Date.now() - 4_000).toISOString(); + + writeLaunchConfig(teamName, projectPath, leadSessionId, ['alice']); + + const projectRoot = path.join(tempProjectsBase, projectId); + fs.mkdirSync(projectRoot, { recursive: true }); + fs.writeFileSync( + path.join(projectRoot, `${memberSessionId}.jsonl`), + [ + JSON.stringify({ + timestamp: acceptedAt, + teamName, + agentName: 'alice', + type: 'user', + message: { + role: 'user', + content: `You are bootstrapping into team "${teamName}" as member "alice".`, + }, + }), + JSON.stringify({ + timestamp: successAt, + teamName, + agentName: 'alice', + type: 'assistant', + message: { + role: 'assistant', + content: [ + { + type: 'text', + text: `Bootstrap выполнен для \`alice\` в команде \`${teamName}\`.`, + }, + ], + }, + }), + ].join('\n') + '\n', + 'utf8' + ); + + const svc = new TeamProvisioningService(); + const run = { + runId: 'run-live-success-1', + teamName, + startedAt: new Date(Date.now() - 60_000).toISOString(), + request: { + members: [], + }, + expectedMembers: ['alice'], + memberSpawnStatuses: new Map([ + [ + 'alice', + { + status: 'waiting', + launchState: 'runtime_pending_bootstrap', + error: undefined, + updatedAt: acceptedAt, + runtimeAlive: true, + livenessSource: 'process', + bootstrapConfirmed: false, + hardFailure: false, + agentToolAccepted: true, + firstSpawnAcceptedAt: acceptedAt, + lastHeartbeatAt: undefined, + }, + ], + ]), + provisioningOutputParts: [], + activeToolCalls: new Map(), + isLaunch: false, + } as any; + + await (svc as any).reconcileBootstrapTranscriptSuccesses(run); + + expect(run.memberSpawnStatuses.get('alice')).toMatchObject({ + status: 'online', + launchState: 'confirmed_alive', + runtimeAlive: true, + bootstrapConfirmed: true, + }); + expect(run.provisioningOutputParts.join('\n')).toContain('bootstrap confirmed via transcript'); + }); + + it('marks a live teammate bootstrap as confirmed from transcript even when runtime discovery is stale', async () => { + allowConsoleLogs(); + const teamName = 'zz-live-bootstrap-transcript-success-without-runtime'; + const leadSessionId = 'lead-session'; + const memberSessionId = 'atlas-session'; + const projectPath = '/Users/test/proj'; + const projectId = '-Users-test-proj'; + const acceptedAt = new Date(Date.now() - 5_000).toISOString(); + const successAt = new Date(Date.now() - 4_000).toISOString(); + + writeLaunchConfig(teamName, projectPath, leadSessionId, ['atlas']); + + const projectRoot = path.join(tempProjectsBase, projectId); + fs.mkdirSync(projectRoot, { recursive: true }); + fs.writeFileSync( + path.join(projectRoot, `${memberSessionId}.jsonl`), + [ + JSON.stringify({ + timestamp: acceptedAt, + teamName, + agentName: 'atlas', + type: 'user', + message: { + role: 'user', + content: `You are bootstrapping into team "${teamName}" as member "atlas".`, + }, + }), + JSON.stringify({ + timestamp: successAt, + teamName, + agentName: 'atlas', + type: 'assistant', + message: { + role: 'assistant', + content: [ + { + type: 'text', + text: `Bootstrap выполнен для \`atlas\` в команде \`${teamName}\`.`, + }, + ], + }, + }), + ].join('\n') + '\n', + 'utf8' + ); + + const svc = new TeamProvisioningService(); + const run = { + runId: 'run-live-success-2', + teamName, + startedAt: new Date(Date.now() - 60_000).toISOString(), + request: { + members: [], + }, + expectedMembers: ['atlas'], + memberSpawnStatuses: new Map([ + [ + 'atlas', + { + status: 'waiting', + launchState: 'runtime_pending_bootstrap', + error: undefined, + updatedAt: acceptedAt, + runtimeAlive: false, + livenessSource: undefined, + bootstrapConfirmed: false, + hardFailure: false, + agentToolAccepted: true, + firstSpawnAcceptedAt: acceptedAt, + lastHeartbeatAt: undefined, + }, + ], + ]), + provisioningOutputParts: [], + activeToolCalls: new Map(), + isLaunch: false, + } as any; + + await (svc as any).reconcileBootstrapTranscriptSuccesses(run); + + expect(run.memberSpawnStatuses.get('atlas')).toMatchObject({ + status: 'online', + launchState: 'confirmed_alive', + runtimeAlive: false, + bootstrapConfirmed: true, + }); + expect(run.provisioningOutputParts.join('\n')).toContain('bootstrap confirmed via transcript'); + }); + it('marks a persisted online teammate bootstrap as failed when transcript shows model unavailability', async () => { allowConsoleLogs(); const teamName = 'zz-persisted-live-bootstrap-model-unavailable'; @@ -6136,6 +6439,144 @@ describe('TeamProvisioningService', () => { ); }); + it('recovers stale mixed secondary lanes from live OpenCode runtime reconcile before degrading them', async () => { + const teamName = 'relay-works-7'; + writeTeamMeta(teamName, { + providerId: 'codex', + providerBackendId: 'codex-native', + model: 'gpt-5.4', + }); + writeMembersMeta(teamName, [ + { + name: 'atlas', + providerId: 'opencode', + model: 'opencode/nemotron-3-super-free', + }, + { + name: 'bob', + providerId: 'codex', + model: 'gpt-5.4', + }, + { + name: 'nova', + providerId: 'codex', + model: 'gpt-5.4', + }, + { + name: 'tom', + providerId: 'opencode', + model: 'opencode/minimax-m2.5-free', + }, + ]); + writeLaunchConfig(teamName, '/Users/test/proj', 'lead-session', ['bob', 'nova']); + writeBootstrapState(teamName, [ + { name: 'bob', status: 'registered' }, + { name: 'nova', status: 'registered' }, + ]); + await upsertOpenCodeRuntimeLaneIndexEntry({ + teamsBasePath: tempTeamsBase, + teamName, + laneId: 'secondary:opencode:atlas', + state: 'active', + }); + await upsertOpenCodeRuntimeLaneIndexEntry({ + teamsBasePath: tempTeamsBase, + teamName, + laneId: 'secondary:opencode:tom', + state: 'active', + }); + + const adapterReconcile = vi.fn(async (input: Record) => { + const member = (input.expectedMembers as Array<{ name: string }>)[0]?.name; + return { + runId: String(input.runId), + teamName, + launchPhase: 'reconciled', + teamLaunchState: 'clean_success', + members: member + ? { + [member]: { + memberName: member, + providerId: 'opencode', + launchState: 'confirmed_alive', + agentToolAccepted: true, + runtimeAlive: true, + bootstrapConfirmed: true, + hardFailure: false, + diagnostics: ['bootstrap confirmed'], + }, + } + : {}, + snapshot: null, + warnings: [], + diagnostics: [], + }; + }); + const svc = new TeamProvisioningService(); + svc.setRuntimeAdapterRegistry( + new TeamRuntimeAdapterRegistry([ + { + providerId: 'opencode', + prepare: vi.fn(), + launch: vi.fn(), + reconcile: adapterReconcile, + stop: vi.fn(), + } as any, + ]) + ); + + const result = await svc.getMemberSpawnStatuses(teamName); + + expect(adapterReconcile).toHaveBeenCalledTimes(2); + expect(adapterReconcile).toHaveBeenCalledWith( + expect.objectContaining({ + teamName, + laneId: 'secondary:opencode:atlas', + reason: 'startup_recovery', + expectedMembers: [ + expect.objectContaining({ + name: 'atlas', + providerId: 'opencode', + cwd: '/Users/test/proj', + }), + ], + }) + ); + expect(adapterReconcile).toHaveBeenCalledWith( + expect.objectContaining({ + teamName, + laneId: 'secondary:opencode:tom', + reason: 'startup_recovery', + expectedMembers: [ + expect.objectContaining({ + name: 'tom', + providerId: 'opencode', + cwd: '/Users/test/proj', + }), + ], + }) + ); + expect(result.expectedMembers).toEqual(expect.arrayContaining(['atlas', 'bob', 'nova', 'tom'])); + expect(result.statuses.atlas).toMatchObject({ + status: 'online', + launchState: 'confirmed_alive', + }); + expect(result.statuses.tom).toMatchObject({ + status: 'online', + launchState: 'confirmed_alive', + }); + await expect(readOpenCodeRuntimeLaneIndex(tempTeamsBase, teamName)).resolves.toMatchObject({ + lanes: { + 'secondary:opencode:atlas': { + state: 'active', + }, + 'secondary:opencode:tom': { + state: 'active', + }, + }, + }); + }); + it('includes queued OpenCode secondary lanes in live spawn statuses before the final mixed snapshot settles', async () => { const svc = new TeamProvisioningService(); vi.spyOn(svc as any, 'refreshMemberSpawnStatusesFromLeadInbox').mockResolvedValue(undefined); @@ -6209,6 +6650,81 @@ describe('TeamProvisioningService', () => { }); }); + it('keeps finished OpenCode secondary lanes pending when runtime evidence has not materialized yet', async () => { + const svc = new TeamProvisioningService(); + vi.spyOn(svc as any, 'refreshMemberSpawnStatusesFromLeadInbox').mockResolvedValue(undefined); + vi.spyOn(svc as any, 'maybeAuditMemberSpawnStatuses').mockResolvedValue(undefined); + + const run = createMemberSpawnRun({ + teamName: 'mixed-live-finished-no-evidence', + runId: 'run-mixed-live-2', + expectedMembers: ['bob'], + memberSpawnStatuses: new Map([ + [ + 'bob', + createMemberSpawnStatusEntry({ + status: 'online', + launchState: 'confirmed_alive', + runtimeAlive: true, + bootstrapConfirmed: true, + livenessSource: 'heartbeat', + }), + ], + ]), + }); + run.isLaunch = true; + run.request = { + teamName: 'mixed-live-finished-no-evidence', + cwd: '/tmp/mixed-live-finished-no-evidence', + providerId: 'codex', + providerBackendId: 'codex-native', + model: 'gpt-5.4', + members: [], + }; + run.effectiveMembers = [ + { + name: 'bob', + providerId: 'codex', + model: 'gpt-5.4', + }, + ]; + run.mixedSecondaryLanes = [ + { + laneId: 'secondary:opencode:atlas', + providerId: 'opencode', + member: { + name: 'atlas', + providerId: 'opencode', + model: 'opencode/nemotron-3-super-free', + }, + runId: 'lane-run-atlas', + state: 'finished', + result: null, + warnings: [], + diagnostics: [], + }, + ]; + run.detectedSessionId = 'lead-session'; + + (svc as any).runs.set(run.runId, run); + (svc as any).provisioningRunByTeam.set(run.teamName, run.runId); + + const result = await svc.getMemberSpawnStatuses(run.teamName); + + expect(result.teamLaunchState).toBe('partial_pending'); + expect(result.expectedMembers).toEqual(expect.arrayContaining(['bob', 'atlas'])); + expect(result.statuses.bob).toMatchObject({ + status: 'online', + launchState: 'confirmed_alive', + }); + expect(result.statuses.atlas).toMatchObject({ + status: 'waiting', + launchState: 'runtime_pending_bootstrap', + hardFailure: false, + hardFailureReason: undefined, + }); + }); + it('includes queued OpenCode secondary lanes in live spawn statuses during createTeam runs', async () => { const svc = new TeamProvisioningService(); vi.spyOn(svc as any, 'refreshMemberSpawnStatusesFromLeadInbox').mockResolvedValue(undefined);