import { AgentAttachmentError, buildClaudeAttachmentDeliveryParts, buildCodexNativeAttachmentDeliveryParts, buildOpenCodeAttachmentDeliveryParts, type CodexNativeImageArgPart, type OpenCodeFilePart, } from '@features/agent-attachments/main'; import { resolveAnthropicEffortSupport, resolveAnthropicFastMode, resolveAnthropicRuntimeSelection, } from '@features/anthropic-runtime-profile/main'; import { buildCodexFastModeArgs, resolveCodexFastMode, resolveCodexRuntimeSelection, } from '@features/codex-runtime-profile/main'; import { buildPlannedMemberLaneIdentity, isMixedOpenCodeSideLanePlan, type TeamRuntimeLanePlan, } from '@features/team-runtime-lanes'; import { createTeamRuntimeLaneCoordinator } from '@features/team-runtime-lanes/main'; import { killTmuxPaneForCurrentPlatformSync, listRuntimeProcessTableForCurrentPlatform, listTmuxPanePidsForCurrentPlatform, listTmuxPaneRuntimeInfoForCurrentPlatform, type RuntimeProcessTableRow, sendKeysToTmuxPaneForCurrentPlatform, type TmuxPaneRuntimeInfo, } from '@features/tmux-installer/main'; import { applyWorkspaceTrustLaunchArgPatches, budgetWorkspaceTrustDiagnosticsManifest, buildWorkspaceTrustPathCandidates, buildWorkspaceTrustPreflightEnv, resolveWorkspaceTrustCanonicalGitRoot, resolveWorkspaceTrustFeatureFlags, resolveWorkspaceTrustFilesystemGitRoot, type WorkspaceTrustArgsOnlyPlanRequest, type WorkspaceTrustArgsOnlyPlanResult, type WorkspaceTrustCoordinator, type WorkspaceTrustDiagnosticsManifest, type WorkspaceTrustExecutionResult, type WorkspaceTrustFeatureFlags, type WorkspaceTrustFullPlanRequest, type WorkspaceTrustFullPlanResult, type WorkspaceTrustLaunchArgPatch, type WorkspaceTrustLaunchArgTargetSurface, type WorkspaceTrustProvider, type WorkspaceTrustWorkspace, } from '@features/workspace-trust/main'; import { ConfigManager } from '@main/services/infrastructure/ConfigManager'; import { NotificationManager } from '@main/services/infrastructure/NotificationManager'; import { notifyTeamWatchScopeChanged } from '@main/services/infrastructure/teamWatchScope'; import { prepareAgentChildProcessWritableEnv } from '@main/services/runtime/agentChildProcessPreflight'; import { getAppIconPath } from '@main/utils/appIcon'; import { execCli, killProcessTree, killTrackedCliProcesses, spawnCli, } from '@main/utils/childProcess'; import { FileReadTimeoutError, readFileUtf8WithTimeout } from '@main/utils/fsRead'; import { encodePath, extractBaseDir, getAutoDetectedClaudeBasePath, getClaudeBasePath, getHomeDir, getProjectsBasePath, getTasksBasePath, getTeamsBasePath, } from '@main/utils/pathDecoder'; import { isPathWithinRoot } from '@main/utils/pathValidation'; import { isProcessAlive } from '@main/utils/processHealth'; import { killProcessByPid } from '@main/utils/processKill'; import { resolveInteractiveShellEnvBestEffort } from '@main/utils/shellEnv'; import { shouldAutoAllow } from '@main/utils/toolApprovalRules'; import { listWindowsProcessTable, listWindowsProcessTableSync, } from '@main/utils/windowsProcessTable'; import { stripAgentBlocks, wrapAgentBlock } from '@shared/constants/agentBlocks'; import { CROSS_TEAM_SENT_SOURCE, CROSS_TEAM_SOURCE, parseCrossTeamPrefix, stripCrossTeamPrefix, } from '@shared/constants/crossTeam'; import { getMemberColorByName } from '@shared/constants/memberColors'; import { type AttachmentMeta, type AttachmentPayload, DEFAULT_TOOL_APPROVAL_SETTINGS, } from '@shared/types/team'; import { resolveLanguageName } from '@shared/utils/agentLanguage'; import { resolveAnthropicLaunchModel } from '@shared/utils/anthropicLaunchModel'; import { getAnthropicDefaultTeamModel } from '@shared/utils/anthropicModelDefaults'; import { parseCliArgs } from '@shared/utils/cliArgsParser'; import { isUsableCodexModelCatalog } from '@shared/utils/codexModelCatalog'; import { deriveContextMetrics, inferContextWindowTokens } from '@shared/utils/contextMetrics'; import { isTeamEffortLevel } from '@shared/utils/effortLevels'; import { getErrorMessage } from '@shared/utils/errorHandling'; import { isInboxNoiseMessage, isMeaningfulBootstrapCheckInMessage, type ParsedPermissionRequest, parsePermissionRequest, } from '@shared/utils/inboxNoise'; import { isLeadAgentType, isLeadMember } from '@shared/utils/leadDetection'; import { createLogger } from '@shared/utils/logger'; import { isOpenCodeWindowsAccessDeniedDiagnostic, normalizeOpenCodeWindowsAccessDeniedDiagnostic, } from '@shared/utils/openCodeWindowsAccessDenied'; import { migrateProviderBackendId } from '@shared/utils/providerBackend'; import { isDefaultProviderModelSelection } from '@shared/utils/providerModelSelection'; import { formatTaskDisplayLabel } from '@shared/utils/taskIdentity'; import { isTeamInternalControlMessageText, stripExactInternalControlEchoPrefix, } from '@shared/utils/teamInternalControlMessages'; import { hasUnsafeProvisionedButNotAliveRuntimeEvidence } from '@shared/utils/teamLaunchFailureReason'; import { parseAllTeammateMessages, type ParsedTeammateContent, } from '@shared/utils/teammateMessageParser'; import { buildTeamMemberColorMap } from '@shared/utils/teamMemberColors'; import { buildTeamMemberMcpSettingSources, normalizeTeamMemberMcpPolicy, requiresStrictTeamMemberMcpConfig, } from '@shared/utils/teamMemberMcpPolicy'; import { createCliAutoSuffixNameGuard, parseNumericSuffixName } from '@shared/utils/teamMemberName'; import { inferTeamProviderIdFromModel, normalizeOptionalTeamProviderId, } from '@shared/utils/teamProvider'; import { extractToolPreview, extractToolResultPreview, formatToolSummaryFromCalls, parseAgentToolResultStatus, } from '@shared/utils/toolSummary'; import * as agentTeamsControllerModule from 'agent-teams-controller'; import { type ChildProcess, execFile, execFileSync, type spawn } from 'child_process'; import { randomUUID } from 'crypto'; import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; import pidusage from 'pidusage'; import * as readline from 'readline'; import { ANTHROPIC_HELPER_MODE_COMPETING_AUTH_ENV_KEYS, type AnthropicTeamApiKeyHelperMaterial, cleanupAnthropicTeamApiKeyHelperForTeam, cleanupAnthropicTeamApiKeyHelperMaterial, cleanupStaleAnthropicTeamApiKeyHelpers, DISABLE_ANTHROPIC_TEAM_API_KEY_HELPER_ENV, materializeAnthropicTeamApiKeyHelper, verifyAnthropicTeamApiKeyHelperMaterial, } from '../runtime/anthropicTeamApiKeyHelper'; import { mergeJsonSettingsArgs, parseJsonSettingsObject } from '../runtime/cliSettingsArgs'; import { type GeminiRuntimeAuthState, resolveGeminiRuntimeAuth, } from '../runtime/geminiRuntimeAuth'; import { buildProviderAwareCliEnv } from '../runtime/providerAwareCliEnv'; import { ProviderConnectionService } from '../runtime/ProviderConnectionService'; import { buildProviderPreflightPingArgs, getProviderModelProbeExpectedOutput, getProviderModelProbeTimeoutMs, normalizeProviderModelProbeFailureReason, } from '../runtime/providerModelProbe'; import { resolveTeamProviderId } from '../runtime/providerRuntimeEnv'; import { materializeTeamRuntimeSettingsBundle, splitSettingsJsonArgs, type TeamRuntimeSettingsJson, } from '../runtime/teamRuntimeSettingsBundle'; import { parseBootstrapRuntimeProofDetail, validateBootstrapRuntimeProofEnvelope, } from './bootstrap/BootstrapProofValidation'; import { buildNativeAppManagedBootstrapSpecs, buildNativeAppManagedBootstrapSpecsWithDiagnostics, type NativeAppManagedBootstrapSpec, } from './bootstrap/NativeAppManagedBootstrapContextBuilder'; import { getSystemLocale } from './provisioning/TeamProvisioningAgentLanguage'; import { buildDeterministicCreateBootstrapSpec, buildDeterministicLaunchBootstrapSpec, getProvisioningRunTimeoutMs, removeDeterministicBootstrapSpecFile, removeDeterministicBootstrapUserPromptFile, type RuntimeBootstrapMemberMcpLaunchConfig, writeDeterministicBootstrapSpecFile, writeDeterministicBootstrapUserPromptFile, } from './provisioning/TeamProvisioningBootstrapSpec'; import { buildCliExitFailurePresentation, buildCombinedLogs, } from './provisioning/TeamProvisioningCliExitPresentation'; import { buildDirectTmuxRestartCommand, hasAnthropicCompatibleAuthTokenEnv, isInteractiveShellCommand, } from './provisioning/TeamProvisioningDirectRestart'; import { assertDeterministicBootstrapPrimaryMemberLimit, assertOpenCodeNotLaunchedThroughLegacyProvisioning, buildLargeDeterministicBootstrapWarning, getMixedLaunchFallbackRecoveryError, isPureOpenCodeProvisioningRequest, mergeProvisioningWarnings, type TeamLaunchCompatibilityReport, } from './provisioning/TeamProvisioningLaunchCompatibility'; import { buildLaunchDiagnosticsFromRun, buildWorkspaceTrustPreflightLaunchDiagnostic, mentionsProcessTableUnavailable, mergeLaunchDiagnosticItem, } from './provisioning/TeamProvisioningLaunchDiagnostics'; import { deriveMemberLaunchState, isAutoClearableLaunchFailureReason, isCliProvisionedButNotAliveFailureReason, isNeverSpawnedDuringLaunchReason, isProvisionedButNotAliveFailureReason, } from './provisioning/TeamProvisioningLaunchFailurePolicy'; import { isOpenCodeOverlayMemberRemoved, matchesExactTeamMemberName, matchesMemberNameOrBase, matchesObservedMemberNameForExpected, matchesTeamMemberIdentity, namesMatchCaseInsensitive, } from './provisioning/TeamProvisioningMemberIdentity'; import { buildRestartDuplicateUnconfirmedReason, buildRestartGraceTimeoutReason, buildRestartStillRunningReason, createInitialMemberSpawnStatusEntry, deriveTaskActivityPauseAt, deriveTaskActivityResumeAt, MEMBER_LAUNCH_GRACE_MS, parseOptionalIsoMs, shouldWarnOnMissingRegisteredMember, shouldWarnOnUnreadableMemberAuditConfig, summarizeMemberSpawnStatusRecord, } from './provisioning/TeamProvisioningMemberSpawnStatusPolicy'; import { buildEffectiveTeamMemberSpec, buildEffectiveTeamMemberSpecs, getExplicitLaunchModelSelection, normalizeTeamMemberProviderId, normalizeTeamProviderLike, teamRequestIncludesCodexMember, } from './provisioning/TeamProvisioningMemberSpecs'; import { boundOpenCodeAppManagedBriefingText, filterStaleOpenCodeOverlayDiagnostics, hasRealOpenCodeFailureDiagnostic, hasRealOpenCodeLaunchDiagnostic, hasStaleOpenCodeDiagnostics, hasStaleOpenCodeSecondaryLaunchDiagnostic, isFileLockTimeoutError, isPersistedOpenCodeSecondaryLaneMember, promoteOpenCodePersistedFailureReasonsFromDiagnostics, } from './provisioning/TeamProvisioningOpenCodeDiagnosticsPolicy'; import { appendDiagnosticOnce, buildOpenCodeSecondaryLaneTimingDiagnostic, buildOpenCodeUncommittedBootstrapDiagnostic, collectOpenCodeSecondaryLaneFailureDiagnostics, createUnexpectedMixedSecondaryLaneFailureResult, downgradeUncommittedOpenCodeBootstrapEvidence, getOpenCodeSecondaryBootstrapStallDiagnosticFromPersisted, hasOpenCodeRuntimeEntryHandle, hasOpenCodeRuntimeHandle, hasOpenCodeRuntimeLivenessMarker, isBootstrapMemberEvidenceCurrentForMember, isDefinitiveOpenCodePreLaunchFailure, isExplicitLegacyOpenCodeBootstrap, isMaterializedOpenCodeSessionId, isRecoverableOpenCodeBootstrapPendingLaunchResult, isRecoverableOpenCodeRuntimeEvidence, isRecoverablePersistedOpenCodeRuntimeCandidate, isRecoverablePersistedOpenCodeTerminalRuntimeCandidate, MEMBER_BOOTSTRAP_STALL_MS, normalizeIsoTimestamp, normalizeRecoverableOpenCodeBootstrapPendingLaunchResult, OPENCODE_APP_MANAGED_BOOTSTRAP_STALLED_DIAGNOSTIC, promoteCommittedOpenCodeAppManagedBootstrapEvidence, resolveOpenCodeBootstrapAcceptedAt, selectOpenCodeSecondaryBootstrapStallDiagnostic, shouldMarkPersistedOpenCodeBootstrapStalled, summarizeRuntimeLaunchResultMembers, } from './provisioning/TeamProvisioningOpenCodeRuntimeEvidencePolicy'; import { buildDeterministicLaunchHydrationPrompt, buildGeminiPostLaunchHydrationPrompt, buildLeadRosterContextBlock, buildMemberSpawnPrompt, buildPersistentLeadContext, buildRestartMemberSpawnMessage, buildTaskBoardSnapshot, extractBootstrapFailureReason, extractHeartbeatTimestamp, extractTranscriptMessageText, getBootstrapTranscriptSuccessSourceFromNormalized, getCanonicalSendMessageFieldRule, getCanonicalSendMessageToolRule, isTaskBoardSnapshotWorkCandidate, normalizeMemberDiagnosticText, shouldUseGeminiStagedLaunch, } from './provisioning/TeamProvisioningPromptBuilders'; import { buildRuntimeLaunchWarning, getAnthropicFastModeDefault, getConfiguredRuntimeBackend, getPromptSizeSummary, getTeamProviderLabel, logRuntimeLaunchSnapshot, } from './provisioning/TeamProvisioningRuntimeDiagnostics'; import type { RuntimeTurnSettledProvider } from '@features/member-work-sync/main'; export type { RuntimeBootstrapMemberMcpLaunchConfig } from './provisioning/TeamProvisioningBootstrapSpec'; export { buildDirectTmuxRestartEnvAssignments } from './provisioning/TeamProvisioningDirectRestart'; export { getMixedLaunchFallbackRecoveryError, getOpenCodeMixedProviderProvisioningError, } from './provisioning/TeamProvisioningLaunchCompatibility'; export { shouldWarnOnMissingRegisteredMember, shouldWarnOnUnreadableMemberAuditConfig, } from './provisioning/TeamProvisioningMemberSpawnStatusPolicy'; export { buildAddMemberSpawnMessage, buildRestartMemberSpawnMessage, } from './provisioning/TeamProvisioningPromptBuilders'; import { openCodeRuntimeApprovalProvider } from './approvals/OpenCodeRuntimeApprovalProvider'; import { RuntimeToolApprovalCoordinator, type RuntimeToolApprovalEntry, } from './approvals/RuntimeToolApprovalCoordinator'; import { isOpenCodeBridgeNoOutputDiagnostic } from './opencode/bridge/OpenCodeBridgeSupportDiagnostics'; import { buildOpenCodePromptDeliveryAttemptId, createOpenCodePromptDeliveryLedgerStore, hashOpenCodePromptDeliveryPayload, isOpenCodePromptDeliveryAttemptDue, isOpenCodePromptResponseStateResponded, isOpenCodeResolvedBehaviorChangedReason, isOpenCodeSessionRefreshResponseState, isOpenCodeSessionTransportChangedReason, OPENCODE_PROMPT_DELIVERY_SESSION_REFRESH_MAX_ATTEMPTS, type OpenCodePromptDeliveryLedgerRecord, type OpenCodePromptDeliveryLedgerStore, type OpenCodePromptDeliveryStatus, } from './opencode/delivery/OpenCodePromptDeliveryLedger'; import { decideOpenCodePromptDeliveryRepair, type OpenCodePromptDeliveryHardFailureKind, } from './opencode/delivery/OpenCodePromptDeliveryRepairPolicy'; import { isOpenCodePromptDeliveryObserveLaterResponseState, isOpenCodePromptDeliveryRetryableResponseState, isOpenCodePromptDeliveryRetryAttemptDue, isOpenCodeVisibleReplyReadCommitAllowed, isOpenCodeVisibleReplySemanticallySufficient, OPENCODE_PROMPT_DELIVERY_OBSERVE_DELAY_MS, OPENCODE_PROMPT_DELIVERY_RETRY_DELAY_MS, OPENCODE_PROMPT_WATCHDOG_GLOBAL_CONCURRENCY, OPENCODE_PROMPT_WATCHDOG_PER_TEAM_CONCURRENCY, type OpenCodeVisibleReplyProof, } from './opencode/delivery/OpenCodePromptDeliveryWatchdog'; import { classifyOpenCodeRuntimeDeliveryReasonCode, decideOpenCodeRuntimeDeliveryAdvisory, isDeferredGenericOpenCodeRuntimeDeliveryReason, isPotentialOpenCodeRuntimeDeliveryError, type OpenCodeRuntimeDeliveryAdvisoryDecision, toOpenCodeRuntimeDeliveryUserVisibleImpact, } from './opencode/delivery/OpenCodeRuntimeDeliveryAdvisoryPolicy'; import { selectOpenCodeRuntimeDeliveryReason } from './opencode/delivery/OpenCodeRuntimeDeliveryDiagnostics'; import { getOpenCodeVisibleReplyInboxCandidates as resolveOpenCodeVisibleReplyInboxCandidates, isOpenCodeLeadReplyRecipientAlias as isOpenCodeLeadReplyRecipientAliasValue, isOpenCodeRecoveredVisibleReplyCandidate as isOpenCodeRecoveredVisibleReplyCandidateValue, isOpenCodeVisibleReplyTimestampEligible as isOpenCodeVisibleReplyTimestampEligibleValue, normalizeOpenCodeTaskRefsForComparison as normalizeOpenCodeTaskRefsForComparisonValue, openCodeTaskRefKey as openCodeTaskRefKeyValue, openCodeTaskRefsIncludeAll as openCodeTaskRefsIncludeAllValue, } from './opencode/delivery/OpenCodeRuntimeDeliveryProofMatching'; import { OpenCodeRuntimeDeliveryProofReader } from './opencode/delivery/OpenCodeRuntimeDeliveryProofReader'; import { inferOpenCodeTaskRefsFromInboxMessage } from './opencode/delivery/OpenCodeRuntimeDeliveryTaskRefInference'; import { createRuntimeDeliveryJournalStore } from './opencode/delivery/RuntimeDeliveryJournal'; import { type RuntimeDeliveryDestinationPort, RuntimeDeliveryDestinationRegistry, RuntimeDeliveryReconciler, RuntimeDeliveryService, } from './opencode/delivery/RuntimeDeliveryService'; import { clearOpenCodeRuntimeLaneStorage, getOpenCodeLaneScopedRuntimeFilePath, getOpenCodeRuntimeManifestPath, getOpenCodeRuntimeRunTombstonesPath, inspectOpenCodeRuntimeLaneStorage, migrateLegacyOpenCodeRuntimeState, OpenCodeRuntimeManifestEvidenceReader, prepareOpenCodeRuntimeLaneForLaunchGeneration, readCommittedOpenCodeBootstrapSessionEvidence, readOpenCodeRuntimeLaneIndex, recoverStaleOpenCodeRuntimeLaneIndexEntry, setOpenCodeRuntimeActiveRunManifest, upsertOpenCodeRuntimeLaneIndexEntry, } from './opencode/store/OpenCodeRuntimeManifestEvidenceReader'; import { createRuntimeRunTombstoneStore, type RuntimeEvidenceKind, RuntimeStaleEvidenceError, } from './opencode/store/RuntimeRunTombstoneStore'; import { createRuntimeStoreManifestStore, createRuntimeStoreReceiptStore, OPENCODE_RUNTIME_STORE_DESCRIPTORS, RuntimeStoreBatchWriter, } from './opencode/store/RuntimeStoreManifest'; import { OpenCodeTaskLogAttributionStore } from './taskLogs/stream/OpenCodeTaskLogAttributionStore'; import { getCurrentAgentTeamsMcpHttpTransportEvidence } from './AgentTeamsMcpHttpServer'; import { isAgentTeamsToolUse } from './agentTeamsToolNames'; import { atomicWriteAsync } from './atomicWrite'; import { peekAutoResumeService } from './AutoResumeService'; import { ClaudeBinaryResolver } from './ClaudeBinaryResolver'; import { getCliFlavorUiOptions, getConfiguredCliCommandLabel, getConfiguredCliFlavor, } from './cliFlavor'; import { withFileLock } from './fileLock'; import { type ClassifiedMainProcessIdle, classifyIdleNotificationForMainProcess, } from './idleNotificationMainProcessSemantics'; import { withInboxLock } from './inboxLock'; import { getEffectiveInboxMessageId } from './inboxMessageIdentity'; import { buildProcessBootstrapPendingDiagnostic, buildProcessBootstrapTimeoutDiagnostic, deriveProcessTransportProjectionPhase, type ProcessBootstrapTransportEvent, type ProcessBootstrapTransportSummary, sanitizeProcessRuntimeEventFilePrefix, summarizeProcessBootstrapTransportEvents, } from './ProcessBootstrapTransportEvidence'; import { boundLaunchDiagnostics, buildProgressLiveOutput, buildProgressLogsTail, buildProgressTraceLine, } from './progressPayload'; import { applyDesktopTeammateModeDecisionToEnv, buildDesktopTeammateModeCliArgs, resolveDesktopTeammateModeDecision, } from './runtimeTeammateMode'; import { TeamAttachmentStore } from './TeamAttachmentStore'; import { choosePreferredLaunchSnapshot, clearBootstrapState, readBootstrapLaunchSnapshot, readBootstrapRealTaskSubmissionState, readBootstrapRuntimeState, } from './TeamBootstrapStateReader'; import { TeamConfigReader } from './TeamConfigReader'; import { TeamInboxReader } from './TeamInboxReader'; import { TeamInboxWriter } from './TeamInboxWriter'; import { isWorkspaceTrustLaunchFailureText, writeTeamLaunchFailureArtifactPack, } from './TeamLaunchFailureArtifactPack'; import { createPersistedLaunchSnapshot, deriveTeamLaunchAggregateState, hasMixedPersistedLaunchMetadata, snapshotFromRuntimeMemberStatuses, snapshotToMemberSpawnStatuses, } from './TeamLaunchStateEvaluator'; import { TeamLaunchStateStore } from './TeamLaunchStateStore'; import { TeamMcpConfigBuilder } from './TeamMcpConfigBuilder'; import { TeamMemberLogsFinder } from './TeamMemberLogsFinder'; import { TeamMembersMetaStore } from './TeamMembersMetaStore'; import { TeamMemberWorktreeManager } from './TeamMemberWorktreeManager'; import { TeamMetaStore } from './TeamMetaStore'; import { commandArgEquals, extractCliArgValues, isStrongRuntimeEvidence, resolveTeamMemberRuntimeLiveness, sanitizeProcessCommandForDiagnostics, } from './TeamRuntimeLivenessResolver'; import { TeamSentMessagesStore } from './TeamSentMessagesStore'; import { TeamTaskActivityIntervalService } from './TeamTaskActivityIntervalService'; import { TeamTaskReader } from './TeamTaskReader'; import { TeamTranscriptProjectResolver } from './TeamTranscriptProjectResolver'; import type { OpenCodeCommittedBootstrapSessionRecord, OpenCodeRuntimeLaneIndexEntry, } from './opencode/store/OpenCodeRuntimeManifestEvidenceReader'; import type { OpenCodeTeamRuntimeMessageInput, OpenCodeTeamRuntimeMessageResult, TeamLaunchRuntimeAdapter, TeamRuntimeAdapterRegistry, TeamRuntimeLaunchInput, TeamRuntimeLaunchResult, TeamRuntimeMemberLaunchEvidence, TeamRuntimeMemberSpec, TeamRuntimePendingPermission, TeamRuntimePermissionListResult, TeamRuntimePrepareResult, TeamRuntimeStopInput, } from './runtime'; type OpenCodeRuntimeMessageAdapter = TeamLaunchRuntimeAdapter & { sendMessageToMember( input: OpenCodeTeamRuntimeMessageInput ): Promise; observeMessageDelivery?( input: OpenCodeTeamRuntimeMessageInput & { prePromptCursor?: string | null; sessionId?: string; runtimePromptMessageId?: string; } ): Promise; }; type OpenCodeRuntimePermissionListingAdapter = TeamLaunchRuntimeAdapter & { listRuntimePermissions(input: { teamName: string; laneId: string; cwd: string; memberName?: string; sessionId?: string | null; }): Promise; }; /** * Kill a team CLI process using SIGKILL (uncatchable). * * Newer Claude CLI versions (≥2.1.x) handle SIGTERM gracefully and run cleanup * that deletes team files (config.json, inboxes/, tasks/). SIGKILL prevents this. * * ALWAYS use this instead of killProcessTree() for team processes. * stdin.end() is also forbidden — EOF triggers the same cleanup. */ function killTeamProcess(child: ChildProcess | null | undefined): void { killProcessTree(child, 'SIGKILL'); } function buildRelayInboxView(messages: RelayInboxMessage[]): RelayInboxMessageView[] { return messages.map((message) => { const isCrossTeamLike = message.source === CROSS_TEAM_SOURCE || message.source === CROSS_TEAM_SENT_SOURCE; return { message, idle: isCrossTeamLike ? null : classifyIdleNotificationForMainProcess(message.text), isCoarseNoise: isCrossTeamLike ? false : isInboxNoiseMessage(message.text), }; }); } interface PersistedRuntimeMemberLike { name?: string; agentId?: string; tmuxPaneId?: string; backendType?: string; providerId?: string; cwd?: string; bootstrapExpectedAfter?: string; bootstrapProofToken?: string; bootstrapRunId?: string; bootstrapProofMode?: string; bootstrapContextHash?: string; bootstrapBriefingHash?: string; bootstrapRuntimeEventsPath?: string; runtimePid?: number; runtimeSessionId?: string; } interface PersistedTeamConfigCacheEntry { path: string; size: number; mtimeMs: number; ctimeMs: number; projectPath: string | null; members: PersistedRuntimeMemberLike[]; } type RelayInboxMessage = InboxMessage & { messageId: string }; interface RelayInboxMessageView { message: RelayInboxMessage; idle: ClassifiedMainProcessIdle | null; isCoarseNoise: boolean; } interface OpenCodeRuntimeControlAck { ok: true; providerId: 'opencode'; teamName: string; runId: string; state: 'accepted' | 'delivered' | 'duplicate' | 'recorded'; memberName?: string; runtimeSessionId?: string; idempotencyKey?: string; location?: unknown; diagnostics: string[]; observedAt: string; } interface LaunchStateWriteResult { snapshot: PersistedTeamLaunchSnapshot; wrote: boolean; } type BootstrapTranscriptSuccessSource = 'member_briefing' | 'assistant_text'; const BOOTSTRAP_RUNTIME_PROOF_TAIL_BYTES = 256 * 1024; const BOOTSTRAP_RUNTIME_EVENT_MAX_LINES = 256; const BOOTSTRAP_RUNTIME_EVENT_MAX_LINE_BYTES = 16 * 1024; const TEAMMATE_RUNTIME_ENV = 'CLAUDE_CODE_TEAMMATE_RUNTIME'; const TEAMMATE_RUNTIME_EVENTS_ENV = 'CLAUDE_CODE_TEAMMATE_RUNTIME_EVENTS_PATH'; const TEAMMATE_BOOTSTRAP_PROOF_TOKEN_ENV = 'CLAUDE_CODE_BOOTSTRAP_PROOF_TOKEN'; const NATIVE_APP_MANAGED_BOOTSTRAP_CONTEXT_ENV = 'CLAUDE_CODE_NATIVE_APP_MANAGED_BOOTSTRAP_CONTEXT_PATH'; function getTeamRuntimeEventsDir(teamName: string): string { return path.join(getTeamsBasePath(), teamName, 'runtime'); } function isProcessBootstrapTransportDiagnostic(value: unknown): value is string { return ( typeof value === 'string' && (value.startsWith('Bootstrap transport ') || value.includes('Last transport stage:') || value.startsWith('bootstrap submit ') || value.startsWith('runtime failed') || value.startsWith('runtime exited')) ); } function realpathIfExists(inputPath: string): string | null { try { return fs.realpathSync.native(inputPath); } catch { return null; } } function isContainedTeamRuntimeEventsPath(teamName: string, candidatePath: string): boolean { const runtimeDir = getTeamRuntimeEventsDir(teamName); const resolvedRuntimeDir = path.resolve(runtimeDir); const resolvedCandidate = path.resolve(candidatePath); if (!isPathWithinRoot(resolvedCandidate, resolvedRuntimeDir)) { return false; } const realRuntimeDir = realpathIfExists(resolvedRuntimeDir); const realCandidate = realpathIfExists(resolvedCandidate); if (realCandidate) { return isPathWithinRoot(realCandidate, realRuntimeDir ?? resolvedRuntimeDir); } return true; } type BootstrapTranscriptOutcome = | { kind: 'success'; observedAt: string; source: BootstrapTranscriptSuccessSource; } | { kind: 'failure'; observedAt: string; reason: string; }; interface BootstrapTranscriptOutcomeCacheEntry { mtimeMs: number; size: number; outcome: BootstrapTranscriptOutcome | null; } interface BootstrapTranscriptOutcomeLookupCacheEntry { expiresAtMs: number; outcome: BootstrapTranscriptOutcome | null; } interface BootstrapTranscriptOutcomeCandidate { text: string; // text.replace(/\s+/g,' ').trim().toLowerCase(), computed once and reused across // members so success/context detection does not re-normalize the same line. normalizedText: string; observedAt: string; parsedAgentName: string | null; // The shared parsed-tail line this candidate was built from. Used to memoize the // pure extractBootstrapFailureReason() result on the line itself so an N-member // team extracts each line's failure reason at most once instead of once per member. parsedLine: ParsedBootstrapTranscriptTailLine; } interface ParsedBootstrapTranscriptTailLine { rawTimestamp: string | null; timestampMs: number; text: string | null; normalizedText: string | null; parsedAgentName: string | null; // Memoized extractBootstrapFailureReason(text): undefined = not computed yet, // null = computed/no failure reason, string = the failure reason. Lives as long as // this line's parse-cache entry (filePath + mtime + size); a file change re-parses // into fresh line objects, so the memo cannot drift from the line's text. bootstrapFailureReason?: string | null; bootstrapContextCandidateByTeam?: Map; bootstrapContextMemberMatchByName?: Map; bootstrapSuccessSourceByTeamMember?: Map; } interface ParsedBootstrapTranscriptTailCacheEntry { mtimeMs: number; size: number; lines: ParsedBootstrapTranscriptTailLine[]; } function isNormalizedBootstrapTranscriptContextCandidateText( normalizedText: string, normalizedTeamName: string ): boolean { if (!normalizedText || !normalizedTeamName) { return false; } if (!normalizedText.includes(normalizedTeamName)) { return false; } return ( normalizedText.includes('bootstrap') || normalizedText.includes('bootstrapping') || normalizedText.includes('member briefing') || normalizedText.includes('task briefing') ); } function isNormalizedBootstrapTranscriptContextMemberText( normalizedText: string, normalizedMemberName: string ): boolean { return !!normalizedMemberName && normalizedText.includes(normalizedMemberName); } function getCachedBootstrapContextCandidateForLine( line: ParsedBootstrapTranscriptTailLine, normalizedText: string, normalizedTeamName: string ): boolean { let candidateByTeam = line.bootstrapContextCandidateByTeam; if (!candidateByTeam) { candidateByTeam = new Map(); line.bootstrapContextCandidateByTeam = candidateByTeam; } const cached = candidateByTeam.get(normalizedTeamName); if (cached !== undefined) { return cached; } const value = isNormalizedBootstrapTranscriptContextCandidateText( normalizedText, normalizedTeamName ); candidateByTeam.set(normalizedTeamName, value); return value; } function getCachedBootstrapContextMemberMatchForLine( line: ParsedBootstrapTranscriptTailLine, normalizedText: string, normalizedMemberName: string ): boolean { let matchByName = line.bootstrapContextMemberMatchByName; if (!matchByName) { matchByName = new Map(); line.bootstrapContextMemberMatchByName = matchByName; } const cached = matchByName.get(normalizedMemberName); if (cached !== undefined) { return cached; } const value = isNormalizedBootstrapTranscriptContextMemberText( normalizedText, normalizedMemberName ); matchByName.set(normalizedMemberName, value); return value; } function getCachedBootstrapSuccessSourceForLine( line: ParsedBootstrapTranscriptTailLine, normalizedText: string, normalizedTeamName: string, normalizedMemberName: string ): BootstrapTranscriptSuccessSource | null { let sourceByTeamMember = line.bootstrapSuccessSourceByTeamMember; if (!sourceByTeamMember) { sourceByTeamMember = new Map(); line.bootstrapSuccessSourceByTeamMember = sourceByTeamMember; } const cacheKey = `${normalizedTeamName}\0${normalizedMemberName}`; if (sourceByTeamMember.has(cacheKey)) { return sourceByTeamMember.get(cacheKey) ?? null; } const source = getBootstrapTranscriptSuccessSourceFromNormalized( normalizedText, normalizedTeamName, normalizedMemberName ); sourceByTeamMember.set(cacheKey, source); return source; } import type { ActiveToolCall, AgentActionMode, CliProviderModelCatalog, CliProviderRuntimeCapabilities, CliProviderStatus, CrossTeamSendResult, EffortLevel, InboxMessage, LeadContextUsage, MemberLaunchState, MemberSpawnLivenessSource, MemberSpawnStatus, MemberSpawnStatusEntry, MemberSpawnStatusesSnapshot, OpenCodeAppManagedBootstrapCandidate, OpenCodeBootstrapEvidenceSource, OpenCodeRuntimeDeliveryStatus, OpenCodeRuntimeDeliveryUserVisibleImpact, PersistedTeamLaunchMemberState, PersistedTeamLaunchPhase, PersistedTeamLaunchSnapshot, PersistedTeamLaunchSummary, ProviderModelLaunchIdentity, RetryFailedOpenCodeSecondaryLanesResult, TaskRef, TeamAgentRuntimeBackendType, TeamAgentRuntimeDiagnosticSeverity, TeamAgentRuntimeEntry, TeamAgentRuntimeLivenessKind, TeamAgentRuntimeLoadScope, TeamAgentRuntimePidSource, TeamAgentRuntimeResourceSample, TeamAgentRuntimeSnapshot, TeamChangeEvent, TeamConfig, TeamCreateRequest, TeamCreateResponse, TeamFastMode, TeamLaunchAggregateState, TeamLaunchDiagnosticItem, TeamLaunchRequest, TeamLaunchResponse, TeamMember, TeamProviderBackendId, TeamProviderId, TeamProvisioningModelCheckRequest, TeamProvisioningModelVerificationMode, TeamProvisioningPrepareIssue, TeamProvisioningPrepareResult, TeamProvisioningProgress, TeamProvisioningState, TeamProvisioningSupportDiagnostic, TeamRuntimeState, TeamTask, ToolActivityEventPayload, ToolApprovalAutoResolved, ToolApprovalEvent, ToolApprovalRequest, ToolApprovalSettings, ToolCallMeta, } from '@shared/types'; // pidusage's Windows wmic/gwmi fallback needs a non-zero cache window to finish // its initial two-sample pass. Keep this above slow PowerShell startup time, or // the first sample can expire before the recursive second read and loop again. const RUNTIME_PIDUSAGE_OPTIONS = process.platform === 'win32' ? { maxage: 10_000 } : { maxage: 0 }; interface RuntimeProcessUsageStats { rssBytes?: number; cpuPercent?: number; } interface RuntimeProcessLoadStats extends RuntimeProcessUsageStats { primaryCpuPercent?: number; primaryRssBytes?: number; childCpuPercent?: number; childRssBytes?: number; processCount?: number; runtimeLoadScope?: TeamAgentRuntimeLoadScope; runtimeLoadTruncated?: boolean; } type RuntimeTelemetryProcessSource = 'native' | 'wsl' | 'windows-host'; interface RuntimeTelemetryProcessTableRow extends RuntimeProcessTableRow, RuntimeProcessUsageStats { runtimeTelemetrySource?: RuntimeTelemetryProcessSource; } interface RuntimeProcessRowsCacheEntry { expiresAtMs: number; generation: number; runId: string | null; sampledAtMs: number; rows: RuntimeTelemetryProcessTableRow[] | null; includesWindowsHostRows: boolean; } class RuntimeTelemetryTimeoutError extends Error { constructor(message: string) { super(message); this.name = 'RuntimeTelemetryTimeoutError'; } } function normalizeRuntimeTelemetryNumber(value: unknown): number | undefined { if (typeof value === 'number') { return Number.isFinite(value) ? value : undefined; } if (typeof value !== 'string') { return undefined; } const trimmed = value.trim(); if (!trimmed) { return undefined; } const parsed = Number(trimmed); return Number.isFinite(parsed) ? parsed : undefined; } const logger = createLogger('Service:TeamProvisioning'); const PREFLIGHT_DEBUG_LOG_PATH = path.join(os.tmpdir(), 'claude-team-preflight-debug.log'); function appendPreflightDebugLog(event: string, data: Record): void { try { fs.appendFileSync( PREFLIGHT_DEBUG_LOG_PATH, `${JSON.stringify({ at: new Date().toISOString(), event, ...data, })}\n`, 'utf8' ); } catch { // Best-effort debug logging only. } } function truncatePreflightDebugText(value: string, maxLength = 1200): string { if (value.length <= maxLength) { return value; } return `${value.slice(0, maxLength)}...`; } const { AGENT_TEAMS_TEAMMATE_OPERATIONAL_TOOL_NAMES, AGENT_TEAMS_NAMESPACED_LEAD_BOOTSTRAP_TOOL_NAMES, AGENT_TEAMS_NAMESPACED_TEAMMATE_OPERATIONAL_TOOL_NAMES, createController, } = agentTeamsControllerModule; const TEAM_NAME_PATTERN = /^[a-z0-9][a-z0-9-]{0,127}$/; const VERIFY_TIMEOUT_MS = 15_000; const MCP_PREFLIGHT_INITIALIZE_TIMEOUT_MS = 45_000; const PROVIDER_MODEL_LIST_TIMEOUT_MS = 30_000; const PROVIDER_RUNTIME_STATUS_TIMEOUT_MS = 20_000; function asRuntimeRecord(value: unknown): Record { if (!value || typeof value !== 'object' || Array.isArray(value)) { throw new Error('OpenCode runtime payload must be an object'); } return value as Record; } function requireRuntimeString(value: unknown, fieldName: string): string { if (typeof value !== 'string' || value.trim().length === 0) { throw new Error(`OpenCode runtime payload missing ${fieldName}`); } return value.trim(); } function optionalRuntimeString(value: unknown): string | undefined { return typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined; } function normalizeRuntimeIso(value: unknown, fallback: string = nowIso()): string { const raw = optionalRuntimeString(value); if (!raw) { return fallback; } const parsed = Date.parse(raw); return Number.isFinite(parsed) ? new Date(parsed).toISOString() : fallback; } function normalizeRuntimeStringArray(value: unknown): string[] { return Array.isArray(value) ? value.filter((item): item is string => typeof item === 'string' && item.trim().length > 0) : []; } interface RuntimeToolMetadata { runtimePid?: number; processCommand?: string; runtimeVersion?: string; hostPid?: number; cwd?: string; } function normalizeRuntimePositiveInteger(value: unknown): number | undefined { return typeof value === 'number' && Number.isFinite(value) && value > 0 ? Math.trunc(value) : undefined; } function normalizeRuntimeMetadataString(value: unknown, maxLength: number): string | undefined { return typeof value === 'string' && value.trim().length > 0 ? value.trim().slice(0, maxLength) : undefined; } function parseRuntimeToolMetadata(value: unknown): RuntimeToolMetadata { if (!value || typeof value !== 'object' || Array.isArray(value)) { return {}; } const raw = value as Record; return { ...(normalizeRuntimePositiveInteger(raw.runtimePid) ? { runtimePid: normalizeRuntimePositiveInteger(raw.runtimePid) } : {}), ...(normalizeRuntimeMetadataString(raw.processCommand, 500) ? { processCommand: normalizeRuntimeMetadataString(raw.processCommand, 500) } : {}), ...(normalizeRuntimeMetadataString(raw.runtimeVersion, 80) ? { runtimeVersion: normalizeRuntimeMetadataString(raw.runtimeVersion, 80) } : {}), ...(normalizeRuntimePositiveInteger(raw.hostPid) ? { hostPid: normalizeRuntimePositiveInteger(raw.hostPid) } : {}), ...(normalizeRuntimeMetadataString(raw.cwd, 500) ? { cwd: normalizeRuntimeMetadataString(raw.cwd, 500) } : {}), }; } function buildRuntimeToolMetadataDiagnostics(metadata: RuntimeToolMetadata | undefined): string[] { if (!metadata) { return []; } const diagnostics: string[] = []; if (metadata.runtimePid != null) { diagnostics.push(`runtime pid: ${metadata.runtimePid}`); } if (metadata.processCommand) { const processCommand = sanitizeProcessCommandForDiagnostics(metadata.processCommand); if (processCommand) { diagnostics.push(`runtime process command: ${processCommand}`); } } if (metadata.runtimeVersion) { diagnostics.push(`runtime version: ${metadata.runtimeVersion}`); } if (metadata.hostPid != null) { diagnostics.push(`runtime host pid: ${metadata.hostPid}`); } if (metadata.cwd) { diagnostics.push(`runtime cwd: ${metadata.cwd}`); } return diagnostics; } function buildRuntimeDiagnosticForSpawn( metadata: LiveTeamAgentRuntimeMetadata ): string | undefined { const baseDiagnostic = metadata.runtimeDiagnostic; const processTableUnavailable = mentionsProcessTableUnavailable(baseDiagnostic) || metadata.diagnostics?.some((diagnostic) => mentionsProcessTableUnavailable(diagnostic)); if (!processTableUnavailable) { return baseDiagnostic; } if (mentionsProcessTableUnavailable(baseDiagnostic)) { return baseDiagnostic; } return baseDiagnostic ? `${baseDiagnostic}; process table unavailable` : 'process table unavailable'; } function isConfirmedBootstrapStaleRuntimeDiagnostic(reason?: string): boolean { const text = reason?.trim(); return text === 'persisted runtime pid is not alive'; } function isBootstrapProofClearableLaunchFailureReason(reason?: string): boolean { return ( isAutoClearableLaunchFailureReason(reason) || isProvisionedButNotAliveFailureReason(reason) ); } function shouldClearRuntimeDiagnosticAfterBootstrapConfirmation(reason?: string): boolean { return ( isBootstrapProofClearableLaunchFailureReason(reason) || isConfirmedBootstrapStaleRuntimeDiagnostic(reason) ); } function runtimeTaskRefs(teamName: string, value: unknown): InboxMessage['taskRefs'] | undefined { const refs = normalizeRuntimeStringArray(value); return refs.length > 0 ? refs.map((ref) => ({ teamName, taskId: ref, displayId: ref, })) : undefined; } function structuredTaskRefs(value: unknown): TaskRef[] | undefined { if (!Array.isArray(value) || value.length === 0) { return undefined; } const refs = value .filter((item): item is Record => Boolean(item) && typeof item === 'object') .map((item) => ({ taskId: typeof item.taskId === 'string' ? item.taskId.trim() : '', displayId: typeof item.displayId === 'string' ? item.displayId.trim() : '', teamName: typeof item.teamName === 'string' ? item.teamName.trim() : '', })) .filter( (item) => item.taskId.length > 0 && item.displayId.length > 0 && item.teamName.length > 0 ); return refs.length > 0 ? refs : undefined; } function teamToolTaskRefs(teamName: string, value: unknown): TaskRef[] | undefined { return structuredTaskRefs(value) ?? runtimeTaskRefs(teamName, value); } // TODO(team-result-notification-v2): The safest long-term design is a runtime-authored // task_result_notification emitted after task_complete with a validated resultCommentId. // That would let the lead react to authoritative board/runtime state instead of // teammate prose. Keep this relay hardening in place until that contract exists. function buildLeadInboxTaskContextBlock( message: Pick ): string { const taskRefs = Array.isArray(message.taskRefs) ? message.taskRefs : []; const commentId = typeof message.commentId === 'string' && message.commentId.trim().length > 0 ? message.commentId.trim() : undefined; if (taskRefs.length === 0 && !commentId) { return ''; } const lines = [ `Authoritative structured task context for this inbox row. Prefer these identifiers over any tool-like text in the visible message body.`, ]; if (typeof message.source === 'string' && message.source.trim().length > 0) { lines.push(`Source: ${message.source.trim()}`); } if (typeof message.messageKind === 'string' && message.messageKind.trim().length > 0) { lines.push(`Message kind: ${message.messageKind.trim()}`); } if (taskRefs.length > 0) { lines.push(`Task refs:`); for (const taskRef of taskRefs) { lines.push( `- ${formatTaskDisplayLabel({ id: taskRef.taskId, displayId: taskRef.displayId })} => teamName="${taskRef.teamName}", taskId="${taskRef.taskId}", displayId="${taskRef.displayId}"` ); } } if (commentId) { lines.push(`Comment id: "${commentId}"`); } if (commentId && taskRefs.length === 1) { const [taskRef] = taskRefs; if (taskRef) { lines.push( `Fetch the authoritative task comment with: task_get_comment { teamName: "${taskRef.teamName}", taskId: "${taskRef.taskId}", commentId: "${commentId}" }` ); } } return wrapAgentBlock(lines.join('\n')); } function mergeRuntimeDiagnostics( previous: string[] | undefined, incoming: unknown, fallback?: string ): string[] | undefined { const merged = [ ...(previous ?? []), ...normalizeRuntimeStringArray(incoming), ...(fallback ? [fallback] : []), ].filter((value) => value.trim().length > 0); return merged.length > 0 ? [...new Set(merged)] : undefined; } const VERIFY_POLL_MS = 500; const MCP_PREFLIGHT_SHUTDOWN_GRACE_MS = 250; const MCP_PREFLIGHT_SHUTDOWN_TIMEOUT_MS = 2_000; const MCP_PREFLIGHT_SHUTDOWN_POLL_MS = 50; const STDERR_RING_LIMIT = 64 * 1024; const STDOUT_RING_LIMIT = 64 * 1024; // Progress emissions fan out the latest CLI tail + assistant output to the // renderer over IPC. Under load the previous 300ms cadence combined with an // unbounded payload (see `emitLogsProgress`) caused renderer OOM crashes // (≈3 full-history serializations per second, each holding thousands of // lines). The tail cap in `emitLogsProgress` bounds each payload; we also // slow the cadence to ~1s so Zustand can keep up on large teams. const LOG_PROGRESS_THROTTLE_MS = 1000; const UI_LOGS_TAIL_LIMIT = 128 * 1024; const PROBE_CACHE_TTL_MS = 36 * 60 * 60 * 1000; const PREFLIGHT_BINARY_TIMEOUT_MS = 8000; const PREFLIGHT_AUTH_RETRY_DELAY_MS = 2000; const PREFLIGHT_AUTH_MAX_RETRIES = 2; const OPENCODE_PROVIDER_SCOPED_PREPARE_FAILURE_REASONS = new Set([ 'not_installed', 'not_authenticated', 'unsupported_version', 'capabilities_missing', 'runtime_store_blocked', 'mcp_unavailable', 'adapter_disabled', ]); const OPENCODE_RUNTIME_BINARY_UNREACHABLE_DIAGNOSTIC = 'OpenCode runtime binary is not installed or not reachable by launch preflight.'; const OPENCODE_APP_MCP_UNREACHABLE_DIAGNOSTIC = 'OpenCode app MCP is unreachable. Retry launch to refresh the app MCP bridge.'; const OPENCODE_PENDING_PERMISSION_REQUEST_PATTERN = /\b(?:pending permission request(?:\(s\)|s)?|permission[_ -]blocked)\b/i; function pushUniqueLine(lines: string[], line: string): void { const trimmed = line.trim(); if (trimmed.length > 0 && !lines.includes(trimmed)) { lines.push(trimmed); } } function pushUniqueSupportDiagnostics( diagnostics: TeamProvisioningSupportDiagnostic[], incoming: readonly TeamProvisioningSupportDiagnostic[] | undefined ): void { for (const diagnostic of incoming ?? []) { if (!diagnostics.some((existing) => existing.id === diagnostic.id)) { diagnostics.push({ ...diagnostic }); } } } function looksLikeOpenCodeProviderPrepareDiagnostic(value: string): boolean { const lower = value.trim().toLowerCase(); return ( isOpenCodeBridgeNoOutputDiagnostic(value) || isOpenCodeWindowsAccessDeniedDiagnostic(value) || lower.includes('opencode /experimental/tool') || lower.includes('/experimental/tool') || lower.includes('mcp_unavailable') || lower.includes('runtime store') || lower.includes('opencode cli') || lower.includes('opencode runtime binary') || lower.includes('unable to connect') ); } function normalizeOpenCodePrepareDiagnostic(value: string, reason?: string): string { const trimmed = value.trim(); if (!trimmed) { return trimmed; } if (isOpenCodeBridgeNoOutputDiagnostic(trimmed)) { return 'OpenCode runtime check returned no output.'; } const accessDeniedDiagnostic = normalizeOpenCodeWindowsAccessDeniedDiagnostic(trimmed); if (accessDeniedDiagnostic) { return accessDeniedDiagnostic; } if (/opencode cli (?:not detected on path|not found)/i.test(trimmed)) { return OPENCODE_RUNTIME_BINARY_UNREACHABLE_DIAGNOSTIC; } const lower = trimmed.toLowerCase(); if ( lower.includes('unable to connect') && (lower.includes('/experimental/tool') || lower.includes('mcp_unavailable') || reason === 'mcp_unavailable') ) { const detail = trimmed.includes(' - ') ? trimmed.split(' - ').pop()?.trim() : trimmed; return detail && detail !== trimmed ? `${OPENCODE_APP_MCP_UNREACHABLE_DIAGNOSTIC} Details: ${detail}` : OPENCODE_APP_MCP_UNREACHABLE_DIAGNOSTIC; } if (reason === 'mcp_unavailable' && lower.includes('mcp_unavailable')) { return 'OpenCode app MCP is unavailable. Retry launch to refresh the app MCP bridge.'; } return trimmed; } function isRetryableOpenCodePreflightBusyDiagnostic(value: string | null | undefined): boolean { const lower = value?.trim().toLowerCase() ?? ''; if (!lower) { return false; } // Fact: these diagnostics report OpenCode host/session occupancy, not that // the selected model is unavailable or rejected by the provider. return ( lower.includes('opencode session status busy') || lower.includes('session status busy') || lower === 'provider busy' || lower.includes('provider busy') ); } function isOpenCodeModelVerificationTimeoutDiagnostic(value: string | null | undefined): boolean { const lower = value?.trim().toLowerCase() ?? ''; return lower.includes('model verification timed out'); } function selectOpenCodeModelPreparePrimaryReason( prepare: Extract ): string { const providerDiagnostic = selectOpenCodePrepareProviderDiagnostic(prepare); if (providerDiagnostic) { return providerDiagnostic; } const candidates = [...prepare.diagnostics, prepare.reason] .map((entry) => entry?.trim() ?? '') .filter(Boolean); const timeoutReason = candidates.find(isOpenCodeModelVerificationTimeoutDiagnostic); return timeoutReason ?? candidates[0] ?? prepare.reason; } function selectOpenCodePrepareProviderDiagnostic( prepare: Pick ): string | undefined { return [...prepare.diagnostics, ...prepare.warnings].find( (entry) => isOpenCodeBridgeNoOutputDiagnostic(entry) || isOpenCodeWindowsAccessDeniedDiagnostic(entry) ); } function isOpenCodeModelPrepareBusyDeferred( prepare: Extract, primaryReason: string ): boolean { const candidates = [primaryReason, prepare.reason, ...prepare.diagnostics]; return ( prepare.retryable && !candidates.some(isOpenCodeModelVerificationTimeoutDiagnostic) && candidates.some(isRetryableOpenCodePreflightBusyDiagnostic) ); } function buildOpenCodeProviderVerificationDeferredLine(reason: string): string { const normalizedReason = isRetryableOpenCodePreflightBusyDiagnostic(reason) ? 'OpenCode is currently busy with another session. Deep model verification will retry when OpenCode is idle.' : reason; return normalizedReason; } function applyDistinctProvisioningMemberColors< T extends { name: string; color?: string; removedAt?: number }, >(members: readonly T[]): T[] { const colorMap = buildTeamMemberColorMap(members, { preferProvidedColors: false }); return members.map((member) => ({ ...member, color: colorMap.get(member.name) ?? member.color ?? getMemberColorByName(member.name), })); } const FS_MONITOR_POLL_MS = 2000; const TASK_WAIT_FALLBACK_MS = 15_000; const STALL_CHECK_INTERVAL_MS = 10_000; const STALL_WARNING_THRESHOLD_MS = 20_000; const APP_TEAM_RUNTIME_DISALLOWED_TOOLS = 'TeamDelete,TodoWrite,TaskCreate,TaskUpdate,mcp__agent-teams__team_launch,mcp__agent-teams__team_stop'; const CLAUDE_TEAM_RUNTIME_SETTINGS_PATH_ENV = 'CLAUDE_TEAM_RUNTIME_SETTINGS_PATH'; const TEAM_JSON_READ_TIMEOUT_MS = 5_000; const TEAM_CONFIG_MAX_BYTES = 10 * 1024 * 1024; const TEAM_INBOX_MAX_BYTES = 2 * 1024 * 1024; const MEMBER_SPAWN_AUDIT_MIN_INTERVAL_MS = 1_500; const CROSS_TEAM_TOOL_RECIPIENT_NAMES = new Set([ 'cross_team_send', 'cross_team_list_targets', 'cross_team_get_outbox', ]); const HANDLED_STREAM_JSON_TYPES = new Set([ 'user', 'assistant', 'control_request', // Claude Code can emit informational rate-limit allowance snapshots as // top-level stream-json events. They are not errors; actionable retry/error // diagnostics still arrive through system/api_retry and remain handled below. 'rate_limit_event', 'result', 'system', ]); function assertAppDeterministicBootstrapEnabled(): void { if (process.env.CLAUDE_APP_DISABLE_DETERMINISTIC_TEAM_BOOTSTRAP === '1') { throw new Error( 'Deterministic team bootstrap is disabled by the app rollout flag (CLAUDE_APP_DISABLE_DETERMINISTIC_TEAM_BOOTSTRAP=1).' ); } if (process.env.CLAUDE_DISABLE_DETERMINISTIC_TEAM_BOOTSTRAP === '1') { throw new Error( 'Deterministic team bootstrap is disabled by the runtime kill switch (CLAUDE_DISABLE_DETERMINISTIC_TEAM_BOOTSTRAP=1).' ); } } function classifyDeterministicBootstrapFailure(reason: string): { title: string; normalizedReason: string; } { const normalizedReason = reason.trim(); const lower = normalizedReason.toLowerCase(); if (isWorkspaceTrustLaunchFailureText(normalizedReason)) { return { title: 'Workspace trust required', normalizedReason, }; } if (lower.includes('disabled by kill switch')) { return { title: 'Deterministic bootstrap disabled', normalizedReason, }; } if ( lower.includes('requires claude_enable_deterministic_team_bootstrap=1') || lower.includes('unsupported schema version') || lower.includes('regular file and must not be a symlink') ) { return { title: 'Deterministic bootstrap compatibility failure', normalizedReason, }; } return { title: 'Deterministic bootstrap failed', normalizedReason, }; } function getPreflightPingArgs(providerId: TeamProviderId | undefined): string[] { return buildProviderPreflightPingArgs(providerId); } function getPreflightTimeoutMs(providerId: TeamProviderId | undefined): number { return getProviderModelProbeTimeoutMs(providerId); } function getProviderRuntimeFailureLabel(providerId: TeamProviderId): string { switch (providerId) { case 'anthropic': return 'Claude CLI'; case 'codex': return 'Codex runtime'; case 'gemini': return 'Gemini runtime'; case 'opencode': return 'OpenCode runtime'; } } function getRunRuntimeFailureLabel(run: ProvisioningRun): string { const providerIds = new Set(); const addProvider = (providerId: TeamProviderId | undefined): void => { if (providerId) { providerIds.add(providerId); } }; addProvider(normalizeOptionalTeamProviderId(run.request.providerId)); addProvider(inferTeamProviderIdFromModel(run.request.model)); for (const member of run.request.members) { addProvider(normalizeOptionalTeamProviderId(member.providerId)); addProvider(inferTeamProviderIdFromModel(member.model)); } if (providerIds.size === 1) { return getProviderRuntimeFailureLabel([...providerIds][0]); } return getCliFlavorUiOptions(getConfiguredCliFlavor()).displayName; } function buildMissingCliError(): Error { if (getConfiguredCliFlavor() === 'agent_teams_orchestrator') { return new Error( 'Multimodel runtime not found. The packaged app must include resources/runtime/claude-multimodel, or development must provide CLAUDE_AGENT_TEAMS_ORCHESTRATOR_CLI_PATH.' ); } return new Error('Claude CLI not found; install it or provide a valid path'); } function buildProviderCliCommandArgs(providerArgs: string[], args: string[]): string[] { return mergeJsonSettingsArgs([...providerArgs, ...args]); } interface ProviderModelListCommandResponse { schemaVersion?: number; providers?: Record< string, { defaultModel?: string | null; models?: (string | { id?: string; label?: string; description?: string })[]; } >; } interface RuntimeStatusCommandResponse { providers?: Record>; } interface AuthStatusCommandResponse { provider?: string; status?: Partial; loggedIn?: boolean; authMethod?: string | null; providers?: Record>; } interface RuntimeProviderLaunchFacts { defaultModel: string | null; modelIds: Set; modelListParsed?: boolean; modelCatalog: CliProviderModelCatalog | null; runtimeCapabilities: CliProviderRuntimeCapabilities | null; providerStatus?: | (Partial & { providerId?: CliProviderStatus['providerId'] }) | null; } interface ProviderSelectedModelCheck { modelId: string; effort?: EffortLevel; } function extractJsonObjectFromCli(raw: string): T { const trimmed = raw.trim(); try { return JSON.parse(trimmed) as T; } catch (initialError) { const candidates: T[] = []; let lastParseError: unknown = null; for (let start = trimmed.indexOf('{'); start >= 0; start = trimmed.indexOf('{', start + 1)) { const end = findJsonObjectEnd(trimmed, start); if (end < 0) { continue; } try { candidates.push(JSON.parse(trimmed.slice(start, end + 1)) as T); } catch (error) { lastParseError = error; } } let providerResponse: T | null = null; for (let index = candidates.length - 1; index >= 0; index -= 1) { const record = candidates[index] as Record | null; const providers = record && typeof record === 'object' ? record.providers : null; if (providers && typeof providers === 'object' && !Array.isArray(providers)) { providerResponse = candidates[index]; break; } } if (providerResponse) { return providerResponse; } if (candidates.length > 0) { throw new Error('No provider JSON object found in CLI output'); } if (lastParseError instanceof Error) { throw lastParseError; } if (trimmed.includes('{') && initialError instanceof Error) { throw initialError; } throw new Error('No JSON object found in CLI output'); } } function findJsonObjectEnd(source: string, start: number): number { let depth = 0; let inString = false; let escaped = false; for (let index = start; index < source.length; index += 1) { const char = source[index]; if (inString) { if (escaped) { escaped = false; } else if (char === '\\') { escaped = true; } else if (char === '"') { inString = false; } continue; } if (char === '"') { inString = true; } else if (char === '{') { depth += 1; } else if (char === '}') { depth -= 1; if (depth === 0) { return index; } } } return -1; } function getLaunchModelArg( providerId: TeamProviderId, model: string | undefined, launchIdentity?: ProviderModelLaunchIdentity | null ): string | undefined { if (providerId === 'anthropic' && launchIdentity?.resolvedLaunchModel) { return launchIdentity.resolvedLaunchModel; } const explicitModel = getExplicitLaunchModelSelection(model); if (explicitModel) { return explicitModel; } if ( providerId === 'codex' && launchIdentity?.selectedModelKind === 'default' && launchIdentity.resolvedLaunchModel ) { return launchIdentity.resolvedLaunchModel; } return undefined; } function normalizeProviderModelListModels( provider: NonNullable[string] | undefined ): Set { const models = new Set(); for (const entry of provider?.models ?? []) { const modelId = typeof entry === 'string' ? entry : entry.id; const trimmed = modelId?.trim(); if (trimmed) { models.add(trimmed); } } return models; } function normalizeProviderSelectedModelChecks( modelIds: readonly string[], modelChecks?: readonly ProviderSelectedModelCheck[] ): ProviderSelectedModelCheck[] { const checks: ProviderSelectedModelCheck[] = modelChecks && modelChecks.length > 0 ? [...modelChecks] : modelIds.map((modelId) => ({ modelId })); const seen = new Set(); const normalized: ProviderSelectedModelCheck[] = []; for (const check of checks) { const modelId = check.modelId.trim(); if (!modelId) { continue; } const key = `${modelId}\n${check.effort ?? ''}`; if (seen.has(key)) { continue; } seen.add(key); normalized.push({ modelId, ...(check.effort ? { effort: check.effort } : {}), }); } return normalized; } function normalizeProvisioningModelCheckRequests( checks: readonly TeamProvisioningModelCheckRequest[] | undefined ): TeamProvisioningModelCheckRequest[] { const seen = new Set(); const normalized: TeamProvisioningModelCheckRequest[] = []; for (const check of checks ?? []) { const model = check.model.trim(); if (!model) { continue; } const key = `${check.providerId}\n${model}\n${check.effort ?? ''}`; if (seen.has(key)) { continue; } seen.add(key); normalized.push({ providerId: check.providerId, model, ...(check.effort ? { effort: check.effort } : {}), }); } return normalized; } function addModelCatalogLaunchModels( modelIds: Set, catalog: CliProviderModelCatalog ): void { for (const model of catalog.models ?? []) { const launchModel = model.launchModel?.trim(); if (launchModel) { modelIds.add(launchModel); } const catalogId = model.id?.trim(); if (catalogId) { modelIds.add(catalogId); } } } function isLegacySafeEffort(effort: EffortLevel): boolean { return effort === 'low' || effort === 'medium' || effort === 'high'; } function isCodexEffortRuntimeSupported( effort: EffortLevel, capabilities: CliProviderRuntimeCapabilities | null ): boolean { if (isLegacySafeEffort(effort)) { return true; } const reasoning = capabilities?.reasoningEffort; return reasoning?.configPassthrough === true && reasoning.values.includes(effort); } function hasAuthoritativeCodexLaunchCatalog( facts: Pick< RuntimeProviderLaunchFacts, 'modelIds' | 'modelListParsed' | 'modelCatalog' | 'runtimeCapabilities' > ): boolean { if (facts.modelIds.size > 0 || facts.modelCatalog != null) { return true; } return ( facts.modelListParsed === true && facts.runtimeCapabilities?.modelCatalog?.dynamic === false ); } function resolveAnthropicSelectionFromFacts(params: { selectedModel?: string; limitContext?: boolean; facts: Pick; }) { return resolveAnthropicRuntimeSelection({ source: { modelCatalog: params.facts.modelCatalog, runtimeCapabilities: params.facts.runtimeCapabilities, }, selectedModel: params.selectedModel, limitContext: params.limitContext === true, availableLaunchModels: params.facts.modelCatalog ? undefined : params.facts.modelIds, }); } function formatAnthropicEffortSupportFailure(params: { effort: EffortLevel; modelLabel: string; supportedEfforts?: readonly EffortLevel[]; kind: | 'unsupported-by-catalog' | 'unsupported-by-runtime-capability' | 'unverified-catalog-missing'; }): string { if (params.kind === 'unverified-catalog-missing') { return `Anthropic runtime catalog was unavailable, so effort "${params.effort}" for ${params.modelLabel} could not be verified.`; } const supported = params.supportedEfforts?.length ? ` Supported efforts: ${params.supportedEfforts.join(', ')}.` : ''; const runtimeSuffix = params.kind === 'unsupported-by-runtime-capability' ? ' in the current runtime capability data' : ' in the current runtime'; return `${params.modelLabel} does not support Anthropic effort "${params.effort}"${runtimeSuffix}.${supported}`; } function resolveCodexSelectionFromFacts(params: { selectedModel?: string; providerBackendId?: TeamCreateRequest['providerBackendId']; facts: Pick; }) { return resolveCodexRuntimeSelection({ source: { providerStatus: params.facts.providerStatus, providerBackendId: params.providerBackendId, }, selectedModel: params.selectedModel, }); } function buildAnthropicSettingsObject( providerId: TeamProviderId, launchIdentity?: ProviderModelLaunchIdentity | null ): TeamRuntimeSettingsJson | null { if (providerId !== 'anthropic' || typeof launchIdentity?.resolvedFastMode !== 'boolean') { return null; } return launchIdentity.resolvedFastMode ? { fastMode: true, fastModePerSessionOptIn: false, } : { fastMode: false, }; } function buildAnthropicSettingsArgs( providerId: TeamProviderId, launchIdentity?: ProviderModelLaunchIdentity | null ): string[] { const settings = buildAnthropicSettingsObject(providerId, launchIdentity); if (!settings) { return []; } return ['--settings', JSON.stringify(settings)]; } function sanitizeRuntimeSettingsTeamName(teamName: string): string { return teamName.replace(/[^a-zA-Z0-9._-]+/g, '_') || 'team'; } function buildRuntimeSettingsTempDirectory(teamName: string): string { return path.join( os.tmpdir(), 'agent-teams-runtime-settings', `${sanitizeRuntimeSettingsTeamName(teamName)}-${randomUUID()}` ); } function normalizeTeamRuntimeNodeEnv(env: NodeJS.ProcessEnv): void { // Vitest sets NODE_ENV=test in the desktop parent process. Real team runtime // children must run the CLI normally, otherwise source launches can take // test-only startup paths and exit before deterministic bootstrap starts. if (env.NODE_ENV === 'test') { env.NODE_ENV = 'development'; } } function buildProviderFastModeArgs( providerId: TeamProviderId, launchIdentity?: ProviderModelLaunchIdentity | null ): string[] { if (providerId === 'anthropic') { return buildAnthropicSettingsArgs(providerId, launchIdentity); } if (providerId === 'codex') { return buildCodexFastModeArgs(launchIdentity?.resolvedFastMode); } return []; } function filterOutSettingsPathArgs( args: string[], settingsPath: string | null | undefined ): string[] { if (!settingsPath) { return [...args]; } const filtered: string[] = []; let index = 0; while (index < args.length) { const arg = args[index]; if (arg === '--settings' && args[index + 1] === settingsPath) { index += 2; continue; } if (arg === `--settings=${settingsPath}`) { index += 1; continue; } filtered.push(arg); index += 1; } return filtered; } function hasPathBasedSettingsArgs(args: string[]): boolean { let index = 0; while (index < args.length) { const arg = args[index]; if (arg === '--settings') { const value = args[index + 1]; if (typeof value === 'string') { if (!parseJsonSettingsObject(value)) { return true; } index += 2; continue; } if (typeof value !== 'string') { return true; } index += 1; continue; } const prefix = '--settings='; if (arg.startsWith(prefix) && !parseJsonSettingsObject(arg.slice(prefix.length))) { return true; } index += 1; } return false; } function isProbeTimeoutMessage(message: string): boolean { const lower = message.toLowerCase(); return ( lower.includes('timeout running:') || lower.includes('timed out') || lower.includes('did not complete') || lower.includes('etimedout') ); } function resolveRequestedLaunchModel(params: { providerId: TeamProviderId; selectedModel?: string; limitContext?: boolean; facts: Pick; }): string | null { if (params.providerId === 'anthropic') { return resolveAnthropicLaunchModel({ selectedModel: params.selectedModel, limitContext: params.limitContext === true, availableLaunchModels: params.facts.modelIds, defaultLaunchModel: params.facts.defaultModel, }); } const explicitModel = getExplicitLaunchModelSelection(params.selectedModel); return explicitModel ?? params.facts.defaultModel; } type TeamsBaseLocation = 'configured' | 'default'; type ValidConfigProbeResult = | { ok: true; location: TeamsBaseLocation; configPath: string } | { ok: false }; function getTeamsBasePathsToProbe(): { location: TeamsBaseLocation; basePath: string }[] { const configured = getTeamsBasePath(); const defaultBase = path.join(getAutoDetectedClaudeBasePath(), 'teams'); if (path.resolve(configured) === path.resolve(defaultBase)) { return [{ location: 'configured', basePath: configured }]; } return [ { location: 'configured', basePath: configured }, { location: 'default', basePath: defaultBase }, ]; } function logsSuggestShutdownOrCleanup(logs: string): boolean { const text = logs.toLowerCase(); return ( text.includes('shutdown') || text.includes('clean up') || text.includes('cleanup') || text.includes('deactivate') || text.includes('deactivated') || text.includes('resources') || // Russian keywords observed in some CLI outputs / user environments text.includes('очист') || text.includes('очищ') || text.includes('заверш') || text.includes('деактив') ); } function looksLikeClaudeStdoutJsonFragment(text: string): boolean { const trimmed = text.trim(); if (!trimmed.startsWith('{') && !trimmed.startsWith('[')) { return false; } return ( /"type"\s*:/.test(trimmed) || /"message"\s*:/.test(trimmed) || /"content"\s*:/.test(trimmed) || /"subtype"\s*:/.test(trimmed) || /"session_id"\s*:/.test(trimmed) ); } const DETERMINISTIC_BOOTSTRAP_COMPLETION_RECOVERY_MS = 12_000; function isTerminalFailureProvisioningState(state: TeamProvisioningProgress['state']): boolean { return state === 'failed' || state === 'cancelled' || state === 'disconnected'; } function shouldIgnoreProvisioningProgressRegression( currentState: TeamProvisioningProgress['state'], nextState: TeamProvisioningProgress['state'] ): boolean { if (currentState === 'ready') { return nextState !== 'ready' && nextState !== 'disconnected'; } if (isTerminalFailureProvisioningState(currentState)) { return nextState !== currentState; } return false; } interface ProvisioningRun { runId: string; teamName: string; startedAt: string; progress: TeamProvisioningProgress; stdoutBuffer: string; stderrBuffer: string; /** Rolling buffer of CLI log lines (oldest -> newest). */ claudeLogLines: string[]; /** Last stream used for claudeLogLines markers. */ lastClaudeLogStream: 'stdout' | 'stderr' | null; /** Carry buffer for stdout line splitting (CLI output). */ stdoutLogLineBuf: string; /** Carry buffer for stderr line splitting (CLI output). */ stderrLogLineBuf: string; /** Raw stdout parser carry that has not been newline-delimited yet. */ stdoutParserCarry: string; /** Whether the current stdout parser carry is a complete JSON fragment. */ stdoutParserCarryIsCompleteJson: boolean; /** Whether the current stdout parser carry looks like Claude stream-json structure. */ stdoutParserCarryLooksLikeClaudeJson: boolean; /** ISO timestamp when the last CLI line was recorded. */ claudeLogsUpdatedAt?: string; /** ISO timestamp when the first accepted deterministic bootstrap event arrived. */ deterministicBootstrapStartedAt?: string; /** Latest accepted deterministic bootstrap event name. */ lastDeterministicBootstrapEvent?: string; /** Latest accepted deterministic bootstrap phase name. */ lastDeterministicBootstrapPhase?: string; /** True after deterministic bootstrap reports that teammate spawning started. */ deterministicBootstrapMemberSpawnSeen: boolean; /** True after deterministic bootstrap reports at least one teammate spawn result. */ deterministicBootstrapMemberResultSeen: boolean; processKilled: boolean; finalizingByTimeout: boolean; cancelRequested: boolean; teamsBasePathsToProbe: { location: TeamsBaseLocation; basePath: string }[]; child: ReturnType | null; timeoutHandle: NodeJS.Timeout | null; fsMonitorHandle: NodeJS.Timeout | null; onProgress: (progress: TeamProvisioningProgress) => void; expectedMembers: string[]; request: TeamCreateRequest; allEffectiveMembers: TeamCreateRequest['members']; effectiveMembers: TeamCreateRequest['members']; launchIdentity: ProviderModelLaunchIdentity | null; mixedSecondaryLanes: MixedSecondaryRuntimeLaneState[]; /** * OpenCode secondary lanes share bridge state files. Launch them sequentially * per team run to avoid file-lock contention while keeping launch non-blocking. */ mixedSecondaryLaneLaunchQueue?: Promise; lastLogProgressAt: number; /** Monotonic ms timestamp of last stdout/stderr data. For stall detection. */ lastDataReceivedAt: number; /** Monotonic ms timestamp of last stdout data only. Stall watchdog uses this * instead of lastDataReceivedAt because stderr emits periodic debug logs * that reset the timer without producing any user-visible output. */ lastStdoutReceivedAt: number; /** Stall watchdog interval handle. Cleared in cleanupRun(). */ stallCheckHandle: NodeJS.Timeout | null; /** Index of the current stall warning in provisioningOutputParts. * Used to replace in-place instead of pushing duplicates. */ stallWarningIndex: number | null; /** The progress.message before the stall watchdog overwrote it. * Restored when stdout resumes and the stall warning is cleared. */ preStallMessage: string | null; /** Monotonic ms timestamp of last api_retry message. When set, the stall * watchdog defers to retry messages for progress.message (retries are * more informative than the generic "CLI not responding" stall text). */ lastRetryAt: number; /** Index of the latest api_retry warning block in provisioningOutputParts. */ apiRetryWarningIndex: number | null; /** True after emitApiErrorWarning() fires once — prevents duplicate warnings and pre-complete false positives. */ apiErrorWarningEmitted: boolean; fsPhase: 'waiting_config' | 'waiting_members' | 'waiting_tasks' | 'all_files_found'; waitingTasksSince: number | null; provisioningComplete: boolean; processClosed: boolean; requiresFirstRealTurnSuccess: boolean; firstRealTurnSucceeded: boolean; /** Path to the generated MCP config file for later cleanup. */ mcpConfigPath: string | null; /** Paths to per-member generated MCP config files consumed by deterministic bootstrap. */ memberMcpConfigPaths: string[]; /** Path to the deterministic bootstrap spec file for later cleanup. */ bootstrapSpecPath: string | null; /** Path to the deferred first-user-task file consumed by runtime after bootstrap. */ bootstrapUserPromptPath: string | null; isLaunch: boolean; launchStateClearedForRun: boolean; deterministicBootstrap: boolean; launchCleanupStateFinalized?: boolean; workspaceTrustPlan?: WorkspaceTrustFullPlanResult | null; workspaceTrustExecution?: WorkspaceTrustExecutionResult | null; workspaceTrustDiagnostics?: WorkspaceTrustDiagnosticsManifest | null; workspaceTrustRetryAttempted?: boolean; leadRelayCapture: { leadName: string; startedAt: string; textParts: string[]; textJoinMode?: 'block' | 'stream'; replyVisibility?: 'user' | 'internal_activity'; hasVisibleSendMessage?: boolean; hasUserVisibleSendMessage?: boolean; settled: boolean; idleHandle: NodeJS.Timeout | null; idleMs: number; resolveOnce: (text: string) => void; rejectOnce: (error: string) => void; timeoutHandle: NodeJS.Timeout; } | null; activeCrossTeamReplyHints: { toTeam: string; conversationId: string; }[]; /** Monotonic counter for individual lead assistant messages. */ leadMsgSeq: number; /** Active text bubble for token-streamed lead assistant output. */ liveLeadTextBuffer: { messageId: string; text: string; timestamp: string; toolCalls?: ToolCallMeta[]; toolSummary?: string; } | null; /** Accumulated tool_use details between text messages. */ pendingToolCalls: ToolCallMeta[]; /** Active runtime tool calls keyed by tool_use_id. */ activeToolCalls: Map; /** True when a direct MCP cross_team_send happened and sentMessages history should refresh. */ pendingDirectCrossTeamSendRefresh: boolean; /** Throttle timestamp for emitting inbox refresh events for lead text. */ lastLeadTextEmitMs: number; /** * When set, the current stdin-injected turn is an internal "forward user DM to teammate" * request triggered by the UI. We suppress any lead→user echo for that turn. */ silentUserDmForward: { target: string; startedAt: string; mode: 'user_dm' | 'member_inbox_relay'; } | null; /** Safety valve: clears silentUserDmForward if turn never completes. */ silentUserDmForwardClearHandle: NodeJS.Timeout | null; /** Exact inbox rows currently being bridged into the live teammate process. */ pendingInboxRelayCandidates: PendingInboxRelayCandidate[]; /** Accumulates assistant text during provisioning phase for live UI preview. */ provisioningOutputParts: string[]; /** Bounded orchestration checkpoints shown in the Live output panel. */ provisioningTraceLines: string[]; /** Last emitted trace key, used to avoid duplicate progress spam. */ lastProvisioningTraceKey: string | null; /** Stable assistant message ids -> provisioningOutputParts index for in-place updates. */ provisioningOutputIndexByMessageId: Map; /** Session ID detected from stream-json output (result.session_id or message.session_id). */ detectedSessionId: string | null; /** Lead process activity: 'active' during turn processing, 'idle' waiting for input, 'offline' after exit. */ leadActivityState: LeadActivityState; /** Whether an auth failure retry was already attempted for this run. */ authFailureRetried: boolean; /** Set to true while auth-failure respawn is in progress to prevent duplicate handling. */ authRetryInProgress: boolean; /** Tracks lead process context window usage from stream-json usage data. */ leadContextUsage: { promptInputTokens: number | null; outputTokens: number | null; contextUsedTokens: number | null; contextWindowTokens: number | null; promptInputSource: LeadContextUsage['promptInputSource']; lastUsageMessageId: string | null; lastEmittedAt: number; } | null; /** Saved spawn context for auth-failure respawn. */ spawnContext: { claudePath: string; args: string[]; cwd: string; env: NodeJS.ProcessEnv; prompt: string; } | null; /** Run-scoped helper material used by Anthropic API-key team runtimes. */ anthropicApiKeyHelper: AnthropicTeamApiKeyHelperMaterial | null; /** Pending tool approval requests awaiting user response (control_request protocol). */ pendingApprovals: Map; /** Teammate permission_request IDs already intercepted (prevents re-processing read messages). */ processedPermissionRequestIds: Set; /** * Post-compact context reinjection lifecycle. * - pendingPostCompactReminder: compact_boundary was received; waiting for idle to inject. * - postCompactReminderInFlight: the reminder turn has been injected via stdin, waiting for result. * - suppressPostCompactReminderOutput: true while processing a reminder turn - suppress * low-value context-refresh acknowledgement text. */ pendingPostCompactReminder: boolean; postCompactReminderInFlight: boolean; suppressPostCompactReminderOutput: boolean; /** Gemini-only phase-2 launch hydration after the first successful provisioning turn. */ pendingGeminiPostLaunchHydration: boolean; geminiPostLaunchHydrationInFlight: boolean; geminiPostLaunchHydrationSent: boolean; suppressGeminiPostLaunchHydrationOutput: boolean; /** Per-member spawn lifecycle statuses tracked from stream-json output. */ memberSpawnStatuses: Map; /** Agent tool_use_id -> teammate name for persistent teammate spawns. */ memberSpawnToolUseIds: Map; /** Explicit restart requests awaiting teammate rejoin or failure. */ pendingMemberRestarts: Map; /** Per-member latest processed lead-inbox bootstrap signal cursor for the current live run. */ memberSpawnLeadInboxCursorByMember: Map; /** Highest accepted deterministic bootstrap event sequence for this run. */ lastDeterministicBootstrapSeq: number; /** Throttles config/inbox audit work triggered by frequent status polling. */ lastMemberSpawnAuditAt: number; /** Throttles repeated audit warnings when config.json is temporarily unreadable. */ lastMemberSpawnAuditConfigReadWarningAt: number; /** Per-member warning throttle for repeated "missing from config" logs. */ lastMemberSpawnAuditMissingWarningAt: Map; /** Prevents duplicate Team Launched notifications for the same live run. */ teamLaunchedNotificationFired?: boolean; } const PROVISIONING_TRACE_STORAGE_LIMIT = 500; interface MixedSecondaryRuntimeLaneState { laneId: string; providerId: 'opencode'; member: TeamCreateRequest['members'][number]; runId: string | null; state: 'queued' | 'launching' | 'finished'; result: TeamRuntimeLaunchResult | null; warnings: string[]; diagnostics: string[]; launchScheduled?: boolean; queuedAtMs?: number; launchStartedAtMs?: number; launchFinishedAtMs?: number; } interface OpenCodeSecondaryRetryCandidate { memberName: string; laneId: string; } interface OpenCodeSecondaryRetryOutcome { launchState: MemberLaunchState; reason?: string; } type MemberLifecycleOperationKind = | 'manual_restart' | 'opencode_retry' | 'opencode_member_added' | 'opencode_member_updated' | 'opencode_member_removed' | 'primary_member_added' | 'primary_member_restored' | 'primary_member_updated' | 'primary_member_removed'; type LiveRosterAttachReason = 'member_added' | 'member_restored' | 'member_updated'; type DirectProcessMemberLaunchReason = 'manual_restart' | LiveRosterAttachReason; interface MemberLifecycleOperation { kind: MemberLifecycleOperationKind; token: symbol; startedAtMs: number; } type LeadActivityState = 'active' | 'idle' | 'offline'; type ProvisioningAuthSource = | 'anthropic_api_key_helper' | 'anthropic_api_key' | 'anthropic_auth_token' | 'configured_api_key_missing' | 'codex_runtime' | 'gemini_runtime' | 'none'; function isAnthropicApiKeyBackedAuthSource(authSource: unknown): boolean { return authSource === 'anthropic_api_key' || authSource === 'anthropic_api_key_helper'; } function isAnthropicDirectCredentialAuthSource(authSource: unknown): boolean { return isAnthropicApiKeyBackedAuthSource(authSource) || authSource === 'anthropic_auth_token'; } function buildAnthropicCrossProviderDirectAuthEnvPatch( env: NodeJS.ProcessEnv, authSource: ProvisioningAuthSource ): NodeJS.ProcessEnv { const envPatch: NodeJS.ProcessEnv = {}; const apiKey = env.ANTHROPIC_API_KEY?.trim(); if (apiKey) { envPatch.ANTHROPIC_API_KEY = apiKey; } const baseUrl = env.ANTHROPIC_BASE_URL?.trim(); if (baseUrl) { envPatch.ANTHROPIC_BASE_URL = baseUrl; } if (authSource === 'anthropic_auth_token' && hasAnthropicCompatibleAuthTokenEnv(env)) { envPatch.ANTHROPIC_API_KEY = apiKey || ''; envPatch.ANTHROPIC_AUTH_TOKEN = env.ANTHROPIC_AUTH_TOKEN?.trim(); return envPatch; } for (const key of ANTHROPIC_HELPER_MODE_COMPETING_AUTH_ENV_KEYS) { if (key !== 'ANTHROPIC_API_KEY') { envPatch[key] = ''; } } return envPatch; } const CODEX_CROSS_PROVIDER_SAFE_ENV_KEYS = [ 'CLAUDE_CODE_CODEX_BACKEND', 'CLAUDE_CODE_CODEX_FORCED_LOGIN_METHOD', 'CODEX_CLI_PATH', 'CODEX_HOME', ] as const; function buildCodexCrossProviderSafeEnvPatch(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv { const envPatch: NodeJS.ProcessEnv = {}; for (const key of CODEX_CROSS_PROVIDER_SAFE_ENV_KEYS) { const value = env[key]?.trim(); if (value) { envPatch[key] = value; } } return envPatch; } function applyAppManagedRuntimeSettingsPathEnv( env: NodeJS.ProcessEnv, settingsPath: string | null ): void { if (settingsPath) { env[CLAUDE_TEAM_RUNTIME_SETTINGS_PATH_ENV] = settingsPath; } else { delete env[CLAUDE_TEAM_RUNTIME_SETTINGS_PATH_ENV]; } } interface TeamRuntimeAuthContext { teamName?: string; authMaterialId?: string; allowAnthropicApiKeyHelper?: boolean; } interface ProvisioningEnvResolution { env: NodeJS.ProcessEnv; authSource: ProvisioningAuthSource; geminiRuntimeAuth: GeminiRuntimeAuthState | null; providerArgs?: string[]; anthropicApiKeyHelper?: AnthropicTeamApiKeyHelperMaterial | null; warning?: string; } interface TeamRuntimeLaunchArgsPlan { settingsArgs: string[]; fastModeArgs: string[]; runtimeTurnSettledHookArgs: string[]; providerArgs: string[]; extraArgs: string[]; inheritedProviderArgs: string[]; appManagedSettingsPath: string | null; } type WorkspaceTrustProviderArgsResolver = (input: { providerId: TeamProviderId; providerArgs: string[]; phase: 'default-model-resolution'; }) => string[]; interface CrossProviderMemberArgsResult { args: string[]; providerArgsByProvider: Map; envPatch: NodeJS.ProcessEnv; usesAnthropicApiKeyHelper: boolean; } const OPENCODE_BOOTSTRAP_EVIDENCE_LOCK_OPTIONS = { acquireTimeoutMs: 45_000, staleTimeoutMs: 60_000, retryIntervalMs: 50, } as const; function nowIso(): string { return new Date().toISOString(); } interface LiveTeamAgentRuntimeMetadata { alive: boolean; backendType?: TeamAgentRuntimeBackendType; providerId?: TeamProviderId; agentId?: string; cwd?: string; pid?: number; metricsPid?: number; model?: string; tmuxPaneId?: string; livenessKind?: TeamAgentRuntimeLivenessKind; pidSource?: TeamAgentRuntimePidSource; processCommand?: string; panePid?: number; paneCurrentCommand?: string; runtimeSessionId?: string; runtimeLastSeenAt?: string; runtimeDiagnostic?: string; runtimeDiagnosticSeverity?: TeamAgentRuntimeDiagnosticSeverity; diagnostics?: string[]; } const OPENCODE_BOOTSTRAP_CHECKIN_RETRY_SENT_PREFIX = 'opencode_bootstrap_checkin_retry_prompt_sent'; function getOpenCodeBootstrapCheckinRetryMarker(runId: string, runtimeSessionId: string): string { return `${OPENCODE_BOOTSTRAP_CHECKIN_RETRY_SENT_PREFIX}:${runId}:${runtimeSessionId}`; } function resolveOpenCodeSecondaryLaneMemberEvidence( lane: MixedSecondaryRuntimeLaneState | undefined, memberName: string ): TeamRuntimeMemberLaunchEvidence | undefined { if (!lane?.result) { return undefined; } return ( lane.result.members[memberName] ?? Object.values(lane.result.members).find((member) => matchesTeamMemberIdentity(member.memberName ?? '', memberName) ) ); } function promoteOpenCodeSecondaryMemberFromCommittedBootstrapEvidence(input: { current: PersistedTeamLaunchMemberState; previous: PersistedTeamLaunchMemberState | null; session: OpenCodeCommittedBootstrapSessionRecord; now: string; }): PersistedTeamLaunchMemberState { const observedAt = input.session.observedAt ?? input.now; const diagnostics = [ ...new Set([ ...filterStaleOpenCodeOverlayDiagnostics(input.current.diagnostics), 'opencode_bootstrap_evidence_committed', ]), ]; const runtimeAlive = true; const livenessKind = input.current.livenessKind === 'runtime_process' || input.current.livenessKind === 'confirmed_bootstrap' ? input.current.livenessKind : 'confirmed_bootstrap'; return { ...input.previous, ...input.current, launchState: 'confirmed_alive', agentToolAccepted: true, bootstrapConfirmed: true, runtimeAlive, hardFailure: false, hardFailureReason: undefined, runtimeRunId: input.session.runId ?? input.current.runtimeRunId, runtimeSessionId: input.session.id, bootstrapEvidenceSource: input.session.source, bootstrapMode: input.session.source === 'app_managed_bootstrap' ? 'app_managed_context' : 'model_tool_checkin', appManagedBootstrapCandidate: input.session.source === 'app_managed_bootstrap' ? input.session.appManagedBootstrapCandidate : undefined, livenessKind, runtimeDiagnostic: input.session.source === 'app_managed_bootstrap' ? 'OpenCode app-managed bootstrap evidence committed.' : 'OpenCode bootstrap evidence committed.', runtimeDiagnosticSeverity: 'info', firstSpawnAcceptedAt: input.current.firstSpawnAcceptedAt ?? input.previous?.firstSpawnAcceptedAt ?? observedAt, lastHeartbeatAt: input.current.lastHeartbeatAt ?? input.previous?.lastHeartbeatAt ?? observedAt, runtimeLastSeenAt: runtimeAlive ? (input.current.runtimeLastSeenAt ?? observedAt) : undefined, lastRuntimeAliveAt: runtimeAlive ? (input.current.lastRuntimeAliveAt ?? input.previous?.lastRuntimeAliveAt ?? observedAt) : input.current.lastRuntimeAliveAt, lastEvaluatedAt: input.now, sources: { ...(input.previous?.sources ?? {}), ...(input.current.sources ?? {}), nativeHeartbeat: true, processAlive: runtimeAlive || undefined, }, diagnostics, }; } function isTmuxNoServerRunningError(error: unknown): boolean { const text = error instanceof Error ? error.message : String(error ?? ''); return ( /no server running on /i.test(text) || /error connecting to .*no such file or directory/i.test(text) ); } interface PendingMemberRestartContext { requestedAt: string; desired: Pick< TeamCreateRequest['members'][number], 'name' | 'role' | 'workflow' | 'isolation' | 'providerId' | 'model' | 'effort' >; } function normalizeTeamAgentRuntimeBackendType( value: string | undefined, isLead: boolean ): TeamAgentRuntimeBackendType | undefined { if (isLead) return 'lead'; const normalized = value?.trim().toLowerCase(); if (normalized === 'tmux' || normalized === 'iterm2' || normalized === 'in-process') { return normalized; } return normalized ? 'process' : undefined; } interface MemberSpawnInboxCursor { timestamp: string; messageId: string; } type LeadInboxMemberSpawnMessage = InboxMessage & { messageId: string }; type LeadInboxLaunchReconcileMessage = Pick< InboxMessage, 'from' | 'text' | 'timestamp' | 'messageId' >; function compareMemberSpawnInboxCursor( left: MemberSpawnInboxCursor, right: MemberSpawnInboxCursor ): number { const leftMs = Date.parse(left.timestamp); const rightMs = Date.parse(right.timestamp); const leftValid = Number.isFinite(leftMs); const rightValid = Number.isFinite(rightMs); if (leftValid && rightValid && leftMs !== rightMs) { return leftMs - rightMs; } if (leftValid !== rightValid) { return leftValid ? -1 : 1; } return left.messageId.localeCompare(right.messageId); } function toMemberSpawnInboxCursor( message: Pick ): MemberSpawnInboxCursor | null { const messageId = typeof message.messageId === 'string' ? message.messageId.trim() : ''; if (!messageId) { return null; } return { timestamp: message.timestamp, messageId, }; } function maxMemberSpawnInboxCursor( left: MemberSpawnInboxCursor | undefined, right: MemberSpawnInboxCursor ): MemberSpawnInboxCursor { if (!left) { return right; } return compareMemberSpawnInboxCursor(left, right) >= 0 ? left : right; } function isMemberSpawnHeartbeatTimestampNewer( previous: string | undefined, incoming: string | undefined ): boolean { const normalizedIncoming = incoming?.trim(); if (!normalizedIncoming) { return false; } const normalizedPrevious = previous?.trim(); if (!normalizedPrevious) { return true; } const previousMs = Date.parse(normalizedPrevious); const incomingMs = Date.parse(normalizedIncoming); if (Number.isFinite(previousMs) && Number.isFinite(incomingMs)) { return incomingMs > previousMs; } return normalizedIncoming > normalizedPrevious; } function stripWrappedCliFlagValue(raw: string | undefined): string | undefined { const trimmed = raw?.trim(); if (!trimmed) { return undefined; } if ( (trimmed.startsWith('"') && trimmed.endsWith('"')) || (trimmed.startsWith("'") && trimmed.endsWith("'")) ) { const unwrapped = trimmed.slice(1, -1).trim(); return unwrapped.length > 0 ? unwrapped : undefined; } return trimmed; } function extractCliFlagValue(command: string, flagName: string): string | undefined { const escapedFlag = flagName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); const match = new RegExp(`(?:^|\\s)${escapedFlag}\\s+("([^"]*)"|'([^']*)'|([^\\s]+))`).exec( command ); if (!match) { return undefined; } return stripWrappedCliFlagValue(match[2] ?? match[3] ?? match[4] ?? match[1]); } export function shouldAcceptDeterministicBootstrapEvent(params: { runId: string; teamName: string; lastSeq: number; msg: Record; }): { accept: boolean; nextSeq: number } { const msgRunId = typeof params.msg.run_id === 'string' ? params.msg.run_id.trim() : ''; if (msgRunId && msgRunId !== params.runId) { return { accept: false, nextSeq: params.lastSeq }; } const msgTeamName = typeof params.msg.team_name === 'string' ? params.msg.team_name.trim() : ''; if (msgTeamName && msgTeamName !== params.teamName) { return { accept: false, nextSeq: params.lastSeq }; } const seq = typeof params.msg.seq === 'number' ? params.msg.seq : NaN; if (Number.isFinite(seq)) { if (!Number.isInteger(seq) || seq <= params.lastSeq) { return { accept: false, nextSeq: params.lastSeq }; } return { accept: true, nextSeq: seq }; } return { accept: true, nextSeq: params.lastSeq }; } function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } async function waitForPidsToExit( pids: readonly number[], opts: { timeoutMs: number; pollMs: number } ): Promise { if (pids.length === 0) { return []; } const deadline = Date.now() + opts.timeoutMs; let remainingPids = [...new Set(pids)]; while (Date.now() < deadline) { remainingPids = remainingPids.filter((pid) => isProcessAlive(pid)); if (remainingPids.length === 0) { return []; } await sleep(opts.pollMs); } return remainingPids; } async function waitForTmuxPanesToExit( paneIds: readonly string[], opts: { timeoutMs: number; pollMs: number } ): Promise { const normalizedPaneIds = [...new Set(paneIds.map((paneId) => paneId.trim()).filter(Boolean))]; if (normalizedPaneIds.length === 0) { return []; } const deadline = Date.now() + opts.timeoutMs; let remainingPaneIds = normalizedPaneIds; let lastError: unknown = null; while (Date.now() < deadline) { let livePanePidById: Map; try { livePanePidById = await listTmuxPanePidsForCurrentPlatform(remainingPaneIds); lastError = null; } catch (error) { if (isTmuxNoServerRunningError(error)) { return []; } lastError = error; await sleep(opts.pollMs); continue; } remainingPaneIds = remainingPaneIds.filter((paneId) => livePanePidById.has(paneId)); if (remainingPaneIds.length === 0) { return []; } await sleep(opts.pollMs); } if (lastError) { throw lastError instanceof Error ? lastError : new Error(getErrorMessage(lastError)); } return remainingPaneIds; } async function waitForChildProcessToExit( child: ChildProcess | null | undefined, timeoutMs: number ): Promise { if (!child?.pid || !isProcessAlive(child.pid)) { return; } await new Promise((resolve) => { let settled = false; let timeoutHandle: ReturnType | null = null; const finish = (): void => { if (settled) { return; } settled = true; if (timeoutHandle) { clearTimeout(timeoutHandle); } child.off('close', finish); child.off('exit', finish); child.off('error', finish); resolve(); }; timeoutHandle = setTimeout(finish, timeoutMs); child.once('close', finish); child.once('exit', finish); child.once('error', finish); }); } async function tryReadRegularFileUtf8( filePath: string, opts: { timeoutMs: number; maxBytes: number } ): Promise { let stat: fs.Stats; try { stat = await fs.promises.stat(filePath); } catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') { return null; } return null; } if (!stat.isFile() || stat.size > opts.maxBytes) { return null; } try { return await readFileUtf8WithTimeout(filePath, opts.timeoutMs); } catch (error) { if (error instanceof FileReadTimeoutError) { return null; } if ((error as NodeJS.ErrnoException).code === 'ENOENT') { return null; } return null; } } async function ensureCwdExists(cwd: string): Promise { await fs.promises.mkdir(cwd, { recursive: true }); const stat = await fs.promises.stat(cwd); if (!stat.isDirectory()) { throw new Error('cwd must be a directory'); } } function isMissingCwdSpawnError(message: string): boolean { const lower = message.toLowerCase(); return lower.includes('spawn ') && lower.includes(' enoent'); } async function pathExistsAsDirectory(candidatePath: string): Promise { try { const stat = await fs.promises.stat(candidatePath); return stat.isDirectory(); } catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') { return false; } throw error; } } /** @deprecated Use wrapAgentBlock from @shared/constants/agentBlocks instead. */ const wrapInAgentBlock = wrapAgentBlock; /** * Unconditionally clears all post-compact reminder state on a run. * Called from cleanupRun, cancel, and error paths. */ function clearPostCompactReminderState(run: ProvisioningRun): void { run.pendingPostCompactReminder = false; run.postCompactReminderInFlight = false; run.suppressPostCompactReminderOutput = false; } function clearGeminiPostLaunchHydrationState(run: ProvisioningRun): void { run.pendingGeminiPostLaunchHydration = false; run.geminiPostLaunchHydrationInFlight = false; run.suppressGeminiPostLaunchHydrationOutput = false; } function buildProvisioningTraceDetail( extras?: Pick< TeamProvisioningProgress, 'pid' | 'error' | 'warnings' | 'configReady' | 'launchDiagnostics' > ): string | undefined { const parts = [ extras?.pid != null ? `pid=${extras.pid}` : undefined, extras?.configReady === true ? 'configReady=true' : undefined, extras?.error ? `error=${extras.error}` : undefined, extras?.warnings?.length ? `warnings=${extras.warnings.join('; ')}` : undefined, extras?.launchDiagnostics?.length ? `launchDiagnostics=${extras.launchDiagnostics.length}` : undefined, ].filter((part): part is string => Boolean(part)); return parts.length > 0 ? parts.join(' | ') : undefined; } function appendProvisioningTrace( run: ProvisioningRun, state: Exclude, message: string, detail?: string ): void { run.provisioningTraceLines ??= []; run.lastProvisioningTraceKey ??= null; const key = `${state}\u0000${message}\u0000${detail ?? ''}`; if (run.lastProvisioningTraceKey === key) { return; } run.lastProvisioningTraceKey = key; run.provisioningTraceLines.push( buildProgressTraceLine({ timestamp: nowIso(), state, message, detail, }) ); if (run.provisioningTraceLines.length > PROVISIONING_TRACE_STORAGE_LIMIT) { run.provisioningTraceLines.splice( 0, run.provisioningTraceLines.length - PROVISIONING_TRACE_STORAGE_LIMIT ); } } function buildProvisioningLiveOutput(run: ProvisioningRun): string | undefined { return buildProgressLiveOutput(run.provisioningTraceLines, run.provisioningOutputParts); } function initializeProvisioningTrace(run: ProvisioningRun): void { appendProvisioningTrace(run, run.progress.state, run.progress.message); run.progress = { ...run.progress, assistantOutput: buildProvisioningLiveOutput(run) ?? run.progress.assistantOutput, }; } function emitProvisioningCheckpoint(run: ProvisioningRun, message: string, detail?: string): void { appendProvisioningTrace(run, run.progress.state, message, detail); run.progress = { ...run.progress, updatedAt: nowIso(), assistantOutput: buildProvisioningLiveOutput(run) ?? run.progress.assistantOutput, }; run.onProgress(run.progress); } function updateProgress( run: ProvisioningRun, state: Exclude, message: string, extras?: Pick< TeamProvisioningProgress, | 'pid' | 'error' | 'warnings' | 'cliLogsTail' | 'configReady' | 'messageSeverity' | 'launchDiagnostics' > ): TeamProvisioningProgress { if (shouldIgnoreProvisioningProgressRegression(run.progress.state, state)) { return run.progress; } // Cap assistant output on every progress tick. `updateProgress` is invoked // from ~20 event-driven sites (auth retries, stall warnings, spawn events), // and an unbounded `provisioningOutputParts.join` was part of the same OOM // class that `emitLogsProgress` already guards against. appendProvisioningTrace(run, state, message, buildProvisioningTraceDetail(extras)); const assistantOutput = buildProvisioningLiveOutput(run) ?? run.progress.assistantOutput; run.progress = { ...run.progress, state, message, updatedAt: nowIso(), pid: extras?.pid ?? run.progress.pid, error: extras?.error, warnings: extras?.warnings, cliLogsTail: extras?.cliLogsTail ?? run.progress.cliLogsTail, assistantOutput, configReady: extras?.configReady ?? run.progress.configReady, messageSeverity: extras?.messageSeverity, launchDiagnostics: boundLaunchDiagnostics( extras?.launchDiagnostics ?? buildLaunchDiagnosticsFromRun(run) ?? run.progress.launchDiagnostics ), }; return run.progress; } interface AgentTeamsMcpConfigEntry { command?: unknown; args?: unknown; env?: unknown; cwd?: unknown; } interface AgentTeamsMcpConfigFile { mcpServers?: Record; } interface AgentTeamsMcpLaunchSpec { command: string; args: string[]; cwd?: string; env: Record; } interface McpJsonRpcErrorPayload { code?: number; message?: string; } interface McpJsonRpcResponse { id?: number; result?: TResult; error?: McpJsonRpcErrorPayload; } interface McpToolsListResult { tools?: { name?: string; _meta?: Record; }[]; } interface McpToolCallResult { content?: { type?: string; text?: string; }[]; isError?: boolean; } interface AgentTeamsMcpValidationFixture { claudeDir: string; teamName: string; memberName: string; } function isStringArray(value: unknown): value is string[] { return Array.isArray(value) && value.every((entry) => typeof entry === 'string'); } function normalizeRecordStringValues(value: unknown): Record { if (!value || typeof value !== 'object') { return {}; } return Object.fromEntries( Object.entries(value).flatMap(([key, entry]) => typeof entry === 'string' ? [[key, entry]] : [] ) ); } function extractLogsTail( stdoutBuffer: string | undefined, stderrBuffer: string | undefined ): string | undefined { const trimmed = buildCombinedLogs(stdoutBuffer, stderrBuffer).trim(); if (trimmed.length === 0) { return undefined; } return trimmed.slice(-UI_LOGS_TAIL_LIMIT); } /** * Builds provisioning CLI logs from the line-buffered claudeLogLines array * instead of the byte-capped stdoutBuffer/stderrBuffer ring buffers. * * claudeLogLines already contains [stdout]/[stderr] markers and individual lines * in chronological order (up to CLAUDE_LOG_LINES_LIMIT = 50 000 lines), so it * does not suffer from the 64 KB ring-buffer truncation that causes the raw * stdoutBuffer to lose older assistant messages. * * Returns the full launch log history preserved in claudeLogLines. Falls back * to the legacy tail extraction only when claudeLogLines is empty (e.g. early * in provisioning before any output has been line-split). */ function extractCliLogsFromRun(run: ProvisioningRun): string | undefined { const claudeLogLines = Array.isArray(run.claudeLogLines) ? run.claudeLogLines : []; if (claudeLogLines.length > 0) { const joined = claudeLogLines.join('\n').trim(); if (joined.length === 0) { return undefined; } return joined; } return extractLogsTail(run.stdoutBuffer, run.stderrBuffer); } interface RetainedClaudeLogsSnapshot { lines: string[]; updatedAt?: string; } interface PersistedTranscriptClaudeLogsCacheEntry { transcriptPath: string; mtimeMs: number; size: number; snapshot: RetainedClaudeLogsSnapshot; } function buildRetainedClaudeLogsSnapshot(run: ProvisioningRun): RetainedClaudeLogsSnapshot | null { const claudeLogLines = Array.isArray(run.claudeLogLines) ? run.claudeLogLines : []; if (claudeLogLines.length > 0) { return { lines: [...claudeLogLines], updatedAt: run.claudeLogsUpdatedAt, }; } const fallback = extractCliLogsFromRun(run); if (!fallback) { return null; } const lines = fallback .split('\n') .map((line) => (line.endsWith('\r') ? line.slice(0, -1) : line)) .filter((line) => line.length > 0); if (lines.length === 0) { return null; } return { lines, updatedAt: run.claudeLogsUpdatedAt ?? run.progress.updatedAt, }; } function sliceClaudeLogs( linesChronological: string[], updatedAt: string | undefined, query?: { offset?: number; limit?: number } ): { lines: string[]; total: number; hasMore: boolean; updatedAt?: string } { const offsetRaw = query?.offset ?? 0; const limitRaw = query?.limit ?? 100; const offset = Number.isFinite(offsetRaw) ? Math.max(0, Math.floor(offsetRaw)) : 0; const limit = Number.isFinite(limitRaw) ? Math.max(1, Math.min(1000, Math.floor(limitRaw))) : 100; const total = linesChronological.length; if (total === 0) { return { lines: [], total: 0, hasMore: false, updatedAt }; } const newestExclusive = Math.max(0, total - offset); const oldestInclusive = Math.max(0, newestExclusive - limit); const normalizeLine = (line: string): string => { // Back-compat: older builds prefixed every line with "[stdout] " / "[stderr] " if (line.startsWith('[stdout] ') && line !== '[stdout]') { return line.slice('[stdout] '.length); } if (line.startsWith('[stderr] ') && line !== '[stderr]') { return line.slice('[stderr] '.length); } return line; }; const lines = linesChronological .slice(oldestInclusive, newestExclusive) .map(normalizeLine) .toReversed(); return { lines, total, hasMore: oldestInclusive > 0, updatedAt, }; } /** * Emit a throttled progress update for the renderer. Payloads are capped to a * tail window so that the hot emission path (called every LOG_PROGRESS_THROTTLE_MS * under streaming output) cannot accumulate into multi-megabyte IPC messages * that would OOM the renderer's Zustand state. The full history stays in * `run.claudeLogLines` / `run.provisioningOutputParts` for diagnostics and * one-shot completion emissions that intentionally use `extractCliLogsFromRun`. */ function emitLogsProgress(run: ProvisioningRun): void { // Prefer the line-buffered history (already chronological with [stdout]/[stderr] // markers) and fall back to the legacy ring-buffer tail only when no lines // have been captured yet (early in provisioning). const logsTail = buildProgressLogsTail(run.claudeLogLines) ?? extractLogsTail(run.stdoutBuffer, run.stderrBuffer); const assistantOutput = buildProvisioningLiveOutput(run); const assistantOutputChanged = assistantOutput !== undefined && assistantOutput !== run.progress.assistantOutput; if (!logsTail && !assistantOutputChanged) { return; } run.progress = { ...run.progress, updatedAt: nowIso(), ...(logsTail !== undefined && { cliLogsTail: logsTail }), ...(assistantOutputChanged && { assistantOutput }), }; run.onProgress(run.progress); } interface CachedProbeResult { cacheKey: string; claudePath: string; authSource: ProvisioningAuthSource; warning?: string; cachedAtMs: number; } interface ProbeResult { claudePath: string; authSource: ProvisioningAuthSource; warning?: string; } type AuthWarningSource = 'probe' | 'stdout' | 'stderr' | 'assistant' | 'pre-complete'; const cachedProbeResults = new Map(); const probeInFlightByKey = new Map>(); function createProbeCacheKey(cwd: string, providerId: TeamProviderId | undefined): string { return `${path.resolve(cwd)}::${getClaudeBasePath()}::${resolveTeamProviderId(providerId)}`; } function isTransientProbeWarning(warning: string): boolean { const lower = warning.toLowerCase(); return ( lower.includes('timeout running:') || lower.includes('did not complete') || lower.includes('runtime status was unavailable') || lower.includes('runtime status check did not complete') || lower.includes('timed out') || lower.includes('etimedout') || lower.includes('econnreset') || lower.includes('eai_again') ); } function isBinaryProbeWarning(warning: string): boolean { const lower = warning.toLowerCase(); return ( (lower.includes('spawn ') && lower.includes(' enoent')) || lower.includes('eacces') || lower.includes('enoexec') || lower.includes('bad cpu type in executable') || lower.includes('image not found') ); } interface PendingInboxRelayCandidate { recipient: string; sourceMessageId: string; normalizedText: string; normalizedSummary: string; queuedAtMs: number; } interface NativeSameTeamFingerprint { id: string; from: string; text: string; summary: string; seenAt: number; } interface OpenCodeMemberInboxDelivery { delivered: boolean; accepted?: boolean; responsePending?: boolean; acceptanceUnknown?: boolean; responseState?: NonNullable['state']; ledgerStatus?: OpenCodePromptDeliveryStatus; ledgerRecordId?: string; laneId?: string; visibleReplyMessageId?: string; visibleReplyCorrelation?: | 'relayOfMessageId' | 'direct_child_message_send' | 'plain_assistant_text'; queuedBehindMessageId?: string; reason?: string; diagnostics?: string[]; userVisibleImpact?: OpenCodeRuntimeDeliveryUserVisibleImpact; } class InboxRelayInFlightTimeoutError extends Error { constructor(message: string) { super(message); this.name = 'InboxRelayInFlightTimeoutError'; } } function isInboxRelayInFlightTimeoutError(error: unknown): error is InboxRelayInFlightTimeoutError { return error instanceof InboxRelayInFlightTimeoutError; } type OpenCodeVisibleReplyCorrelation = NonNullable< OpenCodePromptDeliveryLedgerRecord['visibleReplyCorrelation'] >; interface OpenCodeRecoveredVisibleReplyProof { visibleReply: OpenCodeVisibleReplyProof; visibleReplyCorrelation: OpenCodeVisibleReplyCorrelation; diagnostics: string[]; } interface OpenCodeMemberDirectory { config: TeamConfig | null; teamMeta: Awaited> | null; metaMembers: Awaited>; } type OpenCodeMemberIdentityResolution = | { ok: true; canonicalMemberName: string; laneId: string; laneIdentity: ReturnType; configMember?: TeamMember; metaMember?: TeamMember; memberRuntimeCwd?: string; } | { ok: false; reason: 'recipient_is_not_opencode' | 'recipient_removed' | 'opencode_recipient_unavailable'; }; interface OpenCodeMemberInboxRelayResult { relayed: number; attempted: number; delivered: number; failed: number; lastDelivery?: OpenCodeMemberInboxDelivery; diagnostics?: string[]; } interface LiveInboxRelayResult { kind: | 'ignored' | 'native_lead' | 'native_member_noop' | 'opencode_member' | 'opencode_lead_unsupported'; relayed: number; diagnostics?: string[]; lastDelivery?: OpenCodeMemberInboxDelivery; } interface OpenCodeMemberInboxRelayOptions { onlyMessageId?: string; source?: 'watcher' | 'ui-send' | 'manual' | 'watchdog' | 'member-work-sync-review-pickup'; deliveryMetadata?: { replyRecipient?: string; actionMode?: AgentActionMode; taskRefs?: TaskRef[]; }; } type MemberWorkSyncProofMissingRecoveryScheduler = (input: { teamName: string; memberName: string; originalMessageId: string; taskRefs?: TaskRef[]; reason?: string; }) => Promise | unknown; type MemberWorkSyncAcceptedReportChecker = (input: { teamName: string; memberName: string; }) => Promise | boolean; function normalizeSameTeamText(text: string): string { return text.trim().replace(/\r\n/g, '\n'); } const SUPPRESSED_LEAD_RELAY_STATE_PHRASES = [ 'open', 'closed', 'merged', 'approved', 'complete', 'completed', 'done', 'blocked', 'pending', 'in_progress', 'in progress', 'needsfix', 'needs fix', 'in review', 'clear', ] as const; function startsWithSuppressedLeadRelayStatePhrase(text: string): boolean { const lowerText = text.toLowerCase(); return SUPPRESSED_LEAD_RELAY_STATE_PHRASES.some((phrase) => { if (!lowerText.startsWith(phrase)) { return false; } const nextChar = lowerText.charAt(phrase.length); return nextChar.length === 0 || !/[a-z0-9_]/i.test(nextChar); }); } function hasSuppressedLeadRelayStatePredicate(normalized: string): boolean { const match = /\b(?:is|are|was|were|stays?|still|now)\s+/i.exec(normalized); if (!match) { return false; } return startsWithSuppressedLeadRelayStatePhrase(normalized.slice(match.index + match[0].length)); } function shouldSuppressUnverifiedLeadRelayStateLine(text: string): boolean { const normalized = text.trim().replace(/\s+/g, ' '); if (normalized.length === 0) { return false; } const hasStateSubject = /#[a-z0-9]{4,}/i.test(normalized) || /\bpr\s*#?\d+\b/i.test(normalized) || /\bpull request\b/i.test(normalized) || /\b(?:task|tasks|kanban|board|review|approval|merge|merged|branch|queue|worktree|commit|mergecommit|mergedat)\b/i.test( normalized ); if (!hasStateSubject) { return false; } return ( /\b(?:confirmed|verified|already|claims?|false|phantom|ground[- ]truth)\b/i.test(normalized) || /\b(?:done|complete(?:d)?|approved|merged|closed|blocked|resolved|failed|succeeded)\b/i.test( normalized ) || hasSuppressedLeadRelayStatePredicate(normalized) || /\b(?:mergecommit|mergedat)\s*=\s*(?:null|[^\s,;]+)/i.test(normalized) || /\bqueue\b.*\bclear\b/i.test(normalized) ); } function getOpenCodeInboxRelayPriority( message: Pick ): number { if (message.messageKind === 'member_work_sync_nudge') { return 30; } if (message.source === 'system_notification') { return 20; } return 0; } function getMemberInboxRelayPriority( message: Pick ): number { if (message.messageKind === 'member_work_sync_nudge') { return 30; } return 0; } function getLeadInboxRelayPriority(message: Pick): number { if (message.messageKind === 'member_work_sync_nudge') { return 30; } return 0; } function compareInboxRelayMessages( a: Pick & { messageId: string }, b: Pick & { messageId: string }, getPriority: (message: Pick) => number ): number { const priorityDelta = getPriority(b) - getPriority(a); if (priorityDelta !== 0) return priorityDelta; const aTime = Date.parse(a.timestamp); const bTime = Date.parse(b.timestamp); if (Number.isFinite(aTime) && Number.isFinite(bTime)) { const timeDelta = aTime - bTime; if (timeDelta !== 0) return timeDelta; } else if (Number.isFinite(aTime)) { return -1; } else if (Number.isFinite(bTime)) { return 1; } return a.messageId.localeCompare(b.messageId); } function compareOpenCodeInboxRelayMessagesByPriority( a: Pick & { messageId: string }, b: Pick & { messageId: string } ): number { return compareInboxRelayMessages(a, b, getOpenCodeInboxRelayPriority); } function compareMemberInboxRelayMessagesByPriority( a: Pick & { messageId: string }, b: Pick & { messageId: string } ): number { return compareInboxRelayMessages(a, b, getMemberInboxRelayPriority); } function compareLeadInboxRelayMessagesByPriority( a: Pick & { messageId: string }, b: Pick & { messageId: string } ): number { return compareInboxRelayMessages(a, b, getLeadInboxRelayPriority); } export class TeamProvisioningService { private readonly runtimeLaneCoordinator = createTeamRuntimeLaneCoordinator(); private readonly providerConnectionService = ProviderConnectionService.getInstance(); private static readonly CLAUDE_LOG_LINES_LIMIT = 50_000; private static readonly BOOTSTRAP_FAILURE_TAIL_BYTES = 128 * 1024; // A transcript whose mtime predates the lookup window (minus slack for clock skew // between the line timestamp source and the filesystem) cannot hold a line at/after // sinceMs, so it is skipped without opening it. The slack keeps detection safe. private static readonly BOOTSTRAP_TRANSCRIPT_MTIME_SLACK_MS = 5_000; private static readonly RECENT_CROSS_TEAM_DELIVERY_TTL_MS = 10 * 60 * 1000; private static readonly PENDING_INBOX_RELAY_TTL_MS = 2 * 60 * 1000; private static readonly SAME_TEAM_NATIVE_DELIVERY_GRACE_MS = 15_000; private static readonly SAME_TEAM_NATIVE_FINGERPRINT_TTL_MS = 60_000; private static readonly SAME_TEAM_MATCH_WINDOW_MS = 30_000; private static readonly SAME_TEAM_RUN_START_SKEW_MS = 1_000; private static readonly SAME_TEAM_PERSIST_RETRY_MS = 2_000; private static readonly AGENT_RUNTIME_SNAPSHOT_CACHE_TTL_MS = 2_000; private static readonly PERSISTED_AGENT_RUNTIME_SNAPSHOT_CACHE_TTL_MS = 10_000; private static readonly RUNTIME_RESOURCE_TELEMETRY_CACHE_TTL_MS = 60_000; private static readonly RUNTIME_RESOURCE_TELEMETRY_FAILURE_CACHE_TTL_MS = 10_000; private static readonly RUNTIME_RESOURCE_SAMPLE_MIN_INTERVAL_MS = 30_000; private static readonly AGENT_RUNTIME_RESOURCE_HISTORY_LIMIT = 60; private static readonly BOOTSTRAP_TRANSCRIPT_OUTCOME_CACHE_MAX_ENTRIES = 2_048; private static readonly PERSISTED_BOOTSTRAP_TRANSCRIPT_OUTCOME_LOOKUP_CACHE_TTL_MS = 10_000; private static readonly MAX_RUNTIME_TREE_PIDS_PER_ROOT = 64; private static readonly MAX_RUNTIME_USAGE_PIDS_PER_SNAPSHOT = 512; private static readonly RUNTIME_PROCESS_TABLE_TIMEOUT_MS = 1_500; private static readonly RUNTIME_WINDOWS_PROCESS_TABLE_TIMEOUT_MS = 1_500; private static readonly RUNTIME_LIVENESS_PROCESS_TABLE_CACHE_TTL_MS = 5_000; private static readonly RUNTIME_LIVENESS_PROCESS_TABLE_FAILURE_CACHE_TTL_MS = 2_000; private static readonly RUNTIME_PROCESS_USAGE_CACHE_TTL_MS = 30_000; private static readonly RUNTIME_PROCESS_USAGE_CACHE_MAX_ENTRIES = 4_096; private static readonly RUNTIME_PIDUSAGE_BATCH_TIMEOUT_MS = 2_000; private static readonly RUNTIME_PIDUSAGE_SINGLE_TIMEOUT_MS = 750; private static readonly RUNTIME_PIDUSAGE_FALLBACK_CONCURRENCY = 16; private static readonly MEMBER_SPAWN_STATUS_SNAPSHOT_CACHE_TTL_MS = 500; private static readonly PERSISTED_MEMBER_SPAWN_STATUS_SNAPSHOT_CACHE_TTL_MS = 5_000; private static readonly LAUNCH_STATE_NOOP_REFRESH_MS = 15_000; private static readonly RETAINED_PROVISIONING_PROGRESS_TTL_MS = 5 * 60_000; private static readonly OPENCODE_RUNTIME_DELIVERY_ADVISORY_EVENT_TTL_MS = 24 * 60 * 60_000; private static readonly OPENCODE_RUNTIME_DELIVERY_LEAD_NOTICE_TTL_MS = 24 * 60 * 60_000; private static readonly INBOX_RELAY_IN_FLIGHT_TIMEOUT_MS = 2 * 60_000; private readonly runs = new Map(); private readonly provisioningRunByTeam = new Map(); private readonly aliveRunByTeam = new Map(); private readonly runtimeAdapterProgressByRunId = new Map(); private retainedProvisioningProgressByRunId: Map | undefined = new Map(); private retainedProvisioningProgressTimersByRunId: | Map> | undefined = new Map>(); private readonly runtimeAdapterTraceLinesByRunId = new Map(); private readonly runtimeAdapterTraceKeyByRunId = new Map(); private readonly runtimeToolApprovalCoordinator = new RuntimeToolApprovalCoordinator({ getSettings: (teamName) => this.getToolApprovalSettings(teamName), answerApproval: ({ entry, allow, message }) => this.answerRuntimeToolApproval(entry, allow, message), emitApprovalEvent: (event) => this.emitToolApprovalEvent(event), showApprovalNotification: (approval) => this.maybeShowToolApprovalOsNotification(undefined, approval), dismissApprovalNotification: (requestId) => this.dismissApprovalNotification(requestId), logWarning: (message) => logger.warn(message), }); private readonly runtimeAdapterRunByTeam = new Map< string, { runId: string; providerId: TeamProviderId; cwd?: string; members?: Record; } >(); private readonly cancelledRuntimeAdapterRunIds = new Set(); private stopAllTeamsGeneration = 0; private readonly transientProbeProcesses = new Set>(); private readonly secondaryRuntimeRunByTeam = new Map< string, Map< string, { runId: string; providerId: 'opencode'; laneId: string; memberName: string; cwd?: string } > >(); private readonly stoppingSecondaryRuntimeTeams = new Set(); private readonly retainedClaudeLogsByTeam = new Map(); private readonly persistedTranscriptClaudeLogsCache = new Map< string, PersistedTranscriptClaudeLogsCacheEntry >(); private readonly bootstrapTranscriptOutcomeCache = new Map< string, BootstrapTranscriptOutcomeCacheEntry >(); // Shared parsed-tail cache keyed by filePath (validated by mtime+size) so the // same growing transcript is read + JSON.parsed ONCE per change instead of once // per member per poll. The per-member outcome scan below is unchanged. private readonly parsedBootstrapTranscriptTailCache = new Map< string, ParsedBootstrapTranscriptTailCacheEntry >(); private readonly bootstrapTranscriptOutcomeLookupCache = new Map< string, BootstrapTranscriptOutcomeLookupCacheEntry >(); private readonly teamOpLocks = new Map>(); private readonly leadInboxRelayInFlight = new Map>(); private readonly relayedLeadInboxMessageIds = new Map>(); private readonly memberInboxRelayInFlight = new Map>(); private readonly openCodeMemberInboxRelayInFlight = new Map< string, Promise >(); private readonly openCodePromptDeliveryWatchdogTimers = new Map(); private readonly openCodePromptDeliveryWatchdogDeadlines = new Map(); private readonly openCodeRuntimeDeliveryAdvisoryReviewTimers = new Map(); private readonly openCodeRuntimeDeliveryAdvisoryEventSentAt = new Map(); private readonly openCodeRuntimeDeliveryLeadNoticeSentAt = new Map(); private readonly openCodeRuntimeDeliveryProofReader = new OpenCodeRuntimeDeliveryProofReader(); private readonly openCodePromptDeliveryWatchdogQueue: { teamName: string; run: () => Promise; }[] = []; private openCodePromptDeliveryWatchdogInFlight = 0; private openCodePromptDeliveryWatchdogDisabledLogged = false; private readonly openCodePromptDeliveryWatchdogInFlightByTeam = new Map(); private readonly relayedMemberInboxMessageIds = new Map>(); private readonly pendingCrossTeamFirstReplies = new Map>(); private readonly recentCrossTeamLeadDeliveryMessageIds = new Map>(); private readonly liveLeadProcessMessages = new Map(); private readonly recentSameTeamNativeFingerprints = new Map< string, NativeSameTeamFingerprint[] >(); private readonly agentRuntimeSnapshotCache = new Map< string, { expiresAtMs: number; snapshot: TeamAgentRuntimeSnapshot } >(); private readonly agentRuntimeResourceHistoryByTeam = new Map< string, Map >(); private readonly runtimeProcessRowsForUsageSnapshotByTeam = new Map< string, RuntimeProcessRowsCacheEntry >(); private readonly runtimeProcessUsageStatsCacheByPid = new Map< number, { expiresAtMs: number; stats: RuntimeProcessUsageStats | null; } >(); private readonly persistedTeamConfigCache = new Map(); private readonly agentRuntimeSnapshotInFlightByTeam = new Map< string, { generationAtStart: number; runIdAtStart: string | null; promise: Promise; } >(); private readonly liveTeamAgentRuntimeMetadataCache = new Map< string, { expiresAtMs: number; metadata: Map; runId: string | null; } >(); private readonly liveTeamAgentRuntimeMetadataInFlightByTeam = new Map< string, { generationAtStart: number; runIdAtStart: string | null; promise: Promise>; } >(); private readonly runtimeSnapshotCacheGenerationByTeam = new Map(); private readonly memberSpawnStatusesSnapshotCache = new Map< string, { expiresAtMs: number; generation: number; runId: string | null; snapshot: MemberSpawnStatusesSnapshot; } >(); private readonly memberSpawnStatusesInFlightByTeam = new Map< string, { generationAtStart: number; runIdAtStart: string; promise: Promise; } >(); private readonly memberSpawnStatusesCacheGenerationByTeam = new Map(); private readonly launchStateStore = new TeamLaunchStateStore(); private readonly launchFailureArtifactPackRunIds = new Set(); private readonly launchStateStoreQueue = new Map>(); private readonly launchStateWrittenRunIdByTeam = new Map(); private readonly failedOpenCodeSecondaryRetryInFlightByTeam = new Map< string, Promise >(); private readonly memberLifecycleOperations = new Map(); private memberRuntimeAdvisoryInvalidator: | ((teamName: string, memberName: string) => void) | null = null; private memberWorkSyncProofMissingRecoveryScheduler: MemberWorkSyncProofMissingRecoveryScheduler | null = null; private memberWorkSyncAcceptedReportChecker: MemberWorkSyncAcceptedReportChecker | null = null; private readonly memberLogsFinder: TeamMemberLogsFinder; private readonly transcriptProjectResolver: TeamTranscriptProjectResolver; private readonly taskActivityIntervalService = new TeamTaskActivityIntervalService(); private readonly leadTaskActivitySyncedRunKeys = new Set(); private readonly crashRepairedActivityIntervalsByTeam = new Set(); private readonly pendingCrashRepairSnapshotByTeam = new Map< string, PersistedTeamLaunchSnapshot | null >(); private teamChangeEmitter: ((event: TeamChangeEvent) => void) | null = null; private helpOutputCache: string | null = null; private helpOutputCacheTime = 0; private static readonly HELP_CACHE_TTL_MS = 5 * 60 * 1000; private toolApprovalSettingsByTeam = new Map(); private pendingTimeouts = new Map(); private inFlightResponses = new Set(); private readonly prepareForProvisioningInFlight = new Map< string, Promise >(); private runtimeAdapterRegistry: TeamRuntimeAdapterRegistry | null = null; private controlApiBaseUrlResolver: (() => Promise) | null = null; private workspaceTrustCoordinator: WorkspaceTrustCoordinator | null = null; private runtimeTurnSettledHookSettingsProvider: | ((input: { provider: RuntimeTurnSettledProvider }) => Promise | null>) | null = null; private runtimeTurnSettledEnvironmentProvider: | ((input: { provider: RuntimeTurnSettledProvider }) => Promise | null>) | null = null; private readonly stoppedTeamOpenCodeRuntimeCleanupInFlight = new Map>(); private readonly cleanedStoppedTeamOpenCodeRuntimeLanes = new Set(); private crossTeamSender: | ((request: { fromTeam: string; fromMember: string; toTeam: string; text: string; summary?: string; messageId?: string; timestamp?: string; conversationId?: string; replyToConversationId?: string; }) => Promise) | null = null; constructor( private readonly configReader: TeamConfigReader = new TeamConfigReader(), private readonly inboxReader: TeamInboxReader = new TeamInboxReader(), private readonly membersMetaStore: TeamMembersMetaStore = new TeamMembersMetaStore(), private readonly sentMessagesStore: TeamSentMessagesStore = new TeamSentMessagesStore(), private readonly mcpConfigBuilder: TeamMcpConfigBuilder = new TeamMcpConfigBuilder(), private readonly teamMetaStore: TeamMetaStore = new TeamMetaStore(), private readonly inboxWriter: TeamInboxWriter = new TeamInboxWriter(), private readonly openCodeTaskLogAttributionStore: OpenCodeTaskLogAttributionStore = new OpenCodeTaskLogAttributionStore(), private readonly memberWorktreeManager: TeamMemberWorktreeManager = new TeamMemberWorktreeManager(), private readonly attachmentStore: TeamAttachmentStore = new TeamAttachmentStore() ) { this.memberLogsFinder = new TeamMemberLogsFinder( this.configReader, this.inboxReader, this.membersMetaStore ); this.transcriptProjectResolver = new TeamTranscriptProjectResolver({ getConfig: (teamName) => this.configReader.getConfigSnapshot(teamName), }); this.scheduleStaleAnthropicTeamApiKeyHelperCleanup(); } private repairStaleTaskActivityIntervalsOnce( teamName: string, launchSnapshot?: PersistedTeamLaunchSnapshot | null ): boolean { if (this.crashRepairedActivityIntervalsByTeam.has(teamName)) return true; const repairSnapshot = this.pendingCrashRepairSnapshotByTeam.has(teamName) ? this.pendingCrashRepairSnapshotByTeam.get(teamName) : (launchSnapshot ?? null); const result = this.taskActivityIntervalService.repairStaleIntervalsAfterCrash( teamName, repairSnapshot ); if (result.failed) { if (!this.pendingCrashRepairSnapshotByTeam.has(teamName)) { this.pendingCrashRepairSnapshotByTeam.set(teamName, launchSnapshot ?? null); } return false; } this.pendingCrashRepairSnapshotByTeam.delete(teamName); this.crashRepairedActivityIntervalsByTeam.add(teamName); return true; } private async readTaskActivityRepairLaunchSnapshot( teamName: string ): Promise { const [bootstrapSnapshot, launchSnapshot] = await Promise.all([ readBootstrapLaunchSnapshot(teamName).catch(() => null), this.launchStateStore.read(teamName).catch(() => null), ]); return choosePreferredLaunchSnapshot(bootstrapSnapshot, launchSnapshot); } private writeLaunchFailureArtifactPackBestEffort( run: ProvisioningRun, options: { reason: string; launchSnapshot?: PersistedTeamLaunchSnapshot | null; } ): void { const key = `${run.teamName}:${run.runId}`; if (this.launchFailureArtifactPackRunIds.has(key)) return; this.launchFailureArtifactPackRunIds.add(key); const memberSpawnStatuses = Object.fromEntries(run.memberSpawnStatuses.entries()); const request = run.request as Partial | undefined; void writeTeamLaunchFailureArtifactPack({ teamName: run.teamName, runId: run.runId, reason: options.reason, startedAt: run.startedAt, cwd: request?.cwd ?? '', pid: run.child?.pid ?? run.progress.pid ?? null, providerId: request?.providerId, providerBackendId: request?.providerBackendId, model: request?.model, expectedMembers: run.expectedMembers, effectiveMembers: run.allEffectiveMembers, progress: run.progress, launchSnapshot: options.launchSnapshot ?? null, launchDiagnostics: run.progress.launchDiagnostics ?? buildLaunchDiagnosticsFromRun(run), memberSpawnStatuses, cliLogs: extractCliLogsFromRun(run), progressTraceLines: run.provisioningTraceLines, runtimeAdapterTraceLines: this.runtimeAdapterTraceLinesByRunId.get(run.runId), flags: { isLaunch: run.isLaunch, provisioningComplete: run.provisioningComplete, deterministicBootstrap: run.deterministicBootstrap, workspaceTrustPreflight: run.workspaceTrustDiagnostics ?? null, processKilled: run.processKilled, finalizingByTimeout: run.finalizingByTimeout, cancelRequested: run.cancelRequested, }, }).catch((error: unknown) => { this.launchFailureArtifactPackRunIds.delete(key); logger.warn( `[${run.teamName}] Failed to write launch failure artifact pack: ${ error instanceof Error ? error.message : String(error) }` ); }); } async repairStaleTaskActivityIntervalsBeforeSnapshot(teamName: string): Promise { if (this.crashRepairedActivityIntervalsByTeam.has(teamName)) { return; } const runId = this.getTrackedRunId(teamName); if (runId && this.runs.has(runId)) { return; } const repairSnapshot = await this.readTaskActivityRepairLaunchSnapshot(teamName); const repaired = this.repairStaleTaskActivityIntervalsOnce(teamName, repairSnapshot); if (!repaired) { throw new Error(`Task activity interval repair failed before snapshot for team ${teamName}`); } } private scheduleStaleAnthropicTeamApiKeyHelperCleanup(): void { void cleanupStaleAnthropicTeamApiKeyHelpers({ baseClaudeDir: getClaudeBasePath(), maxAgeMs: 14 * 24 * 60 * 60 * 1000, }).catch((error: unknown) => { logger.warn( `Failed to cleanup stale Anthropic team API-key helper material: ${ error instanceof Error ? error.message : String(error) }` ); }); } private async readConfigSnapshot(teamName: string): Promise { const configReader = this.configReader as TeamConfigReader & { getConfigSnapshot?: (name: string) => Promise; }; return typeof configReader.getConfigSnapshot === 'function' ? configReader.getConfigSnapshot(teamName) : configReader.getConfig(teamName); } private readConfigForObservation(teamName: string): Promise { return this.readConfigSnapshot(teamName); } private readConfigForStrictDecision(teamName: string): Promise { return this.configReader.getConfig(teamName); } private async readOpenCodeMemberDirectory(teamName: string): Promise { const [config, teamMeta, metaMembers] = await Promise.all([ this.readConfigForObservation(teamName).catch(() => null), this.teamMetaStore.getMeta(teamName).catch(() => null), this.membersMetaStore.getMembers(teamName).catch(() => []), ]); return { config, teamMeta, metaMembers }; } private getRuntimeSnapshotCacheGeneration(teamName: string): number { return this.runtimeSnapshotCacheGenerationByTeam.get(teamName) ?? 0; } private getMemberSpawnStatusesCacheGeneration(teamName: string): number { return this.memberSpawnStatusesCacheGenerationByTeam.get(teamName) ?? 0; } private invalidateMemberSpawnStatusesCache(teamName: string): void { this.memberSpawnStatusesCacheGenerationByTeam.set( teamName, this.getMemberSpawnStatusesCacheGeneration(teamName) + 1 ); this.memberSpawnStatusesSnapshotCache.delete(teamName); this.memberSpawnStatusesInFlightByTeam.delete(teamName); } private invalidateRuntimeSnapshotCaches(teamName: string): void { this.runtimeSnapshotCacheGenerationByTeam.set( teamName, this.getRuntimeSnapshotCacheGeneration(teamName) + 1 ); this.agentRuntimeSnapshotCache.delete(teamName); this.agentRuntimeSnapshotInFlightByTeam.delete(teamName); this.liveTeamAgentRuntimeMetadataCache.delete(teamName); this.liveTeamAgentRuntimeMetadataInFlightByTeam.delete(teamName); this.persistedTeamConfigCache.delete(teamName); // Process table rows are TTL-bound. Resource telemetry can use the longer // TTL, while liveness only reuses rows through a short age gate. } private cloneMemberSpawnStatusesSnapshot( snapshot: MemberSpawnStatusesSnapshot ): MemberSpawnStatusesSnapshot { return { ...snapshot, statuses: Object.fromEntries( Object.entries(snapshot.statuses).map(([memberName, entry]) => [ memberName, { ...entry, ...(entry.pendingPermissionRequestIds ? { pendingPermissionRequestIds: [...entry.pendingPermissionRequestIds] } : {}), }, ]) ), ...(snapshot.expectedMembers ? { expectedMembers: [...snapshot.expectedMembers] } : {}), ...(snapshot.summary ? { summary: { ...snapshot.summary } } : {}), }; } private cloneLiveTeamAgentRuntimeMetadata( metadata: ReadonlyMap ): Map { return new Map( [...metadata.entries()].map(([memberName, entry]) => [ memberName, { ...entry, ...(entry.diagnostics ? { diagnostics: [...entry.diagnostics] } : {}), }, ]) ); } private resolveOpenCodeMemberIdentityFromDirectory( teamName: string, memberName: string, directory: OpenCodeMemberDirectory ): OpenCodeMemberIdentityResolution { const normalizedMemberName = memberName.trim(); const configMember = directory.config?.members?.find( (member) => member.name?.trim().toLowerCase() === normalizedMemberName.toLowerCase() ); const metaMember = directory.metaMembers.find( (member) => member.name?.trim().toLowerCase() === normalizedMemberName.toLowerCase() ); if (!configMember && !metaMember) { return { ok: false, reason: 'opencode_recipient_unavailable' }; } const configProvider = (configMember as { provider?: unknown } | undefined)?.provider; const metaProvider = (metaMember as { provider?: unknown } | undefined)?.provider; const providerId = normalizeTeamProviderLike(metaMember?.providerId) ?? normalizeTeamProviderLike(metaProvider) ?? normalizeTeamProviderLike(configMember?.providerId) ?? normalizeTeamProviderLike(configProvider) ?? inferTeamProviderIdFromModel(metaMember?.model ?? configMember?.model); if (providerId !== 'opencode') { return { ok: false, reason: 'recipient_is_not_opencode' }; } const removedAt = metaMember != null ? metaMember.removedAt : (configMember as { removedAt?: unknown } | undefined)?.removedAt; if (removedAt != null) { return { ok: false, reason: 'recipient_removed' }; } const canonicalMemberName = metaMember?.name?.trim() || configMember?.name?.trim() || normalizedMemberName; const runtimeRun = this.runtimeAdapterRunByTeam.get(teamName); if (runtimeRun?.providerId === 'opencode') { const laneIdentity = buildPlannedMemberLaneIdentity({ leadProviderId: 'opencode', member: { name: canonicalMemberName, providerId: 'opencode', }, }); const memberRuntimeCwd = metaMember?.cwd?.trim() || configMember?.cwd?.trim(); return { ok: true, canonicalMemberName, laneId: laneIdentity.laneId, laneIdentity, ...(configMember ? { configMember } : {}), ...(metaMember ? { metaMember } : {}), ...(memberRuntimeCwd ? { memberRuntimeCwd } : {}), }; } const leadMember = directory.config?.members?.find((member) => isLeadMember(member)); const leadProviderId = normalizeOptionalTeamProviderId(directory.teamMeta?.launchIdentity?.providerId) ?? normalizeOptionalTeamProviderId(directory.teamMeta?.providerId) ?? normalizeOptionalTeamProviderId(leadMember?.providerId); const laneIdentity = buildPlannedMemberLaneIdentity({ leadProviderId, member: { name: canonicalMemberName, providerId, }, }); const memberRuntimeCwd = metaMember?.cwd?.trim() || configMember?.cwd?.trim(); return { ok: true, canonicalMemberName, laneId: laneIdentity.laneId, laneIdentity, ...(configMember ? { configMember } : {}), ...(metaMember ? { metaMember } : {}), ...(memberRuntimeCwd ? { memberRuntimeCwd } : {}), }; } setRuntimeAdapterRegistry(registry: TeamRuntimeAdapterRegistry | null): void { this.runtimeAdapterRegistry = registry; } setMemberRuntimeAdvisoryInvalidator( invalidator: ((teamName: string, memberName: string) => void) | null ): void { this.memberRuntimeAdvisoryInvalidator = invalidator; } setMemberWorkSyncProofMissingRecoveryScheduler( scheduler: MemberWorkSyncProofMissingRecoveryScheduler | null ): void { this.memberWorkSyncProofMissingRecoveryScheduler = scheduler; } setMemberWorkSyncAcceptedReportChecker( checker: MemberWorkSyncAcceptedReportChecker | null ): void { this.memberWorkSyncAcceptedReportChecker = checker; } setCrossTeamSender( sender: | ((request: { fromTeam: string; fromMember: string; toTeam: string; text: string; summary?: string; messageId?: string; timestamp?: string; conversationId?: string; replyToConversationId?: string; }) => Promise) | null ): void { this.crossTeamSender = sender; } setControlApiBaseUrlResolver(resolver: (() => Promise) | null): void { this.controlApiBaseUrlResolver = resolver; } setWorkspaceTrustCoordinator(coordinator: WorkspaceTrustCoordinator | null): void { this.workspaceTrustCoordinator = coordinator; } setRuntimeTurnSettledHookSettingsProvider( provider: | ((input: { provider: RuntimeTurnSettledProvider; }) => Promise | null>) | null ): void { this.runtimeTurnSettledHookSettingsProvider = provider; } setRuntimeTurnSettledEnvironmentProvider( provider: | ((input: { provider: RuntimeTurnSettledProvider; }) => Promise | null>) | null ): void { this.runtimeTurnSettledEnvironmentProvider = provider; } private toWorkspaceTrustProvider(providerId: TeamProviderId): WorkspaceTrustProvider { return providerId === 'anthropic' ? 'claude' : providerId; } private collectWorkspaceTrustProviders(input: { leadProviderId?: TeamProviderId; members: TeamCreateRequest['members']; }): WorkspaceTrustProvider[] { const providers = new Set(['claude']); providers.add(this.toWorkspaceTrustProvider(resolveTeamProviderId(input.leadProviderId))); for (const member of input.members) { const providerId = normalizeTeamMemberProviderId(member.providerId) ?? normalizeTeamMemberProviderId((member as { provider?: unknown }).provider); if (providerId) { providers.add(this.toWorkspaceTrustProvider(providerId)); } } return [...providers]; } private async resolveWorkspaceTrustGitRoot(cwd: string): Promise { const normalizedCwd = cwd.trim(); if (!normalizedCwd) { return null; } const gitRoot = await new Promise((resolve) => { execFile( 'git', ['-C', normalizedCwd, 'rev-parse', '--show-toplevel'], { encoding: 'utf8', maxBuffer: 16 * 1024, timeout: 1000, windowsHide: true, }, (error, stdout) => { if (error) { resolve(null); return; } const gitRoot = stdout.trim(); resolve(gitRoot && path.isAbsolute(gitRoot) ? gitRoot : null); } ); }); return gitRoot ?? resolveWorkspaceTrustFilesystemGitRoot(normalizedCwd); } private async collectWorkspaceTrustWorkspaces(input: { cwd: string; members: TeamCreateRequest['members']; }): Promise { const homeDir = getHomeDir(); const candidates: WorkspaceTrustWorkspace[] = []; const gitRootCache = new Map(); const addPath = async ( cwd: string, source: WorkspaceTrustWorkspace['source'], memberId?: string ): Promise => { const realCwd = await fs.promises.realpath(cwd).catch(() => null); let gitRoot = gitRootCache.get(cwd); if (gitRoot === undefined) { const resolvedGitRoot = await this.resolveWorkspaceTrustGitRoot(cwd); const realGitRoot = resolvedGitRoot ? await fs.promises.realpath(resolvedGitRoot).catch(() => resolvedGitRoot) : null; gitRoot = realGitRoot ? await resolveWorkspaceTrustCanonicalGitRoot(realGitRoot) : null; gitRootCache.set(cwd, gitRoot); } candidates.push( ...buildWorkspaceTrustPathCandidates({ cwd, realCwd, gitRoot, homeDir, source, memberId, platform: process.platform === 'win32' ? 'win32' : 'posix', }) ); }; await addPath(input.cwd, 'team-root'); for (const member of input.members) { const memberCwd = member.cwd?.trim(); if (!memberCwd) { continue; } await addPath( memberCwd, member.isolation === 'worktree' ? 'member-worktree' : 'member-cwd', member.name ); } const seen = new Set(); return candidates.filter((workspace) => { if (seen.has(workspace.comparisonKey)) { return false; } seen.add(workspace.comparisonKey); return true; }); } private applyWorkspaceTrustArgPatches(input: { args: string[]; patches: WorkspaceTrustLaunchArgPatch[]; targetProvider: TeamProviderId; targetSurface: WorkspaceTrustLaunchArgTargetSurface; }): string[] { if (input.patches.length === 0) { return input.args; } return applyWorkspaceTrustLaunchArgPatches({ args: input.args, patches: input.patches, targetProvider: this.toWorkspaceTrustProvider(input.targetProvider), targetSurface: input.targetSurface, }).args; } private createDefaultModelWorkspaceTrustProviderArgsResolver( plan: Pick ): WorkspaceTrustProviderArgsResolver { return (input) => this.applyWorkspaceTrustArgPatches({ args: input.providerArgs, patches: plan.launchArgPatches, targetProvider: input.providerId, targetSurface: 'default_model_probe', }); } private async planWorkspaceTrustArgsOnlySafely( request: WorkspaceTrustArgsOnlyPlanRequest ): Promise { if (!this.workspaceTrustCoordinator) { return { launchArgPatches: [] }; } try { return await this.workspaceTrustCoordinator.planArgsOnly(request); } catch (error) { logger.warn( `Workspace trust args-only planning failed; continuing without trust arg patches: ${ error instanceof Error ? error.message : String(error) }` ); return { launchArgPatches: [] }; } } private async planWorkspaceTrustFullSafely( request: WorkspaceTrustFullPlanRequest ): Promise { if (!this.workspaceTrustCoordinator) { return null; } try { return await this.workspaceTrustCoordinator.planFull(request); } catch (error) { logger.warn( `Workspace trust full planning failed; continuing without trust arg patches: ${ error instanceof Error ? error.message : String(error) }` ); return { workspaces: request.workspaces, launchArgPatches: [] }; } } private isLaunchRunStillCurrent(run: ProvisioningRun): boolean { return ( this.runs.get(run.runId) === run && this.provisioningRunByTeam.get(run.teamName) === run.runId && !run.cancelRequested && !run.processKilled ); } private async buildRuntimeTurnSettledHookSettingsArgs( providerId: TeamProviderId ): Promise { const settings = await this.buildRuntimeTurnSettledHookSettingsObject(providerId); return settings ? ['--settings', JSON.stringify(settings)] : []; } private async prepareWorkspaceTrustForDeterministicRun(input: { mode: 'create' | 'launch'; run: ProvisioningRun; claudePath: string; shellEnv: NodeJS.ProcessEnv; stopAllGenerationAtStart: number; workspaceTrustPlan: WorkspaceTrustFullPlanResult | null; featureFlags: WorkspaceTrustFeatureFlags; provisioningEnv: ProvisioningEnvResolution; }): Promise { if ( !this.workspaceTrustCoordinator || !input.workspaceTrustPlan || !input.featureFlags.enabled ) { return; } input.run.workspaceTrustPlan = input.workspaceTrustPlan; updateProgress(input.run, 'spawning', 'Preparing workspace trust', { warnings: input.run.progress.warnings, }); input.run.onProgress(input.run.progress); let execution: WorkspaceTrustExecutionResult; try { execution = await this.workspaceTrustCoordinator.execute({ claudePath: input.claudePath, workspaces: input.workspaceTrustPlan.workspaces, env: buildWorkspaceTrustPreflightEnv(input.shellEnv), featureFlags: input.featureFlags, isCancelled: () => input.run.cancelRequested || input.run.processKilled || this.stopAllTeamsGeneration !== input.stopAllGenerationAtStart, }); } catch (error) { const message = error instanceof Error ? error.message : String(error); execution = { id: 'workspace-trust-coordinator', provider: 'claude', status: 'soft_failed', workspaceIds: input.workspaceTrustPlan.workspaces.map((workspace) => workspace.id), errorCode: 'workspace_trust_preflight_error', errorMessage: message, evidence: [message], }; } input.run.workspaceTrustExecution = execution; input.run.workspaceTrustDiagnostics = budgetWorkspaceTrustDiagnosticsManifest({ attempt: 1, featureFlags: input.featureFlags, strategyResults: [execution], }); const workspaceTrustLaunchDiagnostic = buildWorkspaceTrustPreflightLaunchDiagnostic(execution); const workspaceTrustLaunchDiagnostics = workspaceTrustLaunchDiagnostic ? boundLaunchDiagnostics( mergeLaunchDiagnosticItem( input.run.progress.launchDiagnostics, workspaceTrustLaunchDiagnostic ) ) : input.run.progress.launchDiagnostics; if (!this.isLaunchRunStillCurrent(input.run)) { if (this.runs.get(input.run.runId) === input.run) { await this.cancelDeterministicRunBeforeSpawn(input.run, { mode: input.mode, provisioningEnv: input.provisioningEnv, }); } throw new Error('Team launch cancelled by app shutdown'); } if (execution.status === 'cancelled') { await this.cancelDeterministicRunBeforeSpawn(input.run, { mode: input.mode, provisioningEnv: input.provisioningEnv, }); } if (execution.status === 'blocked') { await this.failDeterministicRunBeforeSpawn(input.run, { mode: input.mode, message: 'Workspace trust required', error: execution.errorMessage || execution.errorCode || 'Workspace trust preflight blocked this launch.', launchDiagnostics: workspaceTrustLaunchDiagnostics, provisioningEnv: input.provisioningEnv, }); } if (execution.status === 'soft_failed') { const warning = execution.errorMessage || execution.errorCode || 'Workspace trust preflight could not verify trust before launch.'; input.run.progress = { ...input.run.progress, warnings: mergeProvisioningWarnings(input.run.progress.warnings, warning), launchDiagnostics: workspaceTrustLaunchDiagnostics, }; input.run.onProgress(input.run.progress); } else if (workspaceTrustLaunchDiagnostics) { input.run.progress = { ...input.run.progress, updatedAt: nowIso(), launchDiagnostics: workspaceTrustLaunchDiagnostics, }; input.run.onProgress(input.run.progress); } } private async failDeterministicRunBeforeSpawn( run: ProvisioningRun, input: { mode: 'create' | 'launch'; message: string; error: string; launchDiagnostics?: TeamLaunchDiagnosticItem[]; provisioningEnv: ProvisioningEnvResolution; } ): Promise { updateProgress(run, 'failed', input.message, { error: input.error, warnings: run.progress.warnings, launchDiagnostics: input.launchDiagnostics, }); run.onProgress(run.progress); if (input.provisioningEnv.anthropicApiKeyHelper) { await cleanupAnthropicTeamApiKeyHelperMaterial({ directory: input.provisioningEnv.anthropicApiKeyHelper.directory, }).catch(() => undefined); } if (input.mode === 'launch') { await this.restorePrelaunchConfig(run.teamName).catch(() => undefined); } this.cleanupRun(run); throw new Error(input.error); } private async cancelDeterministicRunBeforeSpawn( run: ProvisioningRun, input: { mode: 'create' | 'launch'; provisioningEnv: ProvisioningEnvResolution; } ): Promise { updateProgress(run, 'cancelled', 'Team launch cancelled', { warnings: run.progress.warnings, }); run.cancelRequested = true; run.onProgress(run.progress); if (input.provisioningEnv.anthropicApiKeyHelper) { await cleanupAnthropicTeamApiKeyHelperMaterial({ directory: input.provisioningEnv.anthropicApiKeyHelper.directory, }).catch(() => undefined); } if (input.mode === 'launch') { await this.restorePrelaunchConfig(run.teamName).catch(() => undefined); } this.cleanupRun(run); throw new Error('Team launch cancelled by app shutdown'); } private async buildRuntimeTurnSettledHookSettingsObject( providerId: TeamProviderId ): Promise { if (providerId !== 'anthropic' || !this.runtimeTurnSettledHookSettingsProvider) { return null; } try { const settings = await this.runtimeTurnSettledHookSettingsProvider({ provider: 'claude' }); return settings ?? null; } catch (error) { logger.warn( `Failed to build member work sync Stop hook settings: ${ error instanceof Error ? error.message : String(error) }` ); return null; } } private async buildTeamRuntimeLaunchArgsPlan(input: { teamName: string; providerId: TeamProviderId; launchIdentity?: ProviderModelLaunchIdentity | null; envResolution: ProvisioningEnvResolution; extraArgs?: string[]; inheritedProviderArgs?: string[]; includeAnthropicHelper: boolean; contextLabel: string; }): Promise { const resolvedProviderId = resolveTeamProviderId(input.providerId); const helper = input.includeAnthropicHelper && resolvedProviderId === 'anthropic' ? (input.envResolution.anthropicApiKeyHelper ?? null) : null; const rawProviderArgs = input.envResolution.providerArgs ?? []; const rawExtraArgs = input.extraArgs ?? []; const rawInheritedProviderArgs = input.inheritedProviderArgs ?? []; if (!helper && resolvedProviderId !== 'anthropic') { return { settingsArgs: [], fastModeArgs: buildProviderFastModeArgs(resolvedProviderId, input.launchIdentity), runtimeTurnSettledHookArgs: await this.buildRuntimeTurnSettledHookSettingsArgs(resolvedProviderId), providerArgs: rawProviderArgs, extraArgs: rawExtraArgs, inheritedProviderArgs: rawInheritedProviderArgs, appManagedSettingsPath: null, }; } const providerArgsWithoutHelper = filterOutSettingsPathArgs( rawProviderArgs, helper?.settingsPath ); const splitProviderArgs = splitSettingsJsonArgs(providerArgsWithoutHelper); const splitExtraArgs = splitSettingsJsonArgs(rawExtraArgs); const splitInheritedArgs = splitSettingsJsonArgs(rawInheritedProviderArgs); const shouldCoalesceInheritedSettings = splitInheritedArgs.settingsFragments.length > 0; if ( helper && (hasPathBasedSettingsArgs(splitProviderArgs.passthroughArgs) || hasPathBasedSettingsArgs(splitExtraArgs.passthroughArgs) || hasPathBasedSettingsArgs(splitInheritedArgs.passthroughArgs)) ) { throw new Error( `${input.contextLabel}: app-managed Anthropic API-key helper cannot be combined with path-based --settings. Use inline JSON settings or remove the custom --settings path.` ); } if ( shouldCoalesceInheritedSettings && !helper && (hasPathBasedSettingsArgs(splitProviderArgs.passthroughArgs) || hasPathBasedSettingsArgs(splitExtraArgs.passthroughArgs) || hasPathBasedSettingsArgs(splitInheritedArgs.passthroughArgs)) ) { throw new Error( `${input.contextLabel}: mixed-provider launch cannot combine app-managed inherited settings with path-based --settings. Use inline JSON settings or remove the custom --settings path.` ); } const settingsBundle = await materializeTeamRuntimeSettingsBundle({ teamName: input.teamName, providerId: resolvedProviderId, baseSettings: [ buildAnthropicSettingsObject(resolvedProviderId, input.launchIdentity), await this.buildRuntimeTurnSettledHookSettingsObject(resolvedProviderId), ...splitProviderArgs.settingsFragments, ...splitExtraArgs.settingsFragments, ...splitInheritedArgs.settingsFragments, ], anthropicHelper: helper, settingsDirectory: helper ? null : buildRuntimeSettingsTempDirectory(input.teamName), }); return { settingsArgs: settingsBundle?.args ?? [], fastModeArgs: [], runtimeTurnSettledHookArgs: [], providerArgs: splitProviderArgs.passthroughArgs, extraArgs: splitExtraArgs.passthroughArgs, inheritedProviderArgs: splitInheritedArgs.passthroughArgs, appManagedSettingsPath: settingsBundle?.settingsPath ?? null, }; } private async buildRuntimeTurnSettledEnvironment( providerId: TeamProviderId ): Promise> { if (providerId !== 'codex' || !this.runtimeTurnSettledEnvironmentProvider) { return {}; } try { return (await this.runtimeTurnSettledEnvironmentProvider({ provider: 'codex' })) ?? {}; } catch (error) { logger.warn( `Failed to build member work sync runtime turn-settled environment: ${ error instanceof Error ? error.message : String(error) }` ); return {}; } } private async buildRuntimeTurnSettledEnvironmentForMembers( primaryProviderId: TeamProviderId | undefined, memberSpecs: TeamCreateRequest['members'] ): Promise> { const resolvedPrimaryProviderId = resolveTeamProviderId(primaryProviderId); const needsCodexTurnSettledEnv = memberSpecs.some((member) => { const configuredProviderId = normalizeTeamMemberProviderId(member.providerId); const inferredProviderId = inferTeamProviderIdFromModel(member.model); return ( resolvedPrimaryProviderId === 'codex' || configuredProviderId === 'codex' || inferredProviderId === 'codex' ); }); if (!needsCodexTurnSettledEnv) { return {}; } return this.buildRuntimeTurnSettledEnvironment('codex'); } private async readRuntimeProviderLaunchFacts(params: { claudePath: string; cwd: string; providerId: TeamProviderId; env: NodeJS.ProcessEnv; providerArgs?: string[]; limitContext?: boolean; }): Promise { const providerArgs = params.providerArgs ?? []; const modelListPromise = execCli( params.claudePath, buildProviderCliCommandArgs(providerArgs, [ 'model', 'list', '--json', '--provider', params.providerId, ]), { cwd: params.cwd, env: params.env, timeout: PROVIDER_MODEL_LIST_TIMEOUT_MS, } ); const runtimeStatusPromise = params.providerId === 'codex' || params.providerId === 'anthropic' ? execCli( params.claudePath, buildProviderCliCommandArgs(providerArgs, [ 'runtime', 'status', '--json', '--provider', params.providerId, ]), { cwd: params.cwd, env: params.env, timeout: PROVIDER_RUNTIME_STATUS_TIMEOUT_MS, } ) : null; const [modelListResult, runtimeStatusResult] = await Promise.allSettled([ modelListPromise, runtimeStatusPromise, ]); let defaultModel: string | null = null; let modelIds = new Set(); let modelListParsed = false; if (modelListResult.status === 'fulfilled') { try { const parsed = extractJsonObjectFromCli( modelListResult.value.stdout ); modelListParsed = true; const provider = parsed.providers?.[params.providerId]; defaultModel = typeof provider?.defaultModel === 'string' && provider.defaultModel.trim().length > 0 ? provider.defaultModel.trim() : null; modelIds = normalizeProviderModelListModels(provider); } catch (error) { logger.warn( `[${params.providerId}] Failed to parse runtime model list for launch validation: ${ error instanceof Error ? error.message : String(error) }` ); } } let runtimeCapabilities: CliProviderRuntimeCapabilities | null = null; let modelCatalog: CliProviderModelCatalog | null = null; let providerStatus: RuntimeProviderLaunchFacts['providerStatus'] = null; if ( runtimeStatusResult.status === 'fulfilled' && runtimeStatusResult.value && typeof runtimeStatusResult.value.stdout === 'string' ) { try { const parsed = extractJsonObjectFromCli( runtimeStatusResult.value.stdout ); const parsedProviderStatus = parsed.providers?.[params.providerId] ?? null; providerStatus = parsedProviderStatus ? { ...parsedProviderStatus, providerId: parsedProviderStatus.providerId ?? params.providerId, } : null; runtimeCapabilities = providerStatus?.runtimeCapabilities ?? null; modelCatalog = providerStatus?.modelCatalog?.providerId === params.providerId ? providerStatus.modelCatalog : null; } catch (error) { logger.warn( `[${params.providerId}] Failed to parse runtime capabilities for launch validation: ${ error instanceof Error ? error.message : String(error) }` ); } } if (modelCatalog) { addModelCatalogLaunchModels(modelIds, modelCatalog); defaultModel = modelCatalog.defaultLaunchModel?.trim() || defaultModel; } if ( params.providerId === 'codex' && !isUsableCodexModelCatalog(modelCatalog) && runtimeCapabilities?.modelCatalog?.dynamic === true ) { const codexCatalog = await this.providerConnectionService.getCodexModelCatalog({ cwd: params.cwd, }); if (isUsableCodexModelCatalog(codexCatalog)) { addModelCatalogLaunchModels(modelIds, codexCatalog); modelCatalog = codexCatalog; defaultModel = codexCatalog.defaultLaunchModel?.trim() || defaultModel; } } return { defaultModel: params.providerId === 'anthropic' ? resolveAnthropicLaunchModel({ limitContext: params.limitContext === true, availableLaunchModels: modelCatalog?.models.map((model) => model.launchModel) ?? modelIds, defaultLaunchModel: defaultModel, }) : defaultModel, modelIds, modelListParsed, modelCatalog, runtimeCapabilities, providerStatus, }; } private buildProviderModelLaunchIdentity(params: { request: Pick< TeamCreateRequest, 'providerId' | 'providerBackendId' | 'model' | 'effort' | 'fastMode' | 'limitContext' >; facts: RuntimeProviderLaunchFacts; }): ProviderModelLaunchIdentity { const providerId = resolveTeamProviderId(params.request.providerId); const explicitModel = getExplicitLaunchModelSelection(params.request.model); const resolvedLaunchModel = resolveRequestedLaunchModel({ providerId, selectedModel: params.request.model, limitContext: params.request.limitContext, facts: params.facts, }); if (providerId === 'anthropic') { const selection = resolveAnthropicSelectionFromFacts({ selectedModel: params.request.model, limitContext: params.request.limitContext, facts: params.facts, }); const fastResolution = resolveAnthropicFastMode({ selection, selectedFastMode: params.request.fastMode, providerFastModeDefault: getAnthropicFastModeDefault(), }); return { providerId, providerBackendId: migrateProviderBackendId(providerId, params.request.providerBackendId) ?? null, selectedModel: explicitModel ?? null, selectedModelKind: explicitModel ? 'explicit' : 'default', resolvedLaunchModel: selection.resolvedLaunchModel ?? resolvedLaunchModel, catalogId: selection.catalogModel?.id?.trim() || selection.resolvedLaunchModel || resolvedLaunchModel, catalogSource: selection.catalogSource, catalogFetchedAt: selection.catalogFetchedAt, selectedEffort: params.request.effort ?? null, resolvedEffort: params.request.effort ?? selection.defaultEffort ?? null, selectedFastMode: params.request.fastMode ?? 'inherit', resolvedFastMode: fastResolution.resolvedFastMode, fastResolutionReason: fastResolution.disabledReason, }; } if (providerId === 'codex') { const selection = resolveCodexSelectionFromFacts({ selectedModel: params.request.model, providerBackendId: params.request.providerBackendId, facts: params.facts, }); const fastResolution = resolveCodexFastMode({ selection, selectedFastMode: params.request.fastMode, }); const resolvedCodexModel = selection.resolvedLaunchModel ?? resolvedLaunchModel; return { providerId, providerBackendId: migrateProviderBackendId(providerId, params.request.providerBackendId) ?? selection.providerBackendId, selectedModel: explicitModel ?? null, selectedModelKind: explicitModel ? 'explicit' : 'default', resolvedLaunchModel: resolvedCodexModel, catalogId: selection.catalogModel?.id?.trim() || selection.resolvedLaunchModel || resolvedCodexModel, catalogSource: selection.catalogSource, catalogFetchedAt: selection.catalogFetchedAt, selectedEffort: params.request.effort ?? null, resolvedEffort: params.request.effort ?? null, selectedFastMode: params.request.fastMode ?? 'inherit', resolvedFastMode: fastResolution.resolvedFastMode, fastResolutionReason: fastResolution.disabledReason, }; } const resolvedEffort = params.request.effort ?? null; return { providerId, providerBackendId: migrateProviderBackendId(providerId, params.request.providerBackendId) ?? null, selectedModel: explicitModel ?? null, selectedModelKind: explicitModel ? 'explicit' : 'default', resolvedLaunchModel, catalogId: resolvedLaunchModel, catalogSource: 'runtime', catalogFetchedAt: null, selectedEffort: params.request.effort ?? null, resolvedEffort, }; } private validateRuntimeLaunchSelection(params: { actorLabel: string; providerId: TeamProviderId; model?: string; effort?: EffortLevel; fastMode?: TeamFastMode; limitContext?: boolean; facts: RuntimeProviderLaunchFacts; }): void { const explicitModel = getExplicitLaunchModelSelection(params.model); if (params.providerId === 'anthropic') { const selection = resolveAnthropicSelectionFromFacts({ selectedModel: params.model, limitContext: params.limitContext, facts: params.facts, }); const resolvedLaunchModel = selection.resolvedLaunchModel?.trim() || null; if (!resolvedLaunchModel) { throw new Error( `${params.actorLabel} could not resolve the selected Anthropic model against the current runtime catalog.` ); } if (params.facts.modelIds.size > 0 && !params.facts.modelIds.has(resolvedLaunchModel)) { throw new Error( `${params.actorLabel} resolves to Anthropic model "${resolvedLaunchModel}", but the current runtime does not list it as launchable.` ); } if (params.effort) { const modelLabel = selection.displayName ?? resolvedLaunchModel; const effortSupport = resolveAnthropicEffortSupport({ selection, effort: params.effort, runtimeCapabilities: params.facts.runtimeCapabilities, }); if (effortSupport.kind !== 'supported') { throw new Error( `${params.actorLabel} uses Anthropic effort "${params.effort}", but ${formatAnthropicEffortSupportFailure( { effort: params.effort, modelLabel, kind: effortSupport.kind, supportedEfforts: effortSupport.kind === 'unverified-catalog-missing' ? undefined : effortSupport.supportedEfforts, } )}` ); } } const fastResolution = resolveAnthropicFastMode({ selection, selectedFastMode: params.fastMode, providerFastModeDefault: getAnthropicFastModeDefault(), }); if ((params.fastMode ?? 'inherit') === 'on' && !fastResolution.selectable) { throw new Error( `${params.actorLabel} enables Anthropic Fast mode, but ${ fastResolution.disabledReason ?? 'it is unavailable for the selected runtime or model.' }` ); } return; } if (params.providerId !== 'codex') { if (params.effort && !isLegacySafeEffort(params.effort)) { throw new Error( `${params.actorLabel} uses effort "${params.effort}", but ${getTeamProviderLabel( params.providerId )} currently supports only low, medium, or high effort in Agent Teams.` ); } return; } if ( params.effort && !isCodexEffortRuntimeSupported(params.effort, params.facts.runtimeCapabilities) ) { throw new Error( `${params.actorLabel} uses Codex effort "${params.effort}", but this Agent Teams runtime does not expose Codex reasoning config passthrough yet. Use low, medium, or high for now.` ); } const codexSelection = resolveCodexSelectionFromFacts({ selectedModel: params.model, facts: params.facts, }); const codexFastResolution = resolveCodexFastMode({ selection: codexSelection, selectedFastMode: params.fastMode, }); if ((params.fastMode ?? 'inherit') === 'on' && !codexFastResolution.selectable) { throw new Error( `${params.actorLabel} enables Codex Fast mode, but ${ codexFastResolution.disabledReason ?? 'it is unavailable for the selected runtime, model, or auth mode.' }` ); } if (!explicitModel || params.facts.modelIds.has(explicitModel)) { return; } if (params.facts.runtimeCapabilities?.modelCatalog?.dynamic === true) { return; } if (!hasAuthoritativeCodexLaunchCatalog(params.facts)) { return; } throw new Error( `${params.actorLabel} uses Codex model "${explicitModel}", but this Agent Teams runtime does not declare dynamic Codex model launch support yet. Upgrade the runtime or pick a listed Codex model.` ); } private async resolveAndValidateLaunchIdentity(params: { claudePath: string; cwd: string; env: NodeJS.ProcessEnv; request: Pick< TeamCreateRequest, 'providerId' | 'providerBackendId' | 'model' | 'effort' | 'fastMode' | 'limitContext' >; effectiveMembers: TeamCreateRequest['members']; providerArgsByProvider?: Map; }): Promise { const leadProviderId = resolveTeamProviderId(params.request.providerId); const factsByProvider = new Map(); const getFacts = async (providerId: TeamProviderId): Promise => { const cached = factsByProvider.get(providerId); if (cached) { return cached; } const facts = await this.readRuntimeProviderLaunchFacts({ claudePath: params.claudePath, cwd: params.cwd, providerId, env: params.env, providerArgs: params.providerArgsByProvider?.get(providerId), limitContext: params.request.limitContext, }); factsByProvider.set(providerId, facts); return facts; }; const leadFacts = await getFacts(leadProviderId); this.validateRuntimeLaunchSelection({ actorLabel: 'Team lead', providerId: leadProviderId, model: params.request.model, effort: params.request.effort, fastMode: params.request.fastMode, limitContext: params.request.limitContext, facts: leadFacts, }); for (const member of params.effectiveMembers) { const memberProviderId = resolveTeamProviderId(member.providerId); const memberFacts = await getFacts(memberProviderId); this.validateRuntimeLaunchSelection({ actorLabel: `Member ${member.name}`, providerId: memberProviderId, model: member.model, effort: member.effort, limitContext: params.request.limitContext, facts: memberFacts, }); } return this.buildProviderModelLaunchIdentity({ request: params.request, facts: leadFacts, }); } private async resolveDirectMemberLaunchIdentity(input: { claudePath: string; cwd: string; providerId: TeamProviderId; providerBackendId?: TeamProviderBackendId; provisioningEnv: ProvisioningEnvResolution; memberSpec: TeamCreateRequest['members'][number]; run: ProvisioningRun; }): Promise { const request: Pick< TeamCreateRequest, 'providerId' | 'providerBackendId' | 'model' | 'effort' | 'fastMode' | 'limitContext' > = { providerId: input.providerId, ...(input.providerBackendId ? { providerBackendId: input.providerBackendId } : {}), ...(input.memberSpec.model ? { model: input.memberSpec.model } : {}), ...(input.memberSpec.effort ? { effort: input.memberSpec.effort } : {}), ...(input.memberSpec.fastMode ? { fastMode: input.memberSpec.fastMode } : {}), ...(input.run.request.limitContext ? { limitContext: input.run.request.limitContext } : {}), }; const facts = await this.readRuntimeProviderLaunchFacts({ claudePath: input.claudePath, cwd: input.cwd, providerId: input.providerId, env: input.provisioningEnv.env, providerArgs: input.provisioningEnv.providerArgs, limitContext: input.run.request.limitContext, }); this.validateRuntimeLaunchSelection({ actorLabel: `Member ${input.memberSpec.name}`, providerId: input.providerId, model: input.memberSpec.model, effort: input.memberSpec.effort, fastMode: input.memberSpec.fastMode, limitContext: input.run.request.limitContext, facts, }); return this.buildProviderModelLaunchIdentity({ request, facts, }); } async getClaudeLogs( teamName: string, query?: { offset?: number; limit?: number } ): Promise<{ lines: string[]; total: number; hasMore: boolean; updatedAt?: string }> { const runId = this.getTrackedRunId(teamName); if (runId) { const run = this.runs.get(runId); if (run) { return sliceClaudeLogs(run.claudeLogLines, run.claudeLogsUpdatedAt, query); } } const retained = this.retainedClaudeLogsByTeam.get(teamName); if (!retained) { const transcriptSnapshot = await this.getPersistedTranscriptClaudeLogs(teamName); if (!transcriptSnapshot) { return { lines: [], total: 0, hasMore: false }; } return sliceClaudeLogs(transcriptSnapshot.lines, transcriptSnapshot.updatedAt, query); } return sliceClaudeLogs(retained.lines, retained.updatedAt, query); } private getProvisioningRunId(teamName: string): string | null { return this.provisioningRunByTeam.get(teamName) ?? null; } private getResolvableProvisioningRunId(teamName: string): string | null { const runId = this.getProvisioningRunId(teamName); if (!runId) { return null; } if (this.runs.has(runId) || this.runtimeAdapterProgressByRunId.has(runId)) { return runId; } if (this.provisioningRunByTeam.get(teamName) === runId) { this.provisioningRunByTeam.delete(teamName); } logger.debug(`[${teamName}] Cleared stale provisioning run id before launch: ${runId}`); return null; } private getAliveRunId(teamName: string): string | null { return this.aliveRunByTeam.get(teamName) ?? null; } private setAliveRunId(teamName: string, runId: string): void { if (!teamName || !runId || this.aliveRunByTeam.get(teamName) === runId) { return; } this.aliveRunByTeam.set(teamName, runId); notifyTeamWatchScopeChanged(); } private deleteAliveRunId(teamName: string): void { if (this.aliveRunByTeam.delete(teamName)) { notifyTeamWatchScopeChanged(); } } /** * Snapshot of teams that currently have a live runtime run. Used to keep the * file-watch scope covering running teams (read-only; the map is maintained as * runs start and stop). */ getAliveTeamNames(): string[] { return [...this.aliveRunByTeam.keys()]; } private getTrackedRunId(teamName: string): string | null { return this.getProvisioningRunId(teamName) ?? this.getAliveRunId(teamName); } private getAgentRuntimeSnapshotCacheTtlMs(teamName: string, runId: string | null): number { if (runId || this.runtimeAdapterRunByTeam.has(teamName)) { return TeamProvisioningService.AGENT_RUNTIME_SNAPSHOT_CACHE_TTL_MS; } return TeamProvisioningService.PERSISTED_AGENT_RUNTIME_SNAPSHOT_CACHE_TTL_MS; } private canDeliverToTrackedRuntimeRun(teamName: string, runId: string): boolean { const runtimeProgress = this.runtimeAdapterProgressByRunId.get(runId); if ( runtimeProgress && ['disconnected', 'failed', 'cancelled'].includes(runtimeProgress.state) ) { return false; } const run = this.runs.get(runId); if ( run && (run.processKilled || run.cancelRequested || ['disconnected', 'failed', 'cancelled'].includes(run.progress.state)) ) { return false; } return ( this.runtimeAdapterRunByTeam.get(teamName)?.runId === runId || this.provisioningRunByTeam.get(teamName) === runId || this.aliveRunByTeam.get(teamName) === runId ); } private resolveDeliverableTrackedRuntimeRunId(teamName: string): string | null { const candidates = Array.from( new Set( [ this.provisioningRunByTeam.get(teamName), this.aliveRunByTeam.get(teamName), this.runtimeAdapterRunByTeam.get(teamName)?.runId, ].filter((runId): runId is string => typeof runId === 'string' && runId.trim() !== '') ) ); for (const runId of candidates) { if (this.canDeliverToTrackedRuntimeRun(teamName, runId)) { return runId; } } return null; } private canDeliverToOpenCodeRuntimeForTeam(teamName: string): boolean { if (this.isTeamAlive(teamName)) { return true; } return this.hasAlivePersistedTeamProcess(teamName); } private canAttemptCommittedOpenCodeSessionRecovery(teamName: string): boolean { if (this.canDeliverToOpenCodeRuntimeForTeam(teamName)) { return true; } return !this.hasOnlyExplicitlyStoppedPersistedTeamProcesses(teamName); } private hasAlivePersistedTeamProcess(teamName: string): boolean { const parsed = this.readPersistedTeamProcessRows(teamName); if (!Array.isArray(parsed)) { return false; } return parsed.some((row) => { if (!row || typeof row !== 'object') { return false; } const processRow = row as { pid?: unknown; stoppedAt?: unknown }; return ( typeof processRow.pid === 'number' && Number.isFinite(processRow.pid) && processRow.stoppedAt == null && isProcessAlive(processRow.pid) ); }); } private hasOnlyExplicitlyStoppedPersistedTeamProcesses(teamName: string): boolean { const parsed = this.readPersistedTeamProcessRows(teamName); if (!Array.isArray(parsed) || parsed.length === 0) { return false; } return parsed.every((row) => { if (!row || typeof row !== 'object') { return false; } return (row as { stoppedAt?: unknown }).stoppedAt != null; }); } private readPersistedTeamProcessRows(teamName: string): unknown[] | null { const processesPath = path.join(getTeamsBasePath(), teamName, 'processes.json'); let parsed: unknown; try { parsed = JSON.parse(fs.readFileSync(processesPath, 'utf8')) as unknown; } catch { return null; } if (!Array.isArray(parsed)) { return null; } return parsed; } private cleanupStoppedTeamOpenCodeRuntimeLanesInBackground(teamName: string): void { void this.stopOpenCodeRuntimeLanesForStoppedTeam(teamName).catch((error) => { logger.warn( `[${teamName}] Failed to clean up stopped-team OpenCode runtime lanes: ${ error instanceof Error ? error.message : String(error) }` ); }); } private stopOpenCodeRuntimeLanesForStoppedTeam(teamName: string): Promise { const existing = this.stoppedTeamOpenCodeRuntimeCleanupInFlight.get(teamName); if (existing) { return existing; } const cleanup = this.stopOpenCodeRuntimeLanesForStoppedTeamInternal(teamName).finally(() => { if (this.stoppedTeamOpenCodeRuntimeCleanupInFlight.get(teamName) === cleanup) { this.stoppedTeamOpenCodeRuntimeCleanupInFlight.delete(teamName); } }); this.stoppedTeamOpenCodeRuntimeCleanupInFlight.set(teamName, cleanup); return cleanup; } private async stopOpenCodeRuntimeLanesForStoppedTeamInternal(teamName: string): Promise { if (this.canDeliverToOpenCodeRuntimeForTeam(teamName)) { return 0; } const laneIndex = await readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName).catch( () => null ); const activeLaneIds = Object.entries(laneIndex?.lanes ?? {}) .filter(([, entry]) => entry.state === 'active') .map(([laneId]) => laneId) .sort((left, right) => left.localeCompare(right)); if (activeLaneIds.length === 0) { return 0; } const adapter = this.getOpenCodeRuntimeAdapter(); const previousLaunchState = await this.launchStateStore.read(teamName).catch(() => null); const [config, metaMembers] = await Promise.all([ this.readConfigForObservation(teamName).catch(() => null), this.membersMetaStore.getMembers(teamName).catch(() => []), ]); const evidenceReader = new OpenCodeRuntimeManifestEvidenceReader({ teamsBasePath: getTeamsBasePath(), }); let stopped = 0; let cleaned = 0; for (const laneId of activeLaneIds) { const evidence = await evidenceReader.read(teamName, laneId).catch(() => null); const runId = evidence?.activeRunId?.trim() || null; if (adapter && runId) { try { await adapter.stop({ runId, laneId, teamName, cwd: this.resolveOpenCodeRuntimeLaneCleanupCwd(teamName, laneId, config, metaMembers), providerId: 'opencode', reason: 'cleanup', previousLaunchState, force: true, }); stopped += 1; } catch (error) { logger.warn( `[${teamName}] Failed to stop orphaned OpenCode lane ${laneId}: ${ error instanceof Error ? error.message : String(error) }` ); continue; } } else if (runId) { logger.warn( `[${teamName}] OpenCode lane ${laneId} belongs to stopped team, but runtime adapter is unavailable.` ); continue; } else if (!runId) { const pidStopResult = this.tryStopPersistedOpenCodeRuntimePidForStoppedLane({ teamName, laneId, previousLaunchState, }); if (pidStopResult === 'unsafe') { continue; } } await clearOpenCodeRuntimeLaneStorage({ teamsBasePath: getTeamsBasePath(), teamName, laneId, }).catch(() => undefined); cleaned += 1; this.deleteSecondaryRuntimeRun(teamName, laneId); if (laneId === 'primary') { this.runtimeAdapterRunByTeam.delete(teamName); this.deleteAliveRunId(teamName); this.provisioningRunByTeam.delete(teamName); this.invalidateRuntimeSnapshotCaches(teamName); } } if (cleaned > 0) { this.cleanedStoppedTeamOpenCodeRuntimeLanes.add(teamName); } return stopped; } private tryStopPersistedOpenCodeRuntimePidForStoppedLane(input: { teamName: string; laneId: string; previousLaunchState: PersistedTeamLaunchSnapshot | null; }): 'stopped' | 'no_pid' | 'unsafe' { const persistedMember = Object.values(input.previousLaunchState?.members ?? {}).find( (member) => member.providerId === 'opencode' && member.laneId === input.laneId ); if (!persistedMember) { return 'no_pid'; } const pid = persistedMember.runtimePid; if (typeof pid !== 'number' || !Number.isFinite(pid) || pid <= 0) { return 'no_pid'; } const command = this.readProcessCommandByPid(pid); if (!command) { return 'no_pid'; } const persistedProcessCommand = (persistedMember as { processCommand?: unknown }) .processCommand; const expectedCommand = typeof persistedProcessCommand === 'string' ? persistedProcessCommand.trim() : ''; if (expectedCommand && command !== expectedCommand) { logger.warn( `[${input.teamName}] Refusing to stop persisted OpenCode pid ${pid} for lane ${input.laneId}: process command changed.` ); return 'unsafe'; } if (!this.isOpenCodeServeCommand(command)) { logger.warn( `[${input.teamName}] Refusing to stop persisted OpenCode pid ${pid} for lane ${input.laneId}: process is not opencode serve.` ); return 'unsafe'; } try { killProcessByPid(pid); logger.info( `[${input.teamName}] Killed orphaned OpenCode runtime pid=${pid} for stopped lane ${input.laneId}` ); return 'stopped'; } catch (error) { logger.warn( `[${input.teamName}] Failed to kill orphaned OpenCode runtime pid=${pid} for stopped lane ${ input.laneId }: ${error instanceof Error ? error.message : String(error)}` ); return 'unsafe'; } } private readProcessCommandByPid(pid: number): string | null { if (process.platform === 'win32') { try { return ( listWindowsProcessTableSync() .find((row) => row.pid === pid) ?.command?.trim() || null ); } catch { return null; } } try { return execFileSync('ps', ['-p', String(pid), '-o', 'command='], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], }).trim(); } catch { return null; } } private isOpenCodeServeCommand(command: string): boolean { return /(^|[/\\\s])opencode(?:\.exe)?(\s|$)/i.test(command) && /\sserve(\s|$)/i.test(command); } private resolveOpenCodeRuntimeLaneCleanupCwd( teamName: string, laneId: string, config: TeamConfig | null, metaMembers: readonly TeamMember[] ): string | undefined { const projectPath = config?.projectPath?.trim() || this.readPersistedTeamProjectPath(teamName); const memberName = this.extractOpenCodeRuntimeLaneMemberName(laneId); if (!memberName) { return projectPath || undefined; } const normalized = memberName.toLowerCase(); const configMember = config?.members?.find( (member) => member.name?.trim().toLowerCase() === normalized ); const metaMember = metaMembers.find( (member) => member.name?.trim().toLowerCase() === normalized ); return metaMember?.cwd?.trim() || configMember?.cwd?.trim() || projectPath || undefined; } private extractOpenCodeRuntimeLaneMemberName(laneId: string): string | null { const match = /^secondary:opencode:(.+)$/i.exec(laneId.trim()); return match?.[1]?.trim() || null; } private getOpenCodeRuntimeAdapter(): TeamLaunchRuntimeAdapter | null { if (!this.runtimeAdapterRegistry?.has('opencode')) { return null; } return this.runtimeAdapterRegistry.get('opencode'); } private getOpenCodeRuntimeMessageAdapter(): OpenCodeRuntimeMessageAdapter | null { const adapter = this.getOpenCodeRuntimeAdapter(); if (!adapter || !('sendMessageToMember' in adapter)) { return null; } return adapter as OpenCodeRuntimeMessageAdapter; } private getOpenCodeRuntimePermissionListingAdapter(): OpenCodeRuntimePermissionListingAdapter | null { const adapter = this.getOpenCodeRuntimeAdapter(); if (!adapter || typeof adapter.listRuntimePermissions !== 'function') { return null; } return adapter as OpenCodeRuntimePermissionListingAdapter; } private resolveRuntimeRecipientProviderIdFromSources( memberName: string, config: TeamConfig | null | undefined, metaMembers: readonly TeamMember[] ): TeamProviderId | undefined { const normalizedMemberName = memberName.trim().toLowerCase(); if (!normalizedMemberName) { return undefined; } const configMember = config?.members?.find( (member) => member.name?.trim().toLowerCase() === normalizedMemberName ); const metaMember = metaMembers.find( (member) => member.name?.trim().toLowerCase() === normalizedMemberName ); const configProvider = (configMember as { provider?: unknown } | undefined)?.provider; const metaProvider = (metaMember as { provider?: unknown } | undefined)?.provider; return ( normalizeTeamProviderLike(metaMember?.providerId) ?? normalizeTeamProviderLike(metaProvider) ?? normalizeTeamProviderLike(configMember?.providerId) ?? normalizeTeamProviderLike(configProvider) ?? inferTeamProviderIdFromModel(metaMember?.model ?? configMember?.model) ); } private isOpenCodeRuntimeRecipientFromSources( memberName: string, config: TeamConfig | null | undefined, metaMembers: readonly TeamMember[] ): boolean { return ( this.resolveRuntimeRecipientProviderIdFromSources(memberName, config, metaMembers) === 'opencode' ); } async resolveRuntimeRecipientProviderId( teamName: string, memberName: string ): Promise { const normalizedMemberName = memberName.trim().toLowerCase(); if (!normalizedMemberName) { return undefined; } const [config, metaMembers] = await Promise.all([ this.readConfigSnapshot(teamName).catch(() => null), this.membersMetaStore.getMembers(teamName).catch(() => []), ]); return this.resolveRuntimeRecipientProviderIdFromSources( normalizedMemberName, config, metaMembers ); } async isOpenCodeRuntimeRecipient(teamName: string, memberName: string): Promise { return (await this.resolveRuntimeRecipientProviderId(teamName, memberName)) === 'opencode'; } private async isOpenCodeDeliveryResponseReadCommitAllowed(input: { teamName?: string; memberName?: string; responseState?: NonNullable['state']; actionMode?: AgentActionMode; taskRefs?: TaskRef[]; visibleReply?: OpenCodeVisibleReplyProof | null; ledgerRecord?: OpenCodePromptDeliveryLedgerRecord | null; }): Promise { const state = input.responseState; if (!state || !isOpenCodePromptResponseStateResponded(state)) { return false; } if (input.ledgerRecord?.messageKind === 'member_work_sync_nudge') { return this.isOpenCodeMemberWorkSyncReadCommitAllowed({ teamName: input.teamName, memberName: input.memberName, ledgerRecord: input.ledgerRecord, }); } if (state === 'responded_plain_text') { return this.isOpenCodePlainTextResponseReadCommitAllowed({ actionMode: input.actionMode, taskRefs: input.taskRefs, visibleReply: input.visibleReply, ledgerRecord: input.ledgerRecord, }); } if (state === 'responded_visible_message') { return ( isOpenCodeVisibleReplyReadCommitAllowed({ actionMode: input.actionMode, taskRefs: input.taskRefs, visibleReply: input.visibleReply, transcriptOnlyVisibleReply: !input.visibleReply, }) && this.openCodeTaskRefsIncludeAll(input.visibleReply?.message.taskRefs, input.taskRefs) ); } const hasTaskRefs = (input.taskRefs ?? []).length > 0; if (!hasTaskRefs && input.actionMode !== 'do' && input.actionMode !== 'delegate') { return false; } return this.hasOpenCodeNonVisibleProgressProof(input.ledgerRecord); } private hasOpenCodeNonVisibleProgressProof( ledgerRecord?: OpenCodePromptDeliveryLedgerRecord | null ): boolean { if (ledgerRecord?.messageKind === 'member_work_sync_nudge') { return this.hasOpenCodeMemberWorkSyncReadCommitProof(ledgerRecord); } const toolNames = ledgerRecord?.observedToolCallNames ?? []; return toolNames.some((toolName) => { const normalized = this.normalizeOpenCodeObservedToolName(toolName); return ( normalized === 'task_start' || normalized === 'task_add_comment' || normalized === 'task_complete' || normalized === 'task_set_status' || normalized === 'task_set_clarification' || normalized === 'task_create' || normalized === 'task_link' || normalized === 'runtime_task_event' || normalized === 'write' || normalized === 'edit' || normalized === 'patch' ); }); } private hasOpenCodeMemberWorkSyncReportToolProof( ledgerRecord?: OpenCodePromptDeliveryLedgerRecord | null ): boolean { const toolNames = ledgerRecord?.observedToolCallNames ?? []; return toolNames.some((toolName) => { const normalized = this.normalizeOpenCodeObservedToolName(toolName); return normalized === 'member_work_sync_report'; }); } private hasOpenCodeReviewPickupWorkflowProof( ledgerRecord?: OpenCodePromptDeliveryLedgerRecord | null ): boolean { if (ledgerRecord?.workSyncIntent !== 'review_pickup') { return false; } const toolNames = ledgerRecord?.observedToolCallNames ?? []; return toolNames.some((toolName) => { const normalized = this.normalizeOpenCodeObservedToolName(toolName); return ( normalized === 'review_start' || normalized === 'review_approve' || normalized === 'review_request_changes' ); }); } private hasOpenCodeMemberWorkSyncReadCommitProof( ledgerRecord?: OpenCodePromptDeliveryLedgerRecord | null ): boolean { return ( this.hasOpenCodeMemberWorkSyncReportToolProof(ledgerRecord) || this.hasOpenCodeReviewPickupWorkflowProof(ledgerRecord) ); } private async isOpenCodeMemberWorkSyncReadCommitAllowed(input: { teamName?: string; memberName?: string; ledgerRecord?: OpenCodePromptDeliveryLedgerRecord | null; }): Promise { if (this.hasOpenCodeReviewPickupWorkflowProof(input.ledgerRecord)) { return true; } if (!this.hasOpenCodeMemberWorkSyncReportToolProof(input.ledgerRecord)) { return false; } const teamName = input.teamName?.trim(); const memberName = input.memberName?.trim(); if (!teamName || !memberName) { return false; } return this.hasAcceptedMemberWorkSyncReport({ teamName, memberName }); } private async isLegacyOpenCodeMemberWorkSyncReadCommitAllowed(input: { teamName: string; memberName: string; workSyncIntent?: OpenCodeTeamRuntimeMessageInput['workSyncIntent']; responseObservation?: NonNullable; }): Promise { const state = input.responseObservation?.state; if (!state || !isOpenCodePromptResponseStateResponded(state)) { return false; } const toolNames = input.responseObservation?.toolCallNames ?? []; const hasReviewPickupProof = input.workSyncIntent === 'review_pickup' && toolNames.some((toolName) => { const normalized = this.normalizeOpenCodeObservedToolName(toolName); return ( normalized === 'review_start' || normalized === 'review_approve' || normalized === 'review_request_changes' ); }); if (hasReviewPickupProof) { return true; } const hasReportTool = toolNames.some( (toolName) => this.normalizeOpenCodeObservedToolName(toolName) === 'member_work_sync_report' ); if (!hasReportTool) { return false; } return this.hasAcceptedMemberWorkSyncReport({ teamName: input.teamName, memberName: input.memberName, }); } private hasOpenCodeObservedMessageSendToolCall( ledgerRecord?: OpenCodePromptDeliveryLedgerRecord | null ): boolean { return (ledgerRecord?.observedToolCallNames ?? []).some( (toolName) => this.normalizeOpenCodeObservedToolName(toolName) === 'message_send' ); } private normalizeOpenCodeObservedToolName(toolName: string): string { return toolName .trim() .toLowerCase() .replace(/^mcp__agent[-_]teams__/, '') .replace(/^agent[-_]teams_/, '') .replace(/^mcp__agent_teams__/, '') .replace(/^agent_teams_/, ''); } private isOpenCodePlainTextResponseReadCommitAllowed(input: { actionMode?: AgentActionMode; taskRefs?: TaskRef[]; visibleReply?: OpenCodeVisibleReplyProof | null; ledgerRecord?: OpenCodePromptDeliveryLedgerRecord | null; }): boolean { if (this.isOpenCodeDirectUserPromptDelivery(input.ledgerRecord)) { return ( Boolean( input.ledgerRecord?.visibleReplyInbox?.trim() && input.ledgerRecord?.visibleReplyMessageId?.trim() ) && this.openCodeTaskRefsIncludeAll(input.visibleReply?.message.taskRefs, input.taskRefs) ); } const preview = input.ledgerRecord?.observedAssistantPreview?.trim(); if (!preview) { return true; } return isOpenCodeVisibleReplySemanticallySufficient({ actionMode: input.actionMode, taskRefs: input.taskRefs, text: preview, }).sufficient; } private getOpenCodeDeliveryPendingReason(input: { responseState?: NonNullable['state']; actionMode?: AgentActionMode | null; taskRefs?: TaskRef[]; visibleReply?: OpenCodeVisibleReplyProof | null; ledgerRecord?: OpenCodePromptDeliveryLedgerRecord | null; }): string { const record = input.ledgerRecord; const state = input.responseState ?? record?.responseState; if (record?.messageKind === 'member_work_sync_nudge') { if (state === 'responded_plain_text' || state === 'responded_visible_message') { return 'member_work_sync_report_required'; } if (state === 'responded_non_visible_tool' || state === 'responded_tool_call') { if (record.workSyncIntent !== 'review_pickup') { return 'member_work_sync_report_required'; } if (!this.hasOpenCodeMemberWorkSyncReadCommitProof(record)) { return 'member_work_sync_report_required'; } } if (!this.hasOpenCodeMemberWorkSyncReadCommitProof(record)) { return 'member_work_sync_report_required'; } } if (state === 'responded_visible_message' && !input.visibleReply) { return 'visible_reply_destination_not_found_yet'; } if ( state === 'responded_visible_message' && !this.openCodeTaskRefsIncludeAll(input.visibleReply?.message.taskRefs, input.taskRefs) ) { return 'visible_reply_missing_task_refs'; } if (state === 'responded_plain_text') { const preview = record?.observedAssistantPreview?.trim(); if ( this.isOpenCodeDirectUserPromptDelivery(record) && input.visibleReply && !this.openCodeTaskRefsIncludeAll(input.visibleReply.message.taskRefs, input.taskRefs) ) { return 'visible_reply_missing_task_refs'; } if (record?.lastReason === 'visible_reply_ack_only_still_requires_answer') { return 'visible_reply_ack_only_still_requires_answer'; } if ( preview && !isOpenCodeVisibleReplySemanticallySufficient({ actionMode: input.actionMode, taskRefs: input.taskRefs, text: preview, }).sufficient ) { return 'plain_text_ack_only_still_requires_answer'; } if ( this.isOpenCodeDirectUserPromptDelivery(record) && !record?.visibleReplyMessageId && !record?.inboxReadCommittedAt ) { return 'plain_text_visible_reply_not_materialized_yet'; } } if (record?.lastReason === 'visible_reply_ack_only_still_requires_answer') { return 'visible_reply_ack_only_still_requires_answer'; } if (state === 'responded_non_visible_tool' || state === 'responded_tool_call') { const hasTaskRefs = (input.taskRefs ?? []).length > 0; if (!hasTaskRefs && input.actionMode !== 'do' && input.actionMode !== 'delegate') { return 'visible_reply_still_required'; } if (!this.hasOpenCodeNonVisibleProgressProof(record)) { return 'non_visible_tool_without_task_progress'; } } if (state === 'empty_assistant_turn') { return 'empty_assistant_turn'; } if (state === 'prompt_delivered_no_assistant_message') { return 'prompt_delivered_no_assistant_message'; } if (state === 'tool_error') { return 'tool_error_without_required_delivery_proof'; } return record?.lastReason ?? 'opencode_delivery_response_pending'; } private normalizeOpenCodeDeliveryResponseObservation( observation?: NonNullable ): NonNullable | undefined { if ( observation?.state !== 'empty_assistant_turn' || !observation.deliveredUserMessageId || observation.assistantMessageId || observation.latestAssistantPreview?.trim() || observation.toolCallNames.length > 0 || observation.visibleMessageToolCallId || observation.visibleReplyMessageId ) { return observation; } return { ...observation, state: 'prompt_delivered_no_assistant_message', reason: 'prompt_delivered_no_assistant_message', }; } private isOpenCodePromptAcceptedByObservation( observation?: NonNullable ): boolean { const deliveredUserMessageId = observation?.deliveredUserMessageId; return typeof deliveredUserMessageId === 'string' && deliveredUserMessageId.trim().length > 0; } private hasOpenCodeAcceptedRuntimePrompt( ledgerRecord?: OpenCodePromptDeliveryLedgerRecord | null ): boolean { return Boolean( ledgerRecord?.acceptedAt || ledgerRecord?.runtimePromptMessageId?.trim() || ledgerRecord?.lastRuntimePromptMessageId?.trim() || ledgerRecord?.deliveredUserMessageId?.trim() || (ledgerRecord?.runtimePromptMessageIds ?? []).some((messageId) => messageId.trim()) ); } private isOpenCodeDeliveryRetryablePendingResponse(input: { ledgerRecord: OpenCodePromptDeliveryLedgerRecord; visibleReply?: OpenCodeVisibleReplyProof | null; readAllowed: boolean; }): boolean { if (input.readAllowed) { return false; } if ( input.ledgerRecord.responseState === 'session_stale' && this.hasOpenCodeAcceptedRuntimePrompt(input.ledgerRecord) ) { return false; } if (isOpenCodePromptDeliveryRetryableResponseState(input.ledgerRecord.responseState)) { return true; } if ( input.ledgerRecord.lastReason === 'visible_reply_ack_only_still_requires_answer' || input.ledgerRecord.lastReason === 'plain_text_ack_only_still_requires_answer' || input.ledgerRecord.lastReason === 'visible_reply_missing_task_refs' || input.ledgerRecord.lastReason === 'member_work_sync_report_required' ) { return true; } if ( input.ledgerRecord.messageKind === 'member_work_sync_nudge' && (input.ledgerRecord.responseState === 'responded_visible_message' || input.ledgerRecord.responseState === 'responded_plain_text' || input.ledgerRecord.responseState === 'responded_non_visible_tool' || input.ledgerRecord.responseState === 'responded_tool_call') ) { return true; } if (input.ledgerRecord.responseState === 'responded_visible_message' && !input.visibleReply) { return true; } if ( input.ledgerRecord.responseState === 'responded_non_visible_tool' || input.ledgerRecord.responseState === 'responded_tool_call' || input.ledgerRecord.responseState === 'responded_plain_text' ) { return true; } return false; } private getOpenCodeDeliveryHardFailureKind( record?: OpenCodePromptDeliveryLedgerRecord | null ): OpenCodePromptDeliveryHardFailureKind { if (!record) { return 'none'; } if (record.status === 'failed_terminal') { return 'unknown'; } if (record.responseState === 'permission_blocked') { return 'permission'; } if (record.responseState === 'session_error') { return 'session'; } return 'none'; } private buildOpenCodePromptDeliveryRepairControlText(input: { ledgerRecord?: OpenCodePromptDeliveryLedgerRecord | null; readAllowed: boolean; pendingReason: string; controlUrl?: string | null; }): string | null { const record = input.ledgerRecord; if (!record) { return null; } return decideOpenCodePromptDeliveryRepair({ teamName: record.teamName, memberName: record.memberName, inboxMessageId: record.inboxMessageId, replyRecipient: record.replyRecipient, messageKind: record.messageKind, workSyncIntent: record.workSyncIntent, actionMode: record.actionMode, taskRefs: record.taskRefs, status: record.status, responseState: record.responseState, attempts: record.attempts, maxAttempts: record.maxAttempts, pendingReason: input.pendingReason, readAllowed: input.readAllowed, inboxReadCommitted: Boolean(record.inboxReadCommittedAt), visibleReplyFound: Boolean(record.visibleReplyMessageId), hasKnownProgressProof: this.hasOpenCodeNonVisibleProgressProof(record), toolCallNames: record.observedToolCallNames, acceptanceUnknown: record.acceptanceUnknown, hardFailureKind: this.getOpenCodeDeliveryHardFailureKind(record), controlUrl: input.controlUrl, }).controlText; } private buildOpenCodePromptDeliveryAttemptText(input: { text: string; controlText?: string | null; }): string { const controlText = input.controlText?.trim(); return controlText ? `${controlText}\n\n${input.text}` : input.text; } private isOpenCodePromptAcceptanceUnknownFailure(diagnostics: readonly string[]): boolean { return diagnostics.some((diagnostic) => isProbeTimeoutMessage(diagnostic)); } private isOpenCodeRuntimeManifestWatermarkDeliveryFailure( record: OpenCodePromptDeliveryLedgerRecord ): boolean { return [record.lastReason, ...record.diagnostics].some( (reason) => typeof reason === 'string' && reason.toLowerCase().includes('runtime manifest high watermark is stale') ); } private isOpenCodeNoAssistantDeliveryFailure( record: OpenCodePromptDeliveryLedgerRecord ): boolean { if (record.inboxReadCommittedAt) { return false; } const noAssistantState = record.responseState === 'empty_assistant_turn' || record.responseState === 'prompt_delivered_no_assistant_message'; const reasonText = [record.responseState, record.lastReason, ...record.diagnostics] .filter((reason): reason is string => typeof reason === 'string' && reason.trim().length > 0) .join('\n') .toLowerCase(); if ( !noAssistantState && !reasonText.includes('empty_assistant_turn') && !reasonText.includes('prompt_delivered_no_assistant_message') && !reasonText.includes('accepted the prompt, but no assistant turn was recorded') ) { return false; } return ![record.lastReason, ...record.diagnostics].some((reason) => { const reasonCode = classifyOpenCodeRuntimeDeliveryReasonCode(reason ?? undefined); return ( reasonCode === 'quota_exhausted' || reasonCode === 'auth_error' || reasonCode === 'filesystem_error' ); }); } private isOpenCodeNoAssistantTerminalDeliveryFailure( record: OpenCodePromptDeliveryLedgerRecord ): boolean { return ( record.status === 'failed_terminal' && record.attempts <= record.maxAttempts && this.isOpenCodeNoAssistantDeliveryFailure(record) ); } private async requeueOpenCodeNoAssistantTerminalDeliveryIfNeeded(input: { ledger: OpenCodePromptDeliveryLedgerStore; ledgerRecord: OpenCodePromptDeliveryLedgerRecord; }): Promise { if (!this.isOpenCodeNoAssistantTerminalDeliveryFailure(input.ledgerRecord)) { return input.ledgerRecord; } const scheduledAt = nowIso(); const requeued = await input.ledger.markNextAttemptScheduled({ id: input.ledgerRecord.id, status: 'retry_scheduled', nextAttemptAt: scheduledAt, reason: 'opencode_prompt_delivery_requeued_after_terminal_no_assistant_response', scheduledAt, }); logger.info( 'opencode_prompt_delivery_requeued_after_terminal_no_assistant_response', JSON.stringify({ teamName: requeued.teamName, memberName: requeued.memberName, laneId: requeued.laneId, runId: requeued.runId, inboxMessageId: requeued.inboxMessageId, attempts: requeued.attempts, maxAttempts: requeued.maxAttempts, }) ); return requeued; } private async requeueOpenCodeRuntimeManifestWatermarkDeliveryIfNeeded(input: { ledger: OpenCodePromptDeliveryLedgerStore; ledgerRecord: OpenCodePromptDeliveryLedgerRecord; }): Promise { if ( input.ledgerRecord.status !== 'failed_terminal' || input.ledgerRecord.inboxReadCommittedAt || !this.isOpenCodeRuntimeManifestWatermarkDeliveryFailure(input.ledgerRecord) ) { return input.ledgerRecord; } const scheduledAt = nowIso(); const requeued = await input.ledger.markNextAttemptScheduled({ id: input.ledgerRecord.id, status: 'retry_scheduled', nextAttemptAt: scheduledAt, reason: 'opencode_prompt_delivery_requeued_after_runtime_manifest_high_watermark_fix', scheduledAt, }); logger.info( 'opencode_prompt_delivery_requeued_after_runtime_manifest_high_watermark_fix', JSON.stringify({ teamName: requeued.teamName, memberName: requeued.memberName, laneId: requeued.laneId, runId: requeued.runId, inboxMessageId: requeued.inboxMessageId, attempts: requeued.attempts, }) ); return requeued; } private isOpenCodeDirectUserPromptDelivery( ledgerRecord?: OpenCodePromptDeliveryLedgerRecord | null ): boolean { return ledgerRecord?.replyRecipient?.trim().toLowerCase() === 'user'; } private canMaterializeOpenCodePlainTextReply( ledgerRecord: OpenCodePromptDeliveryLedgerRecord ): boolean { if (ledgerRecord.responseState === 'responded_plain_text') { return true; } return ( ledgerRecord.responseState === 'tool_error' && this.hasOpenCodeObservedMessageSendToolCall(ledgerRecord) ); } private isOpenCodePromptDeliveryWatchdogEnabled(): boolean { const enabled = process.env.CLAUDE_TEAM_OPENCODE_PROMPT_DELIVERY_WATCHDOG !== '0'; if (!enabled && !this.openCodePromptDeliveryWatchdogDisabledLogged) { this.openCodePromptDeliveryWatchdogDisabledLogged = true; logger.info( 'OpenCode prompt delivery watchdog is disabled by CLAUDE_TEAM_OPENCODE_PROMPT_DELIVERY_WATCHDOG=0; using legacy prompt acceptance semantics.' ); } return enabled; } private async markOpenCodePromptLedgerFailedTerminal(input: { ledger: OpenCodePromptDeliveryLedgerStore; id: string; reason: string; diagnostics?: string[]; failedAt: string; eventContext?: Record; }): Promise { const failed = await input.ledger.markFailedTerminal({ id: input.id, reason: input.reason, ...(input.diagnostics ? { diagnostics: input.diagnostics } : {}), failedAt: input.failedAt, }); this.logOpenCodePromptDeliveryEvent('opencode_prompt_delivery_terminal_failure', failed, { reason: input.reason, ...(input.eventContext ?? {}), }); return failed; } private async findOpenCodeVisibleReplyByRelayOfMessageId(input: { teamName: string; replyRecipient?: string | null; from: string; relayOfMessageId: string; expectedMessageId?: string | null; allowUserFallbackForLeadRecipient?: boolean; }): Promise { const relayOfMessageId = input.relayOfMessageId.trim(); if (!relayOfMessageId) { return null; } const expectedMessageId = input.expectedMessageId?.trim() || null; const candidates = await this.getOpenCodeVisibleReplyInboxCandidates({ teamName: input.teamName, replyRecipient: input.replyRecipient, includeUserFallbackForLeadRecipient: Boolean( expectedMessageId || input.allowUserFallbackForLeadRecipient ), }); const explicitRecipient = input.replyRecipient?.trim() || 'user'; const expectedFrom = input.from.trim().toLowerCase(); for (const inboxName of candidates) { const messages = await this.inboxReader .getMessagesFor(input.teamName, inboxName) .catch(() => []); const isUserFallbackForNonUserRecipient = inboxName.trim().toLowerCase() === 'user' && explicitRecipient.trim().toLowerCase() !== 'user'; const matches = messages.filter( (message): message is InboxMessage & { messageId: string } => { const messageId = typeof message.messageId === 'string' ? message.messageId.trim() : ''; const messageRelayOf = typeof message.relayOfMessageId === 'string' ? message.relayOfMessageId.trim() : ''; return ( messageId.length > 0 && (!expectedMessageId || messageId === expectedMessageId) && messageRelayOf === relayOfMessageId && message.from.trim().toLowerCase() === expectedFrom ); } ); const runtimeDeliveryMatches = matches.filter( (message) => message.source === 'runtime_delivery' ); const match = isUserFallbackForNonUserRecipient && !expectedMessageId ? runtimeDeliveryMatches.length === 1 ? runtimeDeliveryMatches[0] : matches.length === 1 ? matches[0] : null : (runtimeDeliveryMatches[0] ?? matches[0] ?? null); if (match) { const matchMessageId = typeof match.messageId === 'string' ? match.messageId.trim() : ''; if (!matchMessageId) { continue; } return { inboxName, message: { ...match, messageId: matchMessageId }, missingRuntimeDeliverySource: match.source !== 'runtime_delivery', }; } } return null; } private isOpenCodeVisibleReplyTimestampEligible(input: { message: InboxMessage; ledgerRecord: OpenCodePromptDeliveryLedgerRecord; }): boolean { return isOpenCodeVisibleReplyTimestampEligibleValue(input); } private isOpenCodeRecoveredVisibleReplyCandidate(input: { message: InboxMessage & { messageId: string }; ledgerRecord: OpenCodePromptDeliveryLedgerRecord; from: string; requireTaskRefs: boolean; }): boolean { return isOpenCodeRecoveredVisibleReplyCandidateValue(input); } private async correlateOpenCodeRecoveredVisibleReply(input: { teamName: string; inboxName: string; memberName: string; ledgerRecord: OpenCodePromptDeliveryLedgerRecord; visibleReply: OpenCodeVisibleReplyProof; diagnostic: string; }): Promise { const expectedRelayOfMessageId = input.ledgerRecord.inboxMessageId.trim(); const currentRelayOfMessageId = typeof input.visibleReply.message.relayOfMessageId === 'string' ? input.visibleReply.message.relayOfMessageId.trim() : ''; if (currentRelayOfMessageId === expectedRelayOfMessageId) { return { visibleReply: input.visibleReply, visibleReplyCorrelation: 'relayOfMessageId', diagnostics: [input.diagnostic], }; } try { const correlated = await this.inboxWriter.correlateRuntimeDeliveryReply(input.teamName, { inboxName: input.inboxName, messageId: input.visibleReply.message.messageId, relayOfMessageId: expectedRelayOfMessageId, from: input.memberName, taskRefs: input.ledgerRecord.taskRefs, }); if (correlated.message) { const visibleReply = { ...input.visibleReply, message: correlated.message, }; if (correlated.updated) { this.emitRuntimeDeliveryReplyAdvisoryRefresh(input.teamName, visibleReply.message); } return { visibleReply, visibleReplyCorrelation: 'relayOfMessageId', diagnostics: [ input.diagnostic, correlated.updated ? 'opencode_visible_reply_relayOfMessageId_repaired' : 'opencode_visible_reply_relayOfMessageId_already_correlated', ], }; } return { visibleReply: input.visibleReply, visibleReplyCorrelation: 'direct_child_message_send', diagnostics: [input.diagnostic, 'opencode_visible_reply_relayOfMessageId_repair_not_found'], }; } catch (error) { logger.warn( `[${input.teamName}] Failed to repair OpenCode visible reply relayOfMessageId for ${input.memberName}/${expectedRelayOfMessageId}: ${getErrorMessage(error)}` ); return { visibleReply: input.visibleReply, visibleReplyCorrelation: 'direct_child_message_send', diagnostics: [input.diagnostic, 'opencode_visible_reply_relayOfMessageId_repair_failed'], }; } } private async findOpenCodeVisibleReplyByObservedMessageId(input: { teamName: string; replyRecipient?: string | null; from: string; ledgerRecord: OpenCodePromptDeliveryLedgerRecord; }): Promise { const expectedMessageId = input.ledgerRecord.visibleReplyMessageId?.trim(); if (!expectedMessageId) { return null; } const candidates = await this.getOpenCodeVisibleReplyInboxCandidates({ teamName: input.teamName, replyRecipient: input.replyRecipient, includeUserFallbackForLeadRecipient: true, }); for (const inboxName of candidates) { const messages = await this.inboxReader .getMessagesFor(input.teamName, inboxName) .catch(() => []); const match = messages.find((message): message is InboxMessage & { messageId: string } => { const messageId = typeof message.messageId === 'string' ? message.messageId.trim() : ''; return ( messageId === expectedMessageId && this.isOpenCodeRecoveredVisibleReplyCandidate({ message: { ...message, messageId }, ledgerRecord: input.ledgerRecord, from: input.from, requireTaskRefs: false, }) ); }); if (!match) { continue; } return await this.correlateOpenCodeRecoveredVisibleReply({ teamName: input.teamName, inboxName, memberName: input.from, ledgerRecord: input.ledgerRecord, visibleReply: { inboxName, message: { ...match, messageId: expectedMessageId }, missingRuntimeDeliverySource: match.source !== 'runtime_delivery', }, diagnostic: 'opencode_visible_reply_recovered_by_observed_message_id', }); } return null; } private async findOpenCodeVisibleReplyByTaskRefs(input: { teamName: string; replyRecipient?: string | null; from: string; ledgerRecord: OpenCodePromptDeliveryLedgerRecord; }): Promise { if (this.normalizeOpenCodeTaskRefsForComparison(input.ledgerRecord.taskRefs).length === 0) { return null; } const candidates = await this.getOpenCodeVisibleReplyInboxCandidates({ teamName: input.teamName, replyRecipient: input.replyRecipient, includeUserFallbackForLeadRecipient: true, }); const matches: OpenCodeVisibleReplyProof[] = []; for (const inboxName of candidates) { const messages = await this.inboxReader .getMessagesFor(input.teamName, inboxName) .catch(() => []); for (const message of messages) { const messageId = typeof message.messageId === 'string' ? message.messageId.trim() : ''; if (!messageId) { continue; } const candidate = { ...message, messageId }; if ( this.isOpenCodeRecoveredVisibleReplyCandidate({ message: candidate, ledgerRecord: input.ledgerRecord, from: input.from, requireTaskRefs: true, }) ) { matches.push({ inboxName, message: candidate, missingRuntimeDeliverySource: candidate.source !== 'runtime_delivery', }); } } } const match = matches.sort((left, right) => { const leftMs = Date.parse(left.message.timestamp); const rightMs = Date.parse(right.message.timestamp); const leftValid = Number.isFinite(leftMs); const rightValid = Number.isFinite(rightMs); if (leftValid && rightValid && leftMs !== rightMs) { return leftMs - rightMs; } if (leftValid !== rightValid) { return leftValid ? -1 : 1; } return left.message.messageId.localeCompare(right.message.messageId); })[0]; if (!match) { return null; } return await this.correlateOpenCodeRecoveredVisibleReply({ teamName: input.teamName, inboxName: match.inboxName, memberName: input.from, ledgerRecord: input.ledgerRecord, visibleReply: match, diagnostic: 'opencode_visible_reply_recovered_by_task_refs', }); } private async getOpenCodeVisibleReplyInboxCandidates(input: { teamName: string; replyRecipient?: string | null; includeUserFallbackForLeadRecipient?: boolean; }): Promise { const configuredLeadName = await this.readConfigForObservation(input.teamName) .then( (config) => config?.members?.find((member) => isLeadMember(member))?.name?.trim() || null ) .catch(() => null); return resolveOpenCodeVisibleReplyInboxCandidates({ replyRecipient: input.replyRecipient, configuredLeadName, includeUserFallbackForLeadRecipient: input.includeUserFallbackForLeadRecipient, }); } private isOpenCodeLeadReplyRecipientAlias(value: string): boolean { return isOpenCodeLeadReplyRecipientAliasValue(value); } private openCodeTaskRefsIncludeAll( actual: readonly TaskRef[] | undefined, expected: readonly TaskRef[] | undefined ): boolean { return openCodeTaskRefsIncludeAllValue(actual, expected); } private normalizeOpenCodeTaskRefsForComparison( taskRefs: readonly TaskRef[] | undefined ): TaskRef[] { return normalizeOpenCodeTaskRefsForComparisonValue(taskRefs); } private openCodeTaskRefKey(taskRef: TaskRef): string { return openCodeTaskRefKeyValue(taskRef); } private async ensureOpenCodeVisibleReplyTaskRefs(input: { teamName: string; memberName: string; ledgerRecord: OpenCodePromptDeliveryLedgerRecord; visibleReply: OpenCodeVisibleReplyProof; }): Promise<{ visibleReply: OpenCodeVisibleReplyProof; diagnostics: string[] }> { const taskRefs = this.normalizeOpenCodeTaskRefsForComparison(input.ledgerRecord.taskRefs); if (taskRefs.length === 0) { return { visibleReply: input.visibleReply, diagnostics: [] }; } if (this.openCodeTaskRefsIncludeAll(input.visibleReply.message.taskRefs, taskRefs)) { return { visibleReply: input.visibleReply, diagnostics: [] }; } const messageId = input.visibleReply.message.messageId.trim(); const relayOfMessageId = typeof input.visibleReply.message.relayOfMessageId === 'string' ? input.visibleReply.message.relayOfMessageId.trim() : ''; if (!messageId || relayOfMessageId !== input.ledgerRecord.inboxMessageId.trim()) { return { visibleReply: input.visibleReply, diagnostics: ['visible_reply_missing_task_refs'], }; } try { const merged = await this.inboxWriter.mergeRuntimeDeliveryTaskRefs(input.teamName, { inboxName: input.visibleReply.inboxName, messageId, relayOfMessageId, from: input.memberName, taskRefs, }); if (merged.message && this.openCodeTaskRefsIncludeAll(merged.message.taskRefs, taskRefs)) { const visibleReply = { ...input.visibleReply, message: merged.message, }; if (merged.updated) { this.emitRuntimeDeliveryReplyAdvisoryRefresh(input.teamName, visibleReply.message); } return { visibleReply, diagnostics: merged.updated ? ['opencode_runtime_delivery_task_refs_inherited_from_relay'] : [], }; } return { visibleReply: input.visibleReply, diagnostics: merged.found ? ['visible_reply_missing_task_refs_after_merge'] : ['visible_reply_missing_task_refs'], }; } catch (error) { logger.warn( `[${input.teamName}] Failed to merge OpenCode runtime delivery taskRefs for ${input.memberName}/${input.ledgerRecord.inboxMessageId}: ${getErrorMessage(error)}` ); return { visibleReply: input.visibleReply, diagnostics: ['visible_reply_task_refs_merge_failed'], }; } } private async applyOpenCodeVisibleDestinationProof(input: { ledger: OpenCodePromptDeliveryLedgerStore; ledgerRecord: OpenCodePromptDeliveryLedgerRecord; teamName: string; replyRecipient?: string | null; memberName: string; }): Promise<{ ledgerRecord: OpenCodePromptDeliveryLedgerRecord; visibleReply: OpenCodeVisibleReplyProof | null; }> { let visibleReply = await this.findOpenCodeVisibleReplyByRelayOfMessageId({ teamName: input.teamName, replyRecipient: input.replyRecipient ?? input.ledgerRecord.replyRecipient, from: input.memberName, relayOfMessageId: input.ledgerRecord.inboxMessageId, expectedMessageId: input.ledgerRecord.visibleReplyCorrelation === 'relayOfMessageId' ? input.ledgerRecord.visibleReplyMessageId : null, allowUserFallbackForLeadRecipient: input.ledgerRecord.visibleReplyCorrelation === 'relayOfMessageId', }); let visibleReplyCorrelation: OpenCodeVisibleReplyCorrelation = 'relayOfMessageId'; let recoveryDiagnostics: string[] = []; if (!visibleReply) { const recoveredByMessageId = await this.findOpenCodeVisibleReplyByObservedMessageId({ teamName: input.teamName, replyRecipient: input.replyRecipient ?? input.ledgerRecord.replyRecipient, from: input.memberName, ledgerRecord: input.ledgerRecord, }); if (recoveredByMessageId) { visibleReply = recoveredByMessageId.visibleReply; visibleReplyCorrelation = recoveredByMessageId.visibleReplyCorrelation; recoveryDiagnostics = recoveredByMessageId.diagnostics; } } if (!visibleReply) { const recoveredByTaskRefs = await this.findOpenCodeVisibleReplyByTaskRefs({ teamName: input.teamName, replyRecipient: input.replyRecipient ?? input.ledgerRecord.replyRecipient, from: input.memberName, ledgerRecord: input.ledgerRecord, }); if (recoveredByTaskRefs) { visibleReply = recoveredByTaskRefs.visibleReply; visibleReplyCorrelation = recoveredByTaskRefs.visibleReplyCorrelation; recoveryDiagnostics = recoveredByTaskRefs.diagnostics; } } if (!visibleReply) { return { ledgerRecord: input.ledgerRecord, visibleReply: null }; } const enriched = await this.ensureOpenCodeVisibleReplyTaskRefs({ teamName: input.teamName, memberName: input.memberName, ledgerRecord: input.ledgerRecord, visibleReply, }); const visibleReplyForProof = enriched.visibleReply; const taskRefsSatisfied = this.openCodeTaskRefsIncludeAll( visibleReplyForProof.message.taskRefs, input.ledgerRecord.taskRefs ); const semantic = isOpenCodeVisibleReplyReadCommitAllowed({ actionMode: input.ledgerRecord.actionMode, taskRefs: input.ledgerRecord.taskRefs, visibleReply: visibleReplyForProof, }) && taskRefsSatisfied; const previousTerminalSuccess = input.ledgerRecord.status === 'responded' && Boolean(input.ledgerRecord.inboxReadCommittedAt || input.ledgerRecord.visibleReplyMessageId); const shouldEmitRecoveryAdvisoryRefresh = semantic && (!previousTerminalSuccess || input.ledgerRecord.status === 'failed_terminal' || Boolean(input.ledgerRecord.failedAt) || Boolean(input.ledgerRecord.lastReason?.trim())); const ledgerRecord = await input.ledger.applyDestinationProof({ id: input.ledgerRecord.id, visibleReplyInbox: visibleReplyForProof.inboxName, visibleReplyMessageId: visibleReplyForProof.message.messageId, visibleReplyCorrelation, semanticallySufficient: semantic, diagnostics: [ ...recoveryDiagnostics, ...(visibleReplyForProof.missingRuntimeDeliverySource ? ['visible_reply_missing_runtime_delivery_source'] : []), ...enriched.diagnostics, ], observedAt: nowIso(), }); if (shouldEmitRecoveryAdvisoryRefresh) { this.emitRuntimeDeliveryReplyAdvisoryRefresh(input.teamName, visibleReplyForProof.message); } return { ledgerRecord, visibleReply: visibleReplyForProof }; } private buildOpenCodePlainTextVisibleReplyMessageId( record: OpenCodePromptDeliveryLedgerRecord ): string { const safeId = record.id.replace(/[^a-zA-Z0-9_-]/g, '-').slice(0, 96); return `opencode-plain-reply-${safeId}`; } private buildOpenCodePlainTextVisibleReplySummary(text: string): string { const normalized = text.replace(/\s+/g, ' ').trim(); return normalized.length > 120 ? `${normalized.slice(0, 117).trimEnd()}...` : normalized; } private async materializeOpenCodePlainTextReplyIfNeeded(input: { ledger: OpenCodePromptDeliveryLedgerStore; ledgerRecord: OpenCodePromptDeliveryLedgerRecord; teamName: string; memberName: string; visibleReply?: OpenCodeVisibleReplyProof | null; }): Promise<{ ledgerRecord: OpenCodePromptDeliveryLedgerRecord; visibleReply: OpenCodeVisibleReplyProof | null; }> { if (input.visibleReply) { return { ledgerRecord: input.ledgerRecord, visibleReply: input.visibleReply }; } const materializedFromMessageSendToolError = input.ledgerRecord.responseState === 'tool_error' && this.hasOpenCodeObservedMessageSendToolCall(input.ledgerRecord); if ( !this.canMaterializeOpenCodePlainTextReply(input.ledgerRecord) || !this.isOpenCodeDirectUserPromptDelivery(input.ledgerRecord) || input.ledgerRecord.visibleReplyMessageId || input.ledgerRecord.visibleReplyInbox ) { return { ledgerRecord: input.ledgerRecord, visibleReply: null }; } const text = input.ledgerRecord.observedAssistantPreview?.trim(); if (!text) { return { ledgerRecord: input.ledgerRecord, visibleReply: null }; } const semantic = isOpenCodeVisibleReplySemanticallySufficient({ actionMode: input.ledgerRecord.actionMode, taskRefs: input.ledgerRecord.taskRefs, text, }); if (!semantic.sufficient) { return { ledgerRecord: input.ledgerRecord, visibleReply: null }; } const messageId = this.buildOpenCodePlainTextVisibleReplyMessageId(input.ledgerRecord); const existing = await this.findOpenCodeVisibleReplyByRelayOfMessageId({ teamName: input.teamName, replyRecipient: 'user', from: input.memberName, relayOfMessageId: input.ledgerRecord.inboxMessageId, expectedMessageId: messageId, }); if (existing) { const enriched = await this.ensureOpenCodeVisibleReplyTaskRefs({ teamName: input.teamName, memberName: input.memberName, ledgerRecord: input.ledgerRecord, visibleReply: existing, }); const existingForProof = enriched.visibleReply; const ledgerRecord = await input.ledger.applyDestinationProof({ id: input.ledgerRecord.id, visibleReplyInbox: existingForProof.inboxName, visibleReplyMessageId: existingForProof.message.messageId, visibleReplyCorrelation: 'plain_assistant_text', semanticallySufficient: this.openCodeTaskRefsIncludeAll( existingForProof.message.taskRefs, input.ledgerRecord.taskRefs ), diagnostics: [ ...(materializedFromMessageSendToolError ? ['opencode_message_send_tool_error_plain_text_reply_materialized'] : []), ...(existingForProof.missingRuntimeDeliverySource ? ['plain_text_visible_reply_missing_runtime_delivery_source'] : []), ...enriched.diagnostics, ], observedAt: nowIso(), }); this.emitRuntimeDeliveryReplyAdvisoryRefresh(input.teamName, existingForProof.message); return { ledgerRecord, visibleReply: existingForProof }; } const timestamp = input.ledgerRecord.respondedAt ?? input.ledgerRecord.lastObservedAt ?? input.ledgerRecord.updatedAt ?? nowIso(); try { const written = await this.inboxWriter.sendMessage(input.teamName, { member: 'user', from: input.memberName, to: 'user', text, summary: this.buildOpenCodePlainTextVisibleReplySummary(text), timestamp, messageId, relayOfMessageId: input.ledgerRecord.inboxMessageId, source: 'runtime_delivery', taskRefs: input.ledgerRecord.taskRefs, }); const visibleReply: OpenCodeVisibleReplyProof = { inboxName: 'user', message: { from: input.memberName, to: 'user', text, timestamp, read: false, summary: this.buildOpenCodePlainTextVisibleReplySummary(text), messageId: written.messageId, relayOfMessageId: input.ledgerRecord.inboxMessageId, source: 'runtime_delivery', taskRefs: input.ledgerRecord.taskRefs, }, }; const ledgerRecord = await input.ledger.applyDestinationProof({ id: input.ledgerRecord.id, visibleReplyInbox: 'user', visibleReplyMessageId: written.messageId, visibleReplyCorrelation: 'plain_assistant_text', semanticallySufficient: true, diagnostics: [ ...(materializedFromMessageSendToolError ? ['opencode_message_send_tool_error_plain_text_reply_materialized'] : []), written.deduplicated ? 'opencode_plain_text_reply_materialized_deduplicated' : 'opencode_plain_text_reply_materialized_to_user_inbox', ], observedAt: nowIso(), }); this.emitRuntimeDeliveryReplyAdvisoryRefresh(input.teamName, visibleReply.message); return { ledgerRecord, visibleReply }; } catch (error) { logger.warn( `[${input.teamName}] Failed to materialize OpenCode plain-text reply for ${input.memberName}/${input.ledgerRecord.inboxMessageId}: ${getErrorMessage(error)}` ); return { ledgerRecord: input.ledgerRecord, visibleReply: null }; } } private async observeOpenCodeDirectUserDeliveryInlineIfNeeded(input: { adapter: OpenCodeRuntimeMessageAdapter; ledger: OpenCodePromptDeliveryLedgerStore; ledgerRecord: OpenCodePromptDeliveryLedgerRecord; teamName: string; memberName: string; laneId: string; cwd: string; text: string; messageId: string; runtimeRunId?: string | null; replyRecipient?: string | null; actionMode?: AgentActionMode; messageKind?: OpenCodeTeamRuntimeMessageInput['messageKind']; workSyncIntent?: OpenCodeTeamRuntimeMessageInput['workSyncIntent']; workSyncReviewRequestEventIds?: string[]; taskRefs?: TaskRef[]; promptAccepted: boolean; visibleReply?: OpenCodeVisibleReplyProof | null; }): Promise<{ ledgerRecord: OpenCodePromptDeliveryLedgerRecord; visibleReply: OpenCodeVisibleReplyProof | null; }> { let ledgerRecord = input.ledgerRecord; let visibleReply = input.visibleReply ?? null; const observeMessageDelivery = input.adapter.observeMessageDelivery; const readAllowed = await this.isOpenCodeDeliveryResponseReadCommitAllowed({ teamName: input.teamName, memberName: input.memberName, responseState: ledgerRecord.responseState, actionMode: ledgerRecord.actionMode ?? undefined, taskRefs: ledgerRecord.taskRefs, visibleReply, ledgerRecord, }); const shouldObserveInline = observeMessageDelivery && input.promptAccepted && this.isOpenCodeDirectUserPromptDelivery(ledgerRecord) && (ledgerRecord.source === 'manual' || (ledgerRecord.responseState === 'tool_error' && this.hasOpenCodeObservedMessageSendToolCall(ledgerRecord))) && !readAllowed && !visibleReply && !ledgerRecord.visibleReplyMessageId; if (!shouldObserveInline || !observeMessageDelivery) { return { ledgerRecord, visibleReply }; } for (let inlineObserveAttempt = 1; inlineObserveAttempt <= 4; inlineObserveAttempt += 1) { await sleep(OPENCODE_PROMPT_DELIVERY_OBSERVE_DELAY_MS); let observed: OpenCodeTeamRuntimeMessageResult; try { observed = await observeMessageDelivery.call(input.adapter, { ...(input.runtimeRunId ? { runId: input.runtimeRunId } : {}), teamName: input.teamName, laneId: input.laneId, memberName: input.memberName, cwd: input.cwd, text: input.text, messageId: input.messageId, replyRecipient: input.replyRecipient ?? undefined, actionMode: input.actionMode, messageKind: input.messageKind, workSyncIntent: input.workSyncIntent, workSyncReviewRequestEventIds: input.workSyncReviewRequestEventIds, taskRefs: input.taskRefs, prePromptCursor: ledgerRecord.prePromptCursor, sessionId: ledgerRecord.runtimeSessionId ?? undefined, runtimePromptMessageId: ledgerRecord.lastRuntimePromptMessageId ?? ledgerRecord.runtimePromptMessageId ?? undefined, }); } catch (error) { const reason = `opencode_direct_user_delivery_inline_observe_failed: ${getErrorMessage( error )}`; await this.maybeSyncOpenCodeRuntimePermissionsAfterDelivery({ teamName: input.teamName, runId: input.runtimeRunId, laneId: input.laneId, memberName: input.memberName, cwd: input.cwd, sessionId: ledgerRecord.runtimeSessionId, reason, diagnostics: [ `opencode_direct_user_delivery_inline_observe_attempt_${inlineObserveAttempt}`, reason, ], }); ledgerRecord = await input.ledger.applyObservation({ id: ledgerRecord.id, responseObservation: { state: 'reconcile_failed', deliveredUserMessageId: null, assistantMessageId: null, toolCallNames: [], visibleMessageToolCallId: null, visibleReplyMessageId: null, visibleReplyCorrelation: null, latestAssistantPreview: null, reason, }, diagnostics: [ `opencode_direct_user_delivery_inline_observe_attempt_${inlineObserveAttempt}`, reason, ], observedAt: nowIso(), }); break; } await this.rememberOpenCodeRuntimePidFromBridge({ teamName: input.teamName, memberName: input.memberName, laneId: input.laneId, runId: input.runtimeRunId, runtimeSessionId: observed.sessionId, runtimePid: observed.runtimePid, reason: 'opencode_delivery_inline_observe_runtime_pid_observed', }); const observedResponse = this.normalizeOpenCodeDeliveryResponseObservation( observed.responseObservation ); await this.maybeSyncOpenCodeRuntimePermissionsAfterDelivery({ teamName: input.teamName, runId: input.runtimeRunId, laneId: input.laneId, memberName: input.memberName, cwd: input.cwd, sessionId: observed.sessionId, responseState: observedResponse?.state, reason: observedResponse?.reason ?? observed.diagnostics[0], diagnostics: observed.diagnostics, }); const hadMessageSendToolError = this.hasOpenCodeObservedMessageSendToolCall(ledgerRecord); ledgerRecord = await input.ledger.applyObservation({ id: ledgerRecord.id, responseObservation: observedResponse ?? { state: observed.ok ? 'not_observed' : 'reconcile_failed', deliveredUserMessageId: null, assistantMessageId: null, toolCallNames: [], visibleMessageToolCallId: null, visibleReplyMessageId: null, visibleReplyCorrelation: null, latestAssistantPreview: null, reason: observed.diagnostics[0] ?? null, }, sessionId: observed.sessionId, runtimePromptMessageId: observed.runtimePromptMessageId, diagnostics: [ `opencode_direct_user_delivery_inline_observe_attempt_${inlineObserveAttempt}`, ...(hadMessageSendToolError ? ['opencode_message_send_tool_error_inline_observe'] : []), ...observed.diagnostics, ], observedAt: nowIso(), }); const proof = await this.applyOpenCodeVisibleDestinationProof({ ledger: input.ledger, ledgerRecord, teamName: input.teamName, replyRecipient: input.replyRecipient, memberName: input.memberName, }); ledgerRecord = proof.ledgerRecord; visibleReply = proof.visibleReply; const materialized = await this.materializeOpenCodePlainTextReplyIfNeeded({ ledger: input.ledger, ledgerRecord, teamName: input.teamName, memberName: input.memberName, visibleReply, }); ledgerRecord = materialized.ledgerRecord; visibleReply = materialized.visibleReply; const observedReadAllowed = await this.isOpenCodeDeliveryResponseReadCommitAllowed({ teamName: input.teamName, memberName: input.memberName, responseState: ledgerRecord.responseState, actionMode: ledgerRecord.actionMode ?? undefined, taskRefs: ledgerRecord.taskRefs, visibleReply, ledgerRecord, }); if (observedReadAllowed) { break; } } return { ledgerRecord, visibleReply }; } private getOpenCodeDeliveryWatchdogKey(input: { teamName: string; memberName: string; messageId: string; }): string { return `opencode-delivery:${input.teamName}:${input.memberName.toLowerCase()}:${input.messageId}`; } private enqueueOpenCodePromptDeliveryWatchdogJob(input: { teamName: string; run: () => Promise; }): void { this.openCodePromptDeliveryWatchdogQueue.push(input); this.drainOpenCodePromptDeliveryWatchdogQueue(); } private drainOpenCodePromptDeliveryWatchdogQueue(): void { while ( this.openCodePromptDeliveryWatchdogInFlight < OPENCODE_PROMPT_WATCHDOG_GLOBAL_CONCURRENCY && this.openCodePromptDeliveryWatchdogQueue.length > 0 ) { const nextIndex = this.openCodePromptDeliveryWatchdogQueue.findIndex( (queued) => (this.openCodePromptDeliveryWatchdogInFlightByTeam.get(queued.teamName) ?? 0) < OPENCODE_PROMPT_WATCHDOG_PER_TEAM_CONCURRENCY ); if (nextIndex < 0) { return; } const [job] = this.openCodePromptDeliveryWatchdogQueue.splice(nextIndex, 1); if (!job) { return; } this.openCodePromptDeliveryWatchdogInFlight += 1; this.openCodePromptDeliveryWatchdogInFlightByTeam.set( job.teamName, (this.openCodePromptDeliveryWatchdogInFlightByTeam.get(job.teamName) ?? 0) + 1 ); void job .run() .catch((error: unknown) => { logger.warn(`OpenCode prompt delivery watchdog job failed: ${getErrorMessage(error)}`); }) .finally(() => { this.openCodePromptDeliveryWatchdogInFlight = Math.max( 0, this.openCodePromptDeliveryWatchdogInFlight - 1 ); const teamInFlight = (this.openCodePromptDeliveryWatchdogInFlightByTeam.get(job.teamName) ?? 1) - 1; if (teamInFlight > 0) { this.openCodePromptDeliveryWatchdogInFlightByTeam.set(job.teamName, teamInFlight); } else { this.openCodePromptDeliveryWatchdogInFlightByTeam.delete(job.teamName); } this.drainOpenCodePromptDeliveryWatchdogQueue(); }); } } private async isStaleOpenCodePromptDeliveryWatchdogError(input: { teamName: string; memberName: string; messageId: string; error: unknown; }): Promise { if (!getErrorMessage(input.error).startsWith('OpenCode prompt delivery record not found:')) { return false; } if (!this.canDeliverToOpenCodeRuntimeForTeam(input.teamName)) { return true; } const inboxMessages = await this.inboxReader .getMessagesFor(input.teamName, input.memberName) .catch(() => []); const targetMessage = inboxMessages.find((message) => message.messageId === input.messageId); if (!targetMessage || targetMessage.read) { return true; } const identity = await this.resolveOpenCodeMemberDeliveryIdentity( input.teamName, input.memberName ).catch(() => null); if (!identity?.ok) { return true; } const laneActive = await this.isOpenCodeRuntimeLaneIndexActive( input.teamName, identity.laneId ).catch(() => false); return !laneActive; } private scheduleOpenCodePromptDeliveryWatchdog(input: { teamName: string; memberName: string; messageId?: string | null; delayMs: number; }): void { if (!this.isOpenCodePromptDeliveryWatchdogEnabled()) { return; } const messageId = input.messageId?.trim(); if (!messageId) return; const key = this.getOpenCodeDeliveryWatchdogKey({ teamName: input.teamName, memberName: input.memberName, messageId, }); const delayMs = Math.max(500, Math.min(input.delayMs, 60_000)); const deadlineMs = Date.now() + delayMs; const existing = this.openCodePromptDeliveryWatchdogTimers.get(key); if (existing) { const existingDeadlineMs = this.openCodePromptDeliveryWatchdogDeadlines.get(key); if (typeof existingDeadlineMs === 'number' && existingDeadlineMs <= deadlineMs + 25) { return; } clearTimeout(existing); } const timer = setTimeout(() => { this.openCodePromptDeliveryWatchdogTimers.delete(key); this.openCodePromptDeliveryWatchdogDeadlines.delete(key); this.enqueueOpenCodePromptDeliveryWatchdogJob({ teamName: input.teamName, run: async () => { if (!this.canDeliverToOpenCodeRuntimeForTeam(input.teamName)) { const recovered = await this.tryRecoverOpenCodeRuntimeLaneForConfiguredMemberBeforeDelivery({ teamName: input.teamName, memberName: input.memberName, }); if (!recovered) { return; } } try { await this.relayOpenCodeMemberInboxMessages(input.teamName, input.memberName, { onlyMessageId: messageId, source: 'watchdog', }); } catch (error) { if ( await this.isStaleOpenCodePromptDeliveryWatchdogError({ teamName: input.teamName, memberName: input.memberName, messageId, error, }) ) { logger.debug( `[${input.teamName}] Ignoring stale OpenCode prompt delivery watchdog job for ${input.memberName}/${messageId}: ${getErrorMessage(error)}` ); return; } throw error; } }, }); }, delayMs); this.openCodePromptDeliveryWatchdogTimers.set(key, timer); this.openCodePromptDeliveryWatchdogDeadlines.set(key, deadlineMs); } private getOpenCodeDeliveryNextDelayMs(input: { responseState?: NonNullable['state']; retry: boolean; ledgerRecord?: OpenCodePromptDeliveryLedgerRecord | null; }): number { if ( input.retry && input.responseState === 'tool_error' && this.hasOpenCodeObservedMessageSendToolCall(input.ledgerRecord) ) { return OPENCODE_PROMPT_DELIVERY_OBSERVE_DELAY_MS; } if (input.retry) { return OPENCODE_PROMPT_DELIVERY_RETRY_DELAY_MS; } if (isOpenCodePromptDeliveryObserveLaterResponseState(input.responseState)) { return OPENCODE_PROMPT_DELIVERY_OBSERVE_DELAY_MS; } return OPENCODE_PROMPT_DELIVERY_RETRY_DELAY_MS; } private isOpenCodePromptDeliveryWatchdogRecordTerminal( record: OpenCodePromptDeliveryLedgerRecord ): boolean { if (record.status === 'failed_terminal') { return true; } if (record.status !== 'responded') { return false; } return !( record.responseState === 'responded_plain_text' && !record.visibleReplyMessageId && !record.inboxReadCommittedAt ); } private isOpenCodeSessionRefreshRetryRecord( record: OpenCodePromptDeliveryLedgerRecord, reason: string | null | undefined ): boolean { const stampedSessionRefreshReason = record.lastSessionRefreshReason?.trim(); const stampedSessionRefreshReasonIsExplicit = this.isExplicitOpenCodeSessionRefreshStamp( stampedSessionRefreshReason ); const currentReason = reason?.trim(); const lastReason = record.lastReason?.trim(); const currentReasonConfirmsStamp = currentReason ? currentReason === stampedSessionRefreshReason : lastReason === stampedSessionRefreshReason; if ( record.responseState === 'session_stale' && stampedSessionRefreshReason && stampedSessionRefreshReasonIsExplicit && currentReasonConfirmsStamp ) { return isOpenCodeSessionRefreshResponseState({ responseState: record.responseState, reason: currentReason ?? stampedSessionRefreshReason, }); } if (record.responseState !== 'session_stale') { return isOpenCodeSessionRefreshResponseState({ responseState: record.responseState, reason, }); } return isOpenCodeSessionRefreshResponseState({ responseState: record.responseState, reason, diagnostics: record.diagnostics, }); } private isExplicitOpenCodeSessionRefreshStamp(reason: string | null | undefined): boolean { return ( isOpenCodeResolvedBehaviorChangedReason(reason) || isOpenCodeSessionTransportChangedReason(reason) ); } private async scheduleOpenCodePromptLedgerFollowUp(input: { ledger: OpenCodePromptDeliveryLedgerStore; ledgerRecord: OpenCodePromptDeliveryLedgerRecord; teamName: string; memberName: string; retry: boolean; reason: string; }): Promise { const now = nowIso(); const sessionRefreshRetry = input.retry && this.isOpenCodeSessionRefreshRetryRecord(input.ledgerRecord, input.reason); const acceptedPromptSessionStaleObservation = !input.retry && input.ledgerRecord.responseState === 'session_stale' && this.hasOpenCodeAcceptedRuntimePrompt(input.ledgerRecord); if (acceptedPromptSessionStaleObservation) { const maxSessionRefreshAttempts = input.ledgerRecord.maxSessionRefreshAttempts ?? OPENCODE_PROMPT_DELIVERY_SESSION_REFRESH_MAX_ATTEMPTS; if ((input.ledgerRecord.sessionRefreshAttempts ?? 0) >= maxSessionRefreshAttempts) { return await this.markOpenCodePromptLedgerFailedTerminal({ ledger: input.ledger, id: input.ledgerRecord.id, reason: 'opencode_session_stale_observe_loop_after_accepted_prompt', diagnostics: [ input.reason, `OpenCode session stayed stale while observing an accepted prompt after ${maxSessionRefreshAttempts} attempt(s).`, ], failedAt: now, eventContext: { observeOnlyAfterAcceptedPrompt: true, sessionRefreshAttempts: input.ledgerRecord.sessionRefreshAttempts ?? 0, maxSessionRefreshAttempts, }, }); } const delayMs = OPENCODE_PROMPT_DELIVERY_RETRY_DELAY_MS; const nextAttemptAt = new Date(Date.now() + delayMs).toISOString(); const ledgerRecord = await input.ledger.markSessionStaleObservationScheduled({ id: input.ledgerRecord.id, nextAttemptAt, reason: input.reason, scheduledAt: now, maxSessionRefreshAttempts, diagnostics: ['opencode_session_stale_observe_scheduled_after_accepted_prompt'], }); this.logOpenCodePromptDeliveryEvent( 'opencode_prompt_delivery_response_observed', ledgerRecord, { retry: false, reason: input.reason, observeOnlyAfterAcceptedPrompt: true, sessionRefreshAttempts: ledgerRecord.sessionRefreshAttempts ?? 0, maxSessionRefreshAttempts, } ); this.scheduleOpenCodePromptDeliveryWatchdog({ teamName: input.teamName, memberName: input.memberName, messageId: input.ledgerRecord.inboxMessageId, delayMs, }); return ledgerRecord; } if (sessionRefreshRetry) { const maxSessionRefreshAttempts = input.ledgerRecord.maxSessionRefreshAttempts ?? OPENCODE_PROMPT_DELIVERY_SESSION_REFRESH_MAX_ATTEMPTS; if ((input.ledgerRecord.sessionRefreshAttempts ?? 0) >= maxSessionRefreshAttempts) { return await this.markOpenCodePromptLedgerFailedTerminal({ ledger: input.ledger, id: input.ledgerRecord.id, reason: 'opencode_session_refresh_loop_after_resolved_behavior_changed', diagnostics: [ input.reason, `OpenCode session stayed stale after ${maxSessionRefreshAttempts} session refresh attempt(s).`, ], failedAt: now, eventContext: { retry: true, sessionRefreshAttempts: input.ledgerRecord.sessionRefreshAttempts ?? 0, maxSessionRefreshAttempts, }, }); } const delayMs = this.getOpenCodeDeliveryNextDelayMs({ responseState: input.ledgerRecord.responseState, retry: input.retry, ledgerRecord: input.ledgerRecord, }); const nextAttemptAt = new Date(Date.now() + delayMs).toISOString(); const ledgerRecord = await input.ledger.markSessionRefreshScheduled({ id: input.ledgerRecord.id, nextAttemptAt, reason: input.reason, scheduledAt: now, maxSessionRefreshAttempts, diagnostics: ['opencode_session_refresh_scheduled_after_resolved_behavior_changed'], }); this.logOpenCodePromptDeliveryEvent( 'opencode_prompt_delivery_session_refresh_scheduled', ledgerRecord, { retry: true, reason: input.reason, sessionRefreshAttempts: ledgerRecord.sessionRefreshAttempts ?? 0, maxSessionRefreshAttempts, } ); this.scheduleOpenCodePromptDeliveryWatchdog({ teamName: input.teamName, memberName: input.memberName, messageId: input.ledgerRecord.inboxMessageId, delayMs, }); return ledgerRecord; } const canScheduleNoAssistantRecoveryRetry = input.retry && input.ledgerRecord.attempts === input.ledgerRecord.maxAttempts && this.isOpenCodeNoAssistantDeliveryFailure(input.ledgerRecord); if ( input.retry && input.ledgerRecord.attempts >= input.ledgerRecord.maxAttempts && !canScheduleNoAssistantRecoveryRetry ) { return await this.markOpenCodePromptLedgerFailedTerminal({ ledger: input.ledger, id: input.ledgerRecord.id, reason: input.reason, failedAt: now, eventContext: { retry: input.retry }, }); } const delayMs = this.getOpenCodeDeliveryNextDelayMs({ responseState: input.ledgerRecord.responseState, retry: input.retry, ledgerRecord: input.ledgerRecord, }); const nextAttemptAt = new Date(Date.now() + delayMs).toISOString(); const ledgerRecord = await input.ledger.markNextAttemptScheduled({ id: input.ledgerRecord.id, status: input.retry ? 'retry_scheduled' : 'accepted', nextAttemptAt, reason: input.reason, scheduledAt: now, }); this.logOpenCodePromptDeliveryEvent( input.retry ? 'opencode_prompt_delivery_retry_scheduled' : 'opencode_prompt_delivery_response_observed', ledgerRecord, { retry: input.retry, reason: input.reason } ); this.scheduleOpenCodePromptDeliveryWatchdog({ teamName: input.teamName, memberName: input.memberName, messageId: input.ledgerRecord.inboxMessageId, delayMs, }); return ledgerRecord; } private async rememberOpenCodeRuntimePidFromBridge(input: { teamName: string; memberName: string; laneId: string; runId?: string | null; runtimeSessionId?: string | null; runtimePid?: number; reason: string; }): Promise { const runtimePid = normalizeRuntimePositiveInteger(input.runtimePid); if (!runtimePid) { return; } const command = this.readProcessCommandByPid(runtimePid); if (!command || !this.isOpenCodeServeCommand(command)) { logger.debug( `[${input.teamName}] Ignoring OpenCode bridge runtime pid ${runtimePid} for ${input.memberName}: process identity is not an active opencode serve host.` ); return; } const observedAt = nowIso(); try { const changed = await this.enqueueLaunchStateStoreOperation(input.teamName, async () => { const previous = await this.launchStateStore.read(input.teamName).catch(() => null); const previousEntry = this.findPersistedLaunchMemberForLane({ previousLaunchState: previous, laneId: input.laneId, memberName: input.memberName, runId: input.runId, }); if (!previous || !previousEntry) { return false; } const previousMember = previousEntry.member; if (!isPersistedOpenCodeSecondaryLaneMember(previousMember)) { return false; } if (previousMember.laneId && previousMember.laneId !== input.laneId) { return false; } const previousRunId = previousMember.runtimeRunId?.trim(); const incomingRunId = input.runId?.trim(); if (previousRunId && incomingRunId && previousRunId !== incomingRunId) { return false; } const previousSessionId = previousMember.runtimeSessionId?.trim(); const incomingSessionId = input.runtimeSessionId?.trim(); if (previousSessionId && incomingSessionId && previousSessionId !== incomingSessionId) { return false; } if ( previousMember.runtimePid === runtimePid && previousMember.pidSource === 'opencode_bridge' ) { return false; } const nextMember: PersistedTeamLaunchMemberState = { ...previousMember, runtimePid, ...(incomingRunId ? { runtimeRunId: incomingRunId } : {}), ...(incomingSessionId ? { runtimeSessionId: incomingSessionId } : {}), pidSource: 'opencode_bridge', lastRuntimeAliveAt: observedAt, lastEvaluatedAt: observedAt, sources: { ...(previousMember.sources ?? {}), processAlive: true, }, diagnostics: mergeRuntimeDiagnostics( previousMember.diagnostics, [`runtime pid: ${runtimePid}`, input.reason], previousMember.runtimeDiagnostic ), }; const nextSnapshot = createPersistedLaunchSnapshot({ teamName: previous.teamName, expectedMembers: previous.expectedMembers, bootstrapExpectedMembers: previous.bootstrapExpectedMembers, leadSessionId: previous.leadSessionId, launchPhase: previous.launchPhase, members: { ...previous.members, [previousEntry.key]: nextMember, }, updatedAt: observedAt, }); await this.writeLaunchStateSnapshotNow(input.teamName, nextSnapshot); return true; }); if (changed) { this.invalidateRuntimeSnapshotCaches(input.teamName); this.teamChangeEmitter?.({ type: 'member-spawn', teamName: input.teamName, ...(input.runId ? { runId: input.runId } : {}), detail: input.memberName, }); } } catch (error) { logger.debug( `[${input.teamName}] Failed to persist OpenCode bridge runtime pid ${runtimePid} for ${input.memberName}: ${getErrorMessage(error)}` ); } } private hasOpenCodePendingPermissionSignal(input: { responseState?: OpenCodeMemberInboxDelivery['responseState']; reason?: string | null; diagnostics?: readonly string[]; }): boolean { if (input.responseState === 'permission_blocked') { return true; } const text = [input.reason ?? undefined, ...(input.diagnostics ?? [])] .filter((value): value is string => Boolean(value?.trim())) .join('\n'); return OPENCODE_PENDING_PERMISSION_REQUEST_PATTERN.test(text); } private findPersistedLaunchMemberForLane(input: { previousLaunchState: PersistedTeamLaunchSnapshot | null | undefined; laneId: string; memberName: string; runId?: string | null; }): { key: string; member: PersistedTeamLaunchMemberState } | null { const members = input.previousLaunchState?.members; if (!members) { return null; } const laneId = input.laneId.trim() || 'primary'; const memberName = input.memberName.trim(); const runId = input.runId?.trim(); const candidates = Object.entries(members).filter(([key, member]) => { const storedName = this.resolvePersistedLaunchMemberDisplayName(key, member); if (storedName !== memberName) { return false; } if ((member.laneId?.trim() || 'primary') !== laneId) { return false; } const memberRunId = member.runtimeRunId?.trim(); return !(runId && memberRunId && memberRunId !== runId); }); if (candidates.length === 0) { return null; } const direct = candidates.find(([key]) => key === memberName); const [key, member] = direct ?? candidates[0]; return { key, member }; } private resolvePersistedLaunchMemberDisplayName( key: string, member: PersistedTeamLaunchMemberState ): string { const storedName = member.name?.trim(); const laneId = member.laneId?.trim(); const laneMemberName = (laneId ? this.extractOpenCodeRuntimeLaneMemberName(laneId) : null) ?? this.extractOpenCodeRuntimeLaneMemberName(key); if (storedName && storedName !== laneId && storedName !== key.trim()) { return storedName; } return laneMemberName ?? storedName ?? key.trim(); } private async maybeSyncOpenCodeRuntimePermissionsAfterDelivery(input: { teamName: string; runId?: string | null; laneId: string; memberName: string; cwd: string; sessionId?: string | null; responseState?: OpenCodeMemberInboxDelivery['responseState']; reason?: string | null; diagnostics?: readonly string[]; teamColor?: string; teamDisplayName?: string; }): Promise { if (!input.runId?.trim()) { return; } const runId = input.runId.trim(); if (this.getTrackedRunId(input.teamName) !== runId) { return; } if (!this.hasOpenCodePendingPermissionSignal(input)) { return; } const adapter = this.getOpenCodeRuntimePermissionListingAdapter(); if (!adapter) { logger.warn( `[${input.teamName}] OpenCode runtime permission signal observed for ${input.memberName}, but permission listing bridge is unavailable.` ); return; } let listed: { permissions: TeamRuntimePendingPermission[]; diagnostics: string[] }; try { listed = await adapter.listRuntimePermissions({ teamName: input.teamName, laneId: input.laneId, cwd: input.cwd, memberName: input.memberName, sessionId: input.sessionId, }); } catch (error) { logger.warn( `[${input.teamName}] Failed to list OpenCode runtime permissions for ${input.memberName}: ${getErrorMessage(error)}` ); return; } if (this.getTrackedRunId(input.teamName) !== runId) { return; } const pendingPermissions = listed.permissions.filter((permission) => this.isOpenCodeRuntimePermissionForDeliveryTarget(input, permission) ); if (pendingPermissions.length === 0) { const listedDiagnostics = listed.diagnostics.length ? ` Diagnostics: ${listed.diagnostics.join(' | ')}` : ''; logger.warn( `[${input.teamName}] OpenCode runtime permission signal observed for ${input.memberName}, but bridge listed no matching pending permissions.${listedDiagnostics}` ); return; } const previousLaunchState = await this.launchStateStore.read(input.teamName).catch(() => null); if (this.getTrackedRunId(input.teamName) !== runId) { return; } const expectedMembers = this.resolveOpenCodeRuntimePermissionExpectedMembers({ teamName: input.teamName, runId, laneId: input.laneId, memberName: input.memberName, cwd: input.cwd, previousLaunchState, }); const permissionsByMember = this.groupOpenCodeRuntimePermissionsByMember({ permissions: pendingPermissions, teamName: input.teamName, laneId: input.laneId, memberName: input.memberName, runId, sessionId: input.sessionId, expectedMembers, previousLaunchState, }); if (permissionsByMember.size === 0) { return; } await this.persistOpenCodeRuntimePendingPermissions({ ...input, permissionsByMember, previousLaunchState, }); if (this.getTrackedRunId(input.teamName) !== runId) { return; } this.syncOpenCodeRuntimePermissionSpawnStatuses({ ...input, permissionsByMember, }); const members: Record = {}; for (const [memberName, permissions] of permissionsByMember) { members[memberName] = this.buildOpenCodePermissionPendingEvidence({ teamName: input.teamName, laneId: input.laneId, memberName, permissions, runId, sessionId: input.sessionId, previousLaunchState, }); } this.syncOpenCodeRuntimeToolApprovals({ teamName: input.teamName, runId: input.runId, laneId: input.laneId, cwd: input.cwd, members, expectedMembers, memberNames: Array.from(permissionsByMember.keys()), teamColor: input.teamColor, teamDisplayName: input.teamDisplayName, }); } private isOpenCodeRuntimePermissionForDeliveryTarget( input: { laneId: string; sessionId?: string | null; }, permission: TeamRuntimePendingPermission ): boolean { const permissionSessionId = permission.sessionId?.trim(); const inputSessionId = input.sessionId?.trim(); if (permissionSessionId && inputSessionId) { return permissionSessionId === inputSessionId; } return true; } private resolveOpenCodeRuntimePermissionExpectedMembers(input: { teamName: string; runId: string; laneId: string; memberName: string; cwd: string; previousLaunchState: PersistedTeamLaunchSnapshot | null; }): TeamRuntimeMemberSpec[] { const members = new Map(); for (const [memberKey, member] of Object.entries(input.previousLaunchState?.members ?? {})) { if (member.providerId !== 'opencode') continue; if ((member.laneId?.trim() || 'primary') !== input.laneId) continue; const memberRunId = member.runtimeRunId?.trim(); if (memberRunId && memberRunId !== input.runId) continue; const displayName = this.resolvePersistedLaunchMemberDisplayName(memberKey, member); members.set(displayName, { name: displayName, role: undefined, workflow: undefined, isolation: undefined, providerId: 'opencode', model: member.model, effort: member.effort, cwd: member.cwd?.trim() || input.cwd, }); } const trackedRunId = this.getTrackedRunId(input.teamName); const trackedRun = trackedRunId ? this.runs.get(trackedRunId) : null; for (const member of [ ...(trackedRun?.allEffectiveMembers ?? []), ...(trackedRun?.effectiveMembers ?? []), ]) { if (member.providerId !== 'opencode' || members.has(member.name)) continue; const laneIdentity = buildPlannedMemberLaneIdentity({ leadProviderId: resolveTeamProviderId(trackedRun?.request.providerId), member: { name: member.name, providerId: 'opencode', }, }); if (laneIdentity.laneId !== input.laneId) continue; members.set(member.name, { name: member.name, role: member.role, workflow: member.workflow, isolation: member.isolation === 'worktree' ? 'worktree' : undefined, providerId: 'opencode', model: member.model, effort: member.effort, cwd: member.cwd?.trim() || input.cwd, }); } const runtimeRun = this.runtimeAdapterRunByTeam.get(input.teamName); if ( (input.laneId.trim() || 'primary') === 'primary' && runtimeRun?.runId === input.runId && runtimeRun.providerId === 'opencode' ) { for (const [memberKey, evidence] of Object.entries(runtimeRun.members ?? {})) { const memberName = evidence.memberName?.trim() || memberKey; if (!memberName || members.has(memberName)) continue; members.set(memberName, { name: memberName, providerId: 'opencode', model: evidence.model, cwd: input.cwd, }); } } if (!members.has(input.memberName)) { members.set(input.memberName, { name: input.memberName, providerId: 'opencode', cwd: input.cwd, }); } return Array.from(members.values()); } private groupOpenCodeRuntimePermissionsByMember(input: { permissions: readonly TeamRuntimePendingPermission[]; teamName: string; laneId: string; memberName: string; runId: string; sessionId?: string | null; expectedMembers: readonly TeamRuntimeMemberSpec[]; previousLaunchState: PersistedTeamLaunchSnapshot | null; }): Map { const sessionToMember = new Map(); for (const [memberName, member] of Object.entries(input.previousLaunchState?.members ?? {})) { if ((member.laneId?.trim() || 'primary') !== input.laneId) continue; const memberRunId = member.runtimeRunId?.trim(); if (memberRunId && memberRunId !== input.runId) continue; const sessionId = member.runtimeSessionId?.trim(); if (sessionId) { sessionToMember.set( sessionId, this.resolvePersistedLaunchMemberDisplayName(memberName, member) ); } } const trackedRunId = this.getTrackedRunId(input.teamName); const trackedRun = trackedRunId ? this.runs.get(trackedRunId) : null; const lane = trackedRun?.mixedSecondaryLanes?.find( (candidate) => candidate.laneId === input.laneId ); for (const [memberName, evidence] of Object.entries(lane?.result?.members ?? {})) { const sessionId = evidence.sessionId?.trim(); if (sessionId) { sessionToMember.set(sessionId, evidence.memberName?.trim() || memberName); } } const runtimeRun = this.runtimeAdapterRunByTeam.get(input.teamName); if ( (input.laneId.trim() || 'primary') === 'primary' && runtimeRun?.runId === input.runId && runtimeRun.providerId === 'opencode' ) { for (const [memberName, evidence] of Object.entries(runtimeRun.members ?? {})) { const sessionId = evidence.sessionId?.trim(); if (sessionId) { sessionToMember.set(sessionId, evidence.memberName?.trim() || memberName); } } } const singleExpectedMember = input.expectedMembers.length === 1 ? input.expectedMembers[0]?.name : undefined; const inputSessionId = input.sessionId?.trim(); const result = new Map(); for (const permission of input.permissions) { const permissionSessionId = permission.sessionId?.trim(); const memberName = permissionSessionId ? (sessionToMember.get(permissionSessionId) ?? (inputSessionId === permissionSessionId ? input.memberName : undefined) ?? singleExpectedMember) : (singleExpectedMember ?? input.memberName); if (!memberName) { continue; } result.set(memberName, [...(result.get(memberName) ?? []), permission]); } return result; } private buildOpenCodePermissionPendingEvidence(input: { teamName: string; laneId: string; memberName: string; permissions: readonly TeamRuntimePendingPermission[]; runId: string; sessionId?: string | null; previousLaunchState: PersistedTeamLaunchSnapshot | null; }): TeamRuntimeMemberLaunchEvidence { const previous = this.findPersistedLaunchMemberForLane({ previousLaunchState: input.previousLaunchState, laneId: input.laneId, memberName: input.memberName, runId: input.runId, })?.member; const ids = Array.from(new Set(input.permissions.map((permission) => permission.requestId))); const sessionId = previous?.runtimeSessionId ?? input.sessionId?.trim() ?? undefined; return { memberName: input.memberName, providerId: 'opencode', ...(previous?.model ? { model: previous.model } : {}), launchState: previous?.launchState === 'confirmed_alive' || previous?.bootstrapConfirmed ? 'confirmed_alive' : 'runtime_pending_permission', agentToolAccepted: previous?.agentToolAccepted ?? true, runtimeAlive: previous?.runtimeAlive ?? false, bootstrapConfirmed: previous?.bootstrapConfirmed ?? false, hardFailure: false, pendingPermissionRequestIds: ids, pendingApprovals: [...input.permissions], pendingPermissions: [...input.permissions], ...(sessionId ? { sessionId } : {}), livenessKind: previous?.livenessKind ?? 'permission_blocked', runtimeDiagnostic: 'OpenCode runtime is waiting for permission approval', runtimeDiagnosticSeverity: 'warning', diagnostics: [ 'OpenCode runtime permission request discovered after delivery was blocked.', ...(previous?.diagnostics ?? []), ], }; } private async persistOpenCodeRuntimePendingPermissions(input: { teamName: string; runId?: string | null; laneId: string; sessionId?: string | null; permissionsByMember: ReadonlyMap; previousLaunchState: PersistedTeamLaunchSnapshot | null; }): Promise { if (!input.previousLaunchState) { return; } const observedAt = nowIso(); try { const changed = await this.enqueueLaunchStateStoreOperation(input.teamName, async () => { const incomingRunId = input.runId?.trim(); if (incomingRunId && this.getTrackedRunId(input.teamName) !== incomingRunId) { return false; } const previous = await this.launchStateStore.read(input.teamName).catch(() => null); if (!previous) { return false; } let didChange = false; const members = { ...previous.members }; for (const [memberName, permissions] of input.permissionsByMember) { const previousEntry = this.findPersistedLaunchMemberForLane({ previousLaunchState: previous, laneId: input.laneId, memberName, runId: input.runId, }); if (!previousEntry || previousEntry.member.providerId !== 'opencode') { continue; } const previousMember = previousEntry.member; if ((previousMember.laneId?.trim() || 'primary') !== input.laneId) { continue; } const previousRunId = previousMember.runtimeRunId?.trim(); if (previousRunId && incomingRunId && previousRunId !== incomingRunId) { continue; } const previousSessionId = previousMember.runtimeSessionId?.trim(); const incomingSessionId = input.sessionId?.trim(); if (previousSessionId && incomingSessionId && previousSessionId !== incomingSessionId) { continue; } const pendingPermissionRequestIds = Array.from( new Set(permissions.map((permission) => permission.requestId.trim()).filter(Boolean)) ); const nextMember: PersistedTeamLaunchMemberState = { ...previousMember, name: memberName, launchState: previousMember.launchState === 'confirmed_alive' || previousMember.bootstrapConfirmed ? 'confirmed_alive' : 'runtime_pending_permission', hardFailure: false, hardFailureReason: undefined, pendingPermissionRequestIds, ...(incomingRunId ? { runtimeRunId: incomingRunId } : {}), ...(incomingSessionId && !previousSessionId ? { runtimeSessionId: incomingSessionId } : {}), livenessKind: previousMember.livenessKind ?? 'permission_blocked', runtimeDiagnostic: 'OpenCode runtime is waiting for permission approval', runtimeDiagnosticSeverity: 'warning', lastEvaluatedAt: observedAt, diagnostics: mergeRuntimeDiagnostics( previousMember.diagnostics, ['waiting for permission approval'], previousMember.runtimeDiagnostic ), }; if ( previousMember.name === nextMember.name && previousMember.launchState === nextMember.launchState && previousMember.hardFailure === nextMember.hardFailure && previousMember.hardFailureReason === nextMember.hardFailureReason && previousMember.pendingPermissionRequestIds?.join('\0') === nextMember.pendingPermissionRequestIds?.join('\0') && previousMember.runtimeRunId === nextMember.runtimeRunId && previousMember.runtimeSessionId === nextMember.runtimeSessionId && previousMember.livenessKind === nextMember.livenessKind && previousMember.runtimeDiagnostic === nextMember.runtimeDiagnostic && previousMember.runtimeDiagnosticSeverity === nextMember.runtimeDiagnosticSeverity ) { continue; } members[previousEntry.key] = nextMember; didChange = true; } if (!didChange) { return false; } const nextSnapshot = createPersistedLaunchSnapshot({ teamName: previous.teamName, expectedMembers: previous.expectedMembers, bootstrapExpectedMembers: previous.bootstrapExpectedMembers, leadSessionId: previous.leadSessionId, launchPhase: previous.launchPhase, members, updatedAt: observedAt, }); await this.writeLaunchStateSnapshotNow(input.teamName, nextSnapshot); return true; }); if (changed) { this.invalidateRuntimeSnapshotCaches(input.teamName); for (const memberName of input.permissionsByMember.keys()) { this.teamChangeEmitter?.({ type: 'member-spawn', teamName: input.teamName, ...(input.runId ? { runId: input.runId } : {}), detail: memberName, }); } } } catch (error) { logger.debug( `[${input.teamName}] Failed to persist OpenCode pending runtime permissions: ${getErrorMessage(error)}` ); } } private syncOpenCodeRuntimePermissionSpawnStatuses(input: { teamName: string; runId?: string | null; laneId: string; permissionsByMember: ReadonlyMap; }): void { const trackedRunId = this.getTrackedRunId(input.teamName); const run = trackedRunId ? this.runs.get(trackedRunId) : null; if (!run || run.runId !== input.runId) { return; } const updatedAt = nowIso(); for (const [memberName, permissions] of input.permissionsByMember) { const prev = run.memberSpawnStatuses.get(memberName) ?? createInitialMemberSpawnStatusEntry(); const lane = run.mixedSecondaryLanes?.find((candidate) => candidate.laneId === input.laneId); const laneEvidence = lane?.result?.members?.[memberName]; const pendingPermissionRequestIds = Array.from( new Set(permissions.map((permission) => permission.requestId.trim()).filter(Boolean)) ); const joinedPendingPermissionRequestIds = pendingPermissionRequestIds.join('\0'); const laneEvidenceNeedsUpdate = Boolean( lane?.result && laneEvidence && (laneEvidence.pendingPermissionRequestIds?.join('\0') !== joinedPendingPermissionRequestIds || laneEvidence.runtimeDiagnostic !== 'OpenCode runtime is waiting for permission approval' || laneEvidence.runtimeDiagnosticSeverity !== 'warning') ); const next: MemberSpawnStatusEntry = { ...prev, status: prev.bootstrapConfirmed || laneEvidence?.bootstrapConfirmed ? 'online' : 'waiting', launchState: prev.launchState, agentToolAccepted: true, runtimeAlive: prev.runtimeAlive === true || laneEvidence?.runtimeAlive === true, bootstrapConfirmed: prev.bootstrapConfirmed === true || laneEvidence?.bootstrapConfirmed === true, hardFailure: false, hardFailureReason: undefined, error: undefined, pendingPermissionRequestIds, livenessKind: prev.livenessKind ?? laneEvidence?.livenessKind ?? 'permission_blocked', runtimeDiagnostic: 'OpenCode runtime is waiting for permission approval', runtimeDiagnosticSeverity: 'warning', updatedAt, }; next.launchState = deriveMemberLaunchState(next); if ( prev.pendingPermissionRequestIds?.join('\0') === joinedPendingPermissionRequestIds && prev.launchState === next.launchState && prev.runtimeDiagnostic === next.runtimeDiagnostic && !laneEvidenceNeedsUpdate ) { continue; } run.memberSpawnStatuses.set(memberName, next); if (lane?.result && laneEvidence) { lane.result = { ...lane.result, members: { ...lane.result.members, [memberName]: { ...laneEvidence, hardFailure: false, hardFailureReason: undefined, pendingPermissionRequestIds, pendingApprovals: [...permissions], pendingPermissions: [...permissions], runtimeDiagnostic: 'OpenCode runtime is waiting for permission approval', runtimeDiagnosticSeverity: 'warning', diagnostics: mergeRuntimeDiagnostics( laneEvidence.diagnostics, ['waiting for permission approval'], laneEvidence.runtimeDiagnostic ) ?? [], }, }, }; } if (this.isCurrentTrackedRun(run)) { this.emitMemberSpawnChange(run, memberName); } } if (run.isLaunch) { void this.persistLaunchStateSnapshot(run, run.provisioningComplete ? 'finished' : 'active'); } } private logOpenCodePromptDeliveryEvent( event: string, record: OpenCodePromptDeliveryLedgerRecord, extra: Record = {} ): void { logger.info( event, JSON.stringify({ teamName: record.teamName, memberName: record.memberName, laneId: record.laneId, runId: record.runId, inboxMessageId: record.inboxMessageId, runtimeSessionId: record.runtimeSessionId, status: record.status, responseState: record.responseState, attempts: record.attempts, nextAttemptAt: record.nextAttemptAt, visibleReplyCorrelation: record.visibleReplyCorrelation, reason: record.lastReason, ...extra, }) ); const shouldNotifyTerminalFailure = event === 'opencode_prompt_delivery_terminal_failure' && record.status === 'failed_terminal'; const shouldNotifyActionRequiredRetry = !shouldNotifyTerminalFailure && isPotentialOpenCodeRuntimeDeliveryError(record); if (shouldNotifyTerminalFailure || shouldNotifyActionRequiredRetry) { void this.handleOpenCodeRuntimeDeliveryUserFacingSideEffects(record).catch((error) => { logger.warn( `[${record.teamName}] Failed to handle OpenCode runtime delivery advisory side effects for ${record.memberName}: ${getErrorMessage(error)}` ); }); } } private emitOpenCodePromptDeliveryTaskLogChange( record: OpenCodePromptDeliveryLedgerRecord, detail: string ): void { if (!record.runtimeSessionId?.trim() || record.taskRefs.length === 0) { return; } const taskIds = new Set( record.taskRefs .map((taskRef) => taskRef.taskId?.trim() || taskRef.displayId?.trim()) .filter((taskId): taskId is string => Boolean(taskId)) ); for (const taskId of taskIds) { this.teamChangeEmitter?.({ type: 'task-log-change', teamName: record.teamName, ...(record.runId ? { runId: record.runId } : {}), taskId, detail, taskSignalKind: 'log', }); } } private async handleOpenCodeRuntimeDeliveryUserFacingSideEffects( record: OpenCodePromptDeliveryLedgerRecord ): Promise { const { record: latestRecord, decision } = await this.decideOpenCodeRuntimeDeliveryUserFacingAdvisory(record); if (decision.action === 'defer') { this.emitOpenCodeRuntimeDeliveryAdvisoryEvent(latestRecord, decision); this.scheduleOpenCodeRuntimeDeliveryAdvisoryReview(latestRecord, decision); return; } if (decision.action === 'suppress') { this.emitOpenCodeRuntimeDeliveryAdvisoryEvent(latestRecord, decision); return; } this.emitOpenCodeRuntimeDeliveryAdvisoryEvent(latestRecord, decision); await this.scheduleOpenCodeProofMissingWorkSyncRecovery(latestRecord, decision); if (decision.severity !== 'error') { return; } await this.fireOpenCodeRuntimeDeliveryErrorNotification(latestRecord, decision); } private async decideOpenCodeRuntimeDeliveryUserFacingAdvisory( record: OpenCodePromptDeliveryLedgerRecord ): Promise<{ record: OpenCodePromptDeliveryLedgerRecord; decision: OpenCodeRuntimeDeliveryAdvisoryDecision; }> { const memberKey = record.memberName.trim().toLowerCase(); let recordsForMember: OpenCodePromptDeliveryLedgerRecord[] = [record]; let ledgerReadSucceeded = false; try { const laneRecords = await this.createOpenCodePromptDeliveryLedger( record.teamName, record.laneId ).list(); ledgerReadSucceeded = true; recordsForMember = laneRecords.filter( (candidate) => candidate.memberName.trim().toLowerCase() === memberKey ); } catch { recordsForMember = [record]; } const latestRecord = recordsForMember.find((candidate) => candidate.id === record.id) ?? null; if (!latestRecord && ledgerReadSucceeded) { return { record, decision: { action: 'suppress' }, }; } const recordForDecision = latestRecord ?? record; const recordsByMember = new Map([ [memberKey, recordsForMember.length > 0 ? recordsForMember : [recordForDecision]], ]); const activeMemberKeys = new Set([memberKey]); const proofIndex = await this.openCodeRuntimeDeliveryProofReader .readProofIndex({ teamName: recordForDecision.teamName, activeMemberKeys, recordsByMember, }) .catch(() => null); return { record: recordForDecision, decision: decideOpenCodeRuntimeDeliveryAdvisory({ record: recordForDecision, proof: proofIndex?.getSnapshot(recordForDecision.memberName, recordForDecision), }), }; } private async fireOpenCodeRuntimeDeliveryErrorNotification( record: OpenCodePromptDeliveryLedgerRecord, decision: OpenCodeRuntimeDeliveryAdvisoryDecision ): Promise { const reason = decision.reason; if (!reason) { return; } const config = await this.readConfigSnapshot(record.teamName).catch(() => null); const teamDisplayName = config?.name?.trim() || record.teamName; const taskLabel = record.taskRefs[0]?.displayId?.trim() ? `#${record.taskRefs[0].displayId.trim()}` : null; const context = taskLabel ? ` while handling ${taskLabel}` : ''; const body = `Team ${teamDisplayName}: @${record.memberName} hit an OpenCode runtime delivery error${context}. ${reason}`; try { await NotificationManager.getInstance().addTeamNotification({ teamEventType: 'api_error', teamName: record.teamName, teamDisplayName, from: record.memberName, summary: taskLabel ? `OpenCode runtime error ${taskLabel}` : 'OpenCode runtime delivery error', body, dedupeKey: `opencode_runtime_delivery_error:${record.teamName}:${record.memberName}:${record.id}`, target: { kind: 'member', teamName: record.teamName, memberName: record.memberName, focus: 'messages', }, projectPath: config?.projectPath, }); } catch (error) { logger.warn( `[${record.teamName}] Failed to store OpenCode runtime delivery error notification for ${record.memberName}: ${getErrorMessage(error)}` ); } await this.notifyLeadAboutOpenCodeRuntimeDeliveryError({ record, reason, taskLabel, }); } private async scheduleOpenCodeProofMissingWorkSyncRecovery( record: OpenCodePromptDeliveryLedgerRecord, decision: OpenCodeRuntimeDeliveryAdvisoryDecision ): Promise { if (decision.reasonCode !== 'protocol_proof_missing') { return; } const scheduler = this.memberWorkSyncProofMissingRecoveryScheduler; if (!scheduler) { return; } try { await scheduler({ teamName: record.teamName, memberName: record.memberName, originalMessageId: record.inboxMessageId, taskRefs: record.taskRefs, ...(decision.reason ? { reason: decision.reason } : {}), }); } catch (error) { logger.warn( `[${record.teamName}] Failed to schedule OpenCode proof-missing work sync recovery for ${record.memberName}: ${getErrorMessage(error)}` ); } } private emitOpenCodeRuntimeDeliveryAdvisoryEvent( record: OpenCodePromptDeliveryLedgerRecord, decision?: OpenCodeRuntimeDeliveryAdvisoryDecision ): void { try { this.memberRuntimeAdvisoryInvalidator?.(record.teamName, record.memberName); } catch (error) { logger.warn( `[${record.teamName}] Failed to invalidate OpenCode runtime advisory cache for ${record.memberName}: ${getErrorMessage(error)}` ); } const reasonKey = this.getOpenCodeRuntimeDeliveryAdvisoryReasonKey(record, decision); const eventKey = `opencode_runtime_delivery_error:${record.teamName}:${record.memberName}:${record.id}:${reasonKey}`; const now = Date.now(); this.pruneOpenCodeRuntimeDeliveryAdvisoryEventDedupe(now); if (this.openCodeRuntimeDeliveryAdvisoryEventSentAt.has(eventKey)) { return; } try { this.teamChangeEmitter?.({ type: 'member-advisory', teamName: record.teamName, detail: `opencode-runtime-delivery-error:${record.memberName}:${record.id}`, }); this.openCodeRuntimeDeliveryAdvisoryEventSentAt.set(eventKey, now); } catch (error) { logger.warn( `[${record.teamName}] Failed to emit member advisory refresh for ${record.memberName}: ${getErrorMessage(error)}` ); } } private emitRuntimeDeliveryReplyAdvisoryRefresh(teamName: string, message: InboxMessage): void { if ( message.source !== 'runtime_delivery' || typeof message.relayOfMessageId !== 'string' || message.relayOfMessageId.trim().length === 0 ) { return; } const memberName = message.from?.trim(); if (!memberName || memberName === 'user' || memberName === 'system') { return; } try { this.memberRuntimeAdvisoryInvalidator?.(teamName, memberName); } catch (error) { logger.warn( `[${teamName}] Failed to invalidate runtime advisory after runtime delivery reply for ${memberName}: ${getErrorMessage(error)}` ); } try { this.teamChangeEmitter?.({ type: 'member-advisory', teamName, detail: `runtime-delivery-reply:${memberName}:${message.relayOfMessageId.trim()}`, }); } catch (error) { logger.warn( `[${teamName}] Failed to emit runtime advisory refresh after runtime delivery reply for ${memberName}: ${getErrorMessage(error)}` ); } } private pruneOpenCodeRuntimeDeliveryAdvisoryEventDedupe(now: number): void { const ttlMs = TeamProvisioningService.OPENCODE_RUNTIME_DELIVERY_ADVISORY_EVENT_TTL_MS; for (const [key, sentAt] of this.openCodeRuntimeDeliveryAdvisoryEventSentAt) { if (now - sentAt > ttlMs) { this.openCodeRuntimeDeliveryAdvisoryEventSentAt.delete(key); } } } private getOpenCodeRuntimeDeliveryAdvisoryReasonKey( record: OpenCodePromptDeliveryLedgerRecord, decision?: OpenCodeRuntimeDeliveryAdvisoryDecision ): string { const reason = decision?.reason ?? selectOpenCodeRuntimeDeliveryReason(record) ?? record.responseState ?? record.status; const action = decision ? `${decision.action}:${decision.severity ?? 'none'}` : 'record:none'; const normalized = reason .toLowerCase() .replace(/https?:\/\/\S+/g, '') .replace(/[^a-z0-9]+/g, '-') .replace(/^-+|-+$/g, '') .slice(0, 96); return `${action}:${normalized || 'unknown'}`; } private scheduleOpenCodeRuntimeDeliveryAdvisoryReview( record: OpenCodePromptDeliveryLedgerRecord, decision: OpenCodeRuntimeDeliveryAdvisoryDecision ): void { const reviewAt = Date.parse(decision.nextReviewAt ?? ''); if (!Number.isFinite(reviewAt)) { return; } const delayMs = Math.max(250, reviewAt - Date.now()); const timerKey = `${record.teamName}:${record.laneId}:${record.id}`; const existing = this.openCodeRuntimeDeliveryAdvisoryReviewTimers.get(timerKey); if (existing) { clearTimeout(existing); } const timer = setTimeout(() => { this.openCodeRuntimeDeliveryAdvisoryReviewTimers.delete(timerKey); void this.handleOpenCodeRuntimeDeliveryUserFacingSideEffects(record).catch((error) => { logger.warn( `[${record.teamName}] Failed to refresh deferred OpenCode runtime delivery advisory for ${record.memberName}: ${getErrorMessage(error)}` ); }); }, delayMs); this.openCodeRuntimeDeliveryAdvisoryReviewTimers.set(timerKey, timer); } private async notifyLeadAboutOpenCodeRuntimeDeliveryError(input: { record: OpenCodePromptDeliveryLedgerRecord; reason: string; taskLabel: string | null; }): Promise { const runId = this.getAliveRunId(input.record.teamName); const run = runId ? this.runs.get(runId) : null; if (!run || run.processKilled || run.cancelRequested) { return; } const noticeKey = `opencode_runtime_delivery_error:${input.record.teamName}:${input.record.memberName}:${input.record.id}`; const now = Date.now(); this.pruneOpenCodeRuntimeDeliveryLeadNoticeDedupe(now); if (this.openCodeRuntimeDeliveryLeadNoticeSentAt.has(noticeKey)) { return; } this.openCodeRuntimeDeliveryLeadNoticeSentAt.set(noticeKey, now); const taskContext = input.taskLabel ? ` while handling ${input.taskLabel}` : ''; const message = [ `System notice: OpenCode teammate @${input.record.memberName} hit a runtime delivery error${taskContext}.`, `Reason: ${input.reason}`, `Treat @${input.record.memberName} as unavailable for that work until retry or restart succeeds.`, `Do not message the human user solely because of this notice unless user action is required.`, ].join(' '); try { await this.sendMessageToRun(run, message); } catch (error) { this.openCodeRuntimeDeliveryLeadNoticeSentAt.delete(noticeKey); logger.warn( `[${input.record.teamName}] Failed to notify lead about OpenCode runtime delivery error for ${input.record.memberName}: ${getErrorMessage(error)}` ); } } private pruneOpenCodeRuntimeDeliveryLeadNoticeDedupe(now: number): void { const ttlMs = TeamProvisioningService.OPENCODE_RUNTIME_DELIVERY_LEAD_NOTICE_TTL_MS; for (const [key, sentAt] of this.openCodeRuntimeDeliveryLeadNoticeSentAt) { if (now - sentAt > ttlMs) { this.openCodeRuntimeDeliveryLeadNoticeSentAt.delete(key); } } } async scanOpenCodePromptDeliveryWatchdog(teamName: string): Promise { if (!this.isOpenCodePromptDeliveryWatchdogEnabled()) { return 0; } const canDeliverToTeamRuntime = this.canDeliverToOpenCodeRuntimeForTeam(teamName); const recoveredLaneIds = await this.tryRecoverOpenCodeRuntimeLanesForDeliveryWatchdog( teamName, { allowCommittedSessionRecoveryWithoutTeamRuntime: !canDeliverToTeamRuntime } ); if (!canDeliverToTeamRuntime && recoveredLaneIds.length === 0) { await this.stopOpenCodeRuntimeLanesForStoppedTeam(teamName); return 0; } const laneIndex = await readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName).catch( () => null ); if (!laneIndex && recoveredLaneIds.length === 0) { return 0; } let activeLaneIds = canDeliverToTeamRuntime ? Object.values(laneIndex?.lanes ?? {}) .filter((lane) => lane.state === 'active') .map((lane) => lane.laneId) : []; activeLaneIds = [...new Set([...activeLaneIds, ...recoveredLaneIds])]; return await this.scanOpenCodePromptDeliveryWatchdogForActiveLanes(teamName, activeLaneIds); } private async scanOpenCodePromptDeliveryWatchdogForActiveLanes( teamName: string, laneIds: string[] ): Promise { if (!this.isOpenCodePromptDeliveryWatchdogEnabled()) { return 0; } let scheduled = 0; for (const laneId of [...new Set(laneIds.map((laneId) => laneId.trim()).filter(Boolean))]) { const ledger = this.createOpenCodePromptDeliveryLedger(teamName, laneId); await ledger.pruneTerminalRecords({ now: new Date() }).catch((error: unknown) => { logger.warn( `[${teamName}] OpenCode prompt delivery ledger prune failed for ${laneId}: ${getErrorMessage(error)}` ); }); const records = await ledger.list().catch(() => []); for (const record of records) { if (this.isOpenCodePromptDeliveryWatchdogRecordTerminal(record)) { continue; } const nextAttemptMs = record.nextAttemptAt ? Date.parse(record.nextAttemptAt) : NaN; const delayMs = Number.isFinite(nextAttemptMs) ? Math.max(500, nextAttemptMs - Date.now()) : OPENCODE_PROMPT_DELIVERY_OBSERVE_DELAY_MS; this.scheduleOpenCodePromptDeliveryWatchdog({ teamName, memberName: record.memberName, messageId: record.inboxMessageId, delayMs, }); scheduled += 1; } const members = await this.resolveOpenCodeMembersForRuntimeLane(teamName, laneId); for (const memberName of members) { const inboxMessages = await this.inboxReader .getMessagesFor(teamName, memberName) .catch(() => []); for (const message of inboxMessages) { if ( message.read || typeof message.text !== 'string' || message.text.trim().length === 0 || !this.hasStableMessageId(message) ) { continue; } const existing = await ledger .getByInboxMessage({ teamName, memberName, laneId, inboxMessageId: message.messageId, }) .catch(() => null); if (existing) { continue; } const replyRecipient = typeof message.from === 'string' && message.from.trim() && message.from.trim().toLowerCase() !== memberName.trim().toLowerCase() ? message.from.trim() : 'user'; const now = nowIso(); const record = await ledger.ensurePending({ teamName, memberName, laneId, runId: await this.resolveCurrentOpenCodeRuntimeRunId(teamName, laneId), inboxMessageId: message.messageId, inboxTimestamp: message.timestamp, source: 'watchdog', replyRecipient, actionMode: message.actionMode ?? null, messageKind: message.messageKind ?? null, workSyncIntent: message.workSyncIntent ?? null, taskRefs: message.taskRefs ?? [], payloadHash: hashOpenCodePromptDeliveryPayload({ text: message.text, replyRecipient, actionMode: message.actionMode ?? null, taskRefs: message.taskRefs ?? [], attachments: message.attachments, source: 'watchdog', }), now, }); const recovered = await ledger.markAcceptanceUnknown({ id: record.id, reason: 'opencode_prompt_delivery_ledger_rebuilt_from_unread_inbox', nextAttemptAt: now, markedAt: now, }); this.logOpenCodePromptDeliveryEvent( 'opencode_prompt_delivery_retry_scheduled', recovered, { acceptanceUnknown: true, reason: recovered.lastReason } ); this.scheduleOpenCodePromptDeliveryWatchdog({ teamName, memberName: recovered.memberName, messageId: recovered.inboxMessageId, delayMs: 500, }); scheduled += 1; } } } return scheduled; } async deliverOpenCodeMemberMessage( teamName: string, input: { memberName: string; text: string; messageId?: string; replyRecipient?: string; actionMode?: AgentActionMode; messageKind?: InboxMessage['messageKind']; workSyncIntent?: InboxMessage['workSyncIntent']; workSyncReviewRequestEventIds?: string[]; taskRefs?: TaskRef[]; attachments?: AttachmentPayload[]; source?: OpenCodeMemberInboxRelayOptions['source']; inboxTimestamp?: string; } ): Promise { const adapter = this.getOpenCodeRuntimeMessageAdapter(); if (!adapter) { return { delivered: false, reason: 'opencode_runtime_message_bridge_unavailable' }; } const directory = await this.readOpenCodeMemberDirectory(teamName); const identity = this.resolveOpenCodeMemberIdentityFromDirectory( teamName, input.memberName, directory ); if (!identity.ok) { return { delivered: false, reason: identity.reason === 'opencode_recipient_unavailable' ? 'recipient_is_not_opencode' : identity.reason, }; } const { config } = directory; const { canonicalMemberName, laneIdentity, configMember, metaMember, memberRuntimeCwd } = identity; const normalizedMemberName = input.memberName.trim(); if ( laneIdentity.laneKind === 'secondary' && laneIdentity.laneOwnerProviderId === 'opencode' && this.stoppingSecondaryRuntimeTeams.has(teamName) ) { return { delivered: false, reason: 'opencode_runtime_not_active' }; } const cwd = laneIdentity.laneKind === 'secondary' && laneIdentity.laneOwnerProviderId === 'opencode' ? memberRuntimeCwd || config?.projectPath?.trim() || this.readPersistedTeamProjectPath(teamName) : config?.projectPath?.trim() || memberRuntimeCwd || this.readPersistedTeamProjectPath(teamName); if (!cwd) { return { delivered: false, reason: 'opencode_project_path_unavailable' }; } const trackedRunId = this.resolveDeliverableTrackedRuntimeRunId(teamName); const trackedRun = trackedRunId ? this.runs.get(trackedRunId) : null; let liveSecondaryLaneRunId: string | null = null; let trackedSecondaryLanePresent = false; let trackedSecondaryLaneSnapshotKnown = false; if ( trackedRun && laneIdentity.laneKind === 'secondary' && laneIdentity.laneOwnerProviderId === 'opencode' ) { const secondaryLanes = trackedRun.mixedSecondaryLanes; trackedSecondaryLaneSnapshotKnown = secondaryLanes.length > 0; const liveLane = secondaryLanes.find( (lane) => lane.laneId === laneIdentity.laneId || lane.member.name.trim().toLowerCase() === normalizedMemberName.toLowerCase() ); trackedSecondaryLanePresent = liveLane != null; liveSecondaryLaneRunId = liveLane?.runId?.trim() || null; if (!liveLane && trackedSecondaryLaneSnapshotKnown) { return { delivered: false, reason: 'opencode_runtime_not_active' }; } } const inMemorySecondaryLaneRunId = laneIdentity.laneKind === 'secondary' && laneIdentity.laneOwnerProviderId === 'opencode' ? this.getCurrentOpenCodeRuntimeRunId(teamName, laneIdentity.laneId) : null; let runtimeRunId = laneIdentity.laneKind === 'secondary' && laneIdentity.laneOwnerProviderId === 'opencode' ? (liveSecondaryLaneRunId ?? inMemorySecondaryLaneRunId ?? (await this.resolveCurrentOpenCodeRuntimeRunId(teamName, laneIdentity.laneId))) : (trackedRunId ?? (await this.resolveCurrentOpenCodeRuntimeRunId(teamName, laneIdentity.laneId))); let runtimeActive = Boolean(runtimeRunId); if (!runtimeActive) { if ( trackedRun && laneIdentity.laneKind === 'secondary' && laneIdentity.laneOwnerProviderId === 'opencode' && !trackedSecondaryLanePresent && trackedSecondaryLaneSnapshotKnown ) { return { delivered: false, reason: 'opencode_runtime_not_active' }; } runtimeActive = await this.isOpenCodeRuntimeLaneIndexActive(teamName, laneIdentity.laneId); } if ( !runtimeActive && laneIdentity.laneKind === 'secondary' && laneIdentity.laneOwnerProviderId === 'opencode' ) { let recovered = await this.tryRecoverOpenCodeRuntimeLaneBeforeDelivery({ teamName, laneId: laneIdentity.laneId, member: { ...(configMember ?? {}), ...(metaMember ?? {}), name: canonicalMemberName, providerId: 'opencode', model: metaMember?.model ?? configMember?.model, role: metaMember?.role ?? configMember?.role, workflow: metaMember?.workflow ?? configMember?.workflow, effort: metaMember?.effort ?? configMember?.effort, cwd: memberRuntimeCwd || undefined, isolation: metaMember?.isolation ?? configMember?.isolation, }, projectPath: config?.projectPath?.trim() || this.readPersistedTeamProjectPath(teamName), }); if (!recovered) { recovered = await this.tryRecoverOpenCodeRuntimeLaneFromCommittedSessionBeforeDelivery({ teamName, laneId: laneIdentity.laneId, member: { ...(configMember ?? {}), ...(metaMember ?? {}), name: canonicalMemberName, providerId: 'opencode', model: metaMember?.model ?? configMember?.model, role: metaMember?.role ?? configMember?.role, workflow: metaMember?.workflow ?? configMember?.workflow, effort: metaMember?.effort ?? configMember?.effort, cwd: memberRuntimeCwd || undefined, isolation: metaMember?.isolation ?? configMember?.isolation, }, projectPath: config?.projectPath?.trim() || this.readPersistedTeamProjectPath(teamName), }); } if (recovered) { runtimeRunId = await this.resolveCurrentOpenCodeRuntimeRunId(teamName, laneIdentity.laneId); runtimeActive = await this.isOpenCodeRuntimeLaneIndexActive(teamName, laneIdentity.laneId); } } if ( runtimeActive && runtimeRunId && laneIdentity.laneKind === 'secondary' && laneIdentity.laneOwnerProviderId === 'opencode' && !liveSecondaryLaneRunId && !inMemorySecondaryLaneRunId ) { const laneStorage = await inspectOpenCodeRuntimeLaneStorage({ teamsBasePath: getTeamsBasePath(), teamName, laneId: laneIdentity.laneId, }); const staleLane = await recoverStaleOpenCodeRuntimeLaneIndexEntry({ teamsBasePath: getTeamsBasePath(), teamName, laneId: laneIdentity.laneId, }); if (!laneStorage.hasRuntimeEvidenceOnDisk) { if (staleLane.stale) { this.deleteSecondaryRuntimeRun(teamName, laneIdentity.laneId); } return { delivered: false, reason: 'opencode_runtime_not_active', diagnostics: staleLane.diagnostics.length ? staleLane.diagnostics : [ `OpenCode runtime bootstrap evidence is not ready for ${canonicalMemberName}. Message was saved and will be retried after runtime check-in.`, ], }; } } if (!runtimeActive) { this.cleanupStoppedTeamOpenCodeRuntimeLanesInBackground(teamName); return { delivered: false, reason: 'opencode_runtime_not_active' }; } let legacyOpenCodeBootstrapSessionToStamp: OpenCodeCommittedBootstrapSessionRecord | null = null; let refreshedOpenCodeBootstrapSessionToStamp: OpenCodeCommittedBootstrapSessionRecord | null = null; let forceOpenCodeSessionRefreshReason: string | undefined; if (laneIdentity.laneOwnerProviderId === 'opencode') { const bootstrapSession = await this.findDeliverableOpenCodeRuntimeBootstrapSessionEvidence({ teamName, runId: runtimeRunId, laneId: laneIdentity.laneId, memberName: canonicalMemberName, }); if (!bootstrapSession) { if (laneIdentity.laneKind === 'secondary') { return { delivered: false, reason: 'opencode_runtime_not_active', diagnostics: [ `OpenCode runtime bootstrap is not confirmed for ${canonicalMemberName}. Message was saved and will be retried after runtime check-in.`, ], }; } } else { if (!bootstrapSession.appMcpTransportHash?.trim()) { legacyOpenCodeBootstrapSessionToStamp = bootstrapSession; } const appMcpTransportMismatch = this.getOpenCodeAppMcpTransportMismatchDiagnostic(bootstrapSession); if (appMcpTransportMismatch) { refreshedOpenCodeBootstrapSessionToStamp = bootstrapSession; forceOpenCodeSessionRefreshReason = appMcpTransportMismatch; logger.info( `[${teamName}] OpenCode delivery detected stale app MCP transport for ${canonicalMemberName}; requesting bridge session refresh before send. ${appMcpTransportMismatch}` ); } } } let openCodeFileParts: OpenCodeFilePart[] = []; if (input.attachments?.length && laneIdentity.laneOwnerProviderId === 'opencode') { try { openCodeFileParts = buildOpenCodeAttachmentDeliveryParts({ text: input.text, model: metaMember?.model ?? configMember?.model ?? '', attachments: input.attachments, }).fileParts; } catch (error) { const reason = error instanceof AgentAttachmentError ? error.code : 'opencode_attachment_delivery_prepare_failed'; const diagnostic = `opencode_attachment_delivery_prepare_failed: ${getErrorMessage(error)}`; const userVisibleMessage = error instanceof AgentAttachmentError ? error.message : 'OpenCode could not prepare the attachment for live delivery.'; return { delivered: false, reason, diagnostics: [diagnostic], userVisibleImpact: { state: 'error', reasonCode: 'backend_error', message: userVisibleMessage, }, }; } } if (!this.isOpenCodePromptDeliveryWatchdogEnabled()) { const controlUrl = input.messageKind === 'member_work_sync_nudge' ? await this.resolveControlApiBaseUrl() : null; const result = await adapter.sendMessageToMember({ ...(runtimeRunId ? { runId: runtimeRunId } : {}), teamName, laneId: laneIdentity.laneId, memberName: canonicalMemberName, cwd, text: input.text, messageId: input.messageId, fileParts: openCodeFileParts, replyRecipient: input.replyRecipient, actionMode: input.actionMode, messageKind: input.messageKind, workSyncIntent: input.workSyncIntent, workSyncReviewRequestEventIds: input.workSyncReviewRequestEventIds, controlUrl: controlUrl ?? undefined, taskRefs: input.taskRefs, forceSessionRefreshReason: forceOpenCodeSessionRefreshReason, }); await this.rememberOpenCodeRuntimePidFromBridge({ teamName, memberName: canonicalMemberName, laneId: laneIdentity.laneId, runId: runtimeRunId, runtimeSessionId: result.sessionId, runtimePid: result.runtimePid, reason: 'opencode_delivery_runtime_pid_observed', }); if (result.ok && legacyOpenCodeBootstrapSessionToStamp) { await this.stampOpenCodeAppMcpTransportEvidenceIfMissing( legacyOpenCodeBootstrapSessionToStamp ); } if (result.ok && result.sessionId && refreshedOpenCodeBootstrapSessionToStamp) { await this.stampOpenCodeAppMcpTransportEvidenceIfMissing( refreshedOpenCodeBootstrapSessionToStamp, { overwriteExistingHash: true, runtimeSessionId: result.sessionId, } ); } const responseObservation = this.normalizeOpenCodeDeliveryResponseObservation( result.responseObservation ); await this.maybeSyncOpenCodeRuntimePermissionsAfterDelivery({ teamName, runId: runtimeRunId, laneId: laneIdentity.laneId, memberName: canonicalMemberName, cwd, sessionId: result.sessionId, responseState: responseObservation?.state, reason: responseObservation?.reason ?? result.diagnostics[0], diagnostics: result.diagnostics, teamColor: config?.color, teamDisplayName: config?.name, }); const legacyWorkSyncReadAllowed = input.messageKind === 'member_work_sync_nudge' && result.ok ? await this.isLegacyOpenCodeMemberWorkSyncReadCommitAllowed({ teamName, memberName: canonicalMemberName, workSyncIntent: input.workSyncIntent, responseObservation, }) : true; const legacyWorkSyncResponsePending = result.ok && input.messageKind === 'member_work_sync_nudge' && !legacyWorkSyncReadAllowed; return { delivered: result.ok, accepted: result.ok, responsePending: legacyWorkSyncResponsePending, responseState: responseObservation?.state, ...(legacyWorkSyncResponsePending ? { reason: responseObservation?.reason ?? 'member_work_sync_report_required' } : result.ok ? {} : { reason: result.diagnostics[0] ?? 'opencode_message_delivery_failed' }), diagnostics: result.diagnostics, }; } const messageId = input.messageId?.trim(); const ledger = messageId && input.source ? this.createOpenCodePromptDeliveryLedger(teamName, laneIdentity.laneId) : null; const now = nowIso(); let active = ledger ? await ledger.getActiveForMember({ teamName, memberName: canonicalMemberName, laneId: laneIdentity.laneId, }) : null; if (active && active.inboxMessageId !== messageId && ledger) { let proof = await this.applyOpenCodeVisibleDestinationProof({ ledger, ledgerRecord: active, teamName, replyRecipient: active.replyRecipient, memberName: canonicalMemberName, }); active = proof.ledgerRecord; proof = await this.materializeOpenCodePlainTextReplyIfNeeded({ ledger, ledgerRecord: active, teamName, memberName: canonicalMemberName, visibleReply: proof.visibleReply, }); active = proof.ledgerRecord; const activeReadAllowed = await this.isOpenCodeDeliveryResponseReadCommitAllowed({ teamName, memberName: canonicalMemberName, responseState: active.responseState, actionMode: active.actionMode ?? undefined, taskRefs: active.taskRefs, visibleReply: proof.visibleReply, ledgerRecord: active, }); if (activeReadAllowed) { this.logOpenCodePromptDeliveryEvent('opencode_prompt_delivery_response_observed', active, { visibleReplySemanticallySufficient: true, unblockedNextDelivery: true, }); active = null; } } if (active && active.inboxMessageId !== messageId) { const activeDueMs = active.nextAttemptAt ? Date.parse(active.nextAttemptAt) : NaN; this.scheduleOpenCodePromptDeliveryWatchdog({ teamName, memberName: canonicalMemberName, messageId: active.inboxMessageId, delayMs: Number.isFinite(activeDueMs) ? Math.max(500, activeDueMs - Date.now()) : OPENCODE_PROMPT_DELIVERY_OBSERVE_DELAY_MS, }); return { delivered: true, accepted: false, responsePending: true, responseState: active.responseState, ledgerStatus: active.status, ledgerRecordId: active.id, laneId: laneIdentity.laneId, queuedBehindMessageId: active.inboxMessageId, reason: 'opencode_delivery_response_pending', diagnostics: [`OpenCode delivery is queued behind ${active.inboxMessageId}.`], }; } let ledgerRecord = messageId ? await ledger?.ensurePending({ teamName, memberName: canonicalMemberName, laneId: laneIdentity.laneId, runId: runtimeRunId ?? null, inboxMessageId: messageId, inboxTimestamp: input.inboxTimestamp ?? now, source: input.source ?? 'manual', replyRecipient: input.replyRecipient ?? 'user', actionMode: input.actionMode ?? null, messageKind: input.messageKind ?? null, workSyncIntent: input.workSyncIntent ?? null, taskRefs: input.taskRefs ?? [], payloadHash: hashOpenCodePromptDeliveryPayload({ text: input.text, replyRecipient: input.replyRecipient ?? 'user', actionMode: input.actionMode ?? null, taskRefs: input.taskRefs ?? [], attachments: input.attachments, source: input.source, }), now, }) : null; if (ledgerRecord?.createdAt === now) { this.logOpenCodePromptDeliveryEvent('opencode_prompt_delivery_ledger_created', ledgerRecord); } const deliveryAttemptId = ledgerRecord ? buildOpenCodePromptDeliveryAttemptId(ledgerRecord) : undefined; if (ledgerRecord && ledger && messageId) { let proof = await this.applyOpenCodeVisibleDestinationProof({ ledger, ledgerRecord, teamName, replyRecipient: input.replyRecipient, memberName: canonicalMemberName, }); ledgerRecord = proof.ledgerRecord; proof = await this.materializeOpenCodePlainTextReplyIfNeeded({ ledger, ledgerRecord, teamName, memberName: canonicalMemberName, visibleReply: proof.visibleReply, }); ledgerRecord = proof.ledgerRecord; let readAllowed = await this.isOpenCodeDeliveryResponseReadCommitAllowed({ teamName, memberName: canonicalMemberName, responseState: ledgerRecord.responseState, actionMode: ledgerRecord.actionMode ?? undefined, taskRefs: ledgerRecord.taskRefs, visibleReply: proof.visibleReply, ledgerRecord, }); if (readAllowed) { this.logOpenCodePromptDeliveryEvent( 'opencode_prompt_delivery_response_observed', ledgerRecord, { visibleReplySemanticallySufficient: true } ); return { delivered: true, accepted: true, responsePending: false, responseState: ledgerRecord.responseState, ledgerStatus: ledgerRecord.status, ledgerRecordId: ledgerRecord.id, laneId: laneIdentity.laneId, visibleReplyMessageId: ledgerRecord.visibleReplyMessageId ?? undefined, visibleReplyCorrelation: ledgerRecord.visibleReplyCorrelation ?? undefined, diagnostics: ledgerRecord.diagnostics, }; } ledgerRecord = await this.requeueOpenCodeRuntimeManifestWatermarkDeliveryIfNeeded({ ledger, ledgerRecord, }); if (ledgerRecord.status === 'failed_terminal') { this.logOpenCodePromptDeliveryEvent( 'opencode_prompt_delivery_terminal_failure', ledgerRecord ); return { delivered: false, accepted: false, responsePending: false, responseState: ledgerRecord.responseState, ledgerStatus: ledgerRecord.status, ledgerRecordId: ledgerRecord.id, laneId: laneIdentity.laneId, reason: ledgerRecord.lastReason ?? 'opencode_prompt_delivery_failed_terminal', diagnostics: ledgerRecord.diagnostics, }; } const attemptDue = isOpenCodePromptDeliveryAttemptDue(ledgerRecord); if (ledgerRecord.status !== 'pending' && !attemptDue) { const nextAttemptMs = ledgerRecord.nextAttemptAt ? Date.parse(ledgerRecord.nextAttemptAt) : NaN; this.scheduleOpenCodePromptDeliveryWatchdog({ teamName, memberName: canonicalMemberName, messageId, delayMs: Number.isFinite(nextAttemptMs) ? Math.max(500, nextAttemptMs - Date.now()) : OPENCODE_PROMPT_DELIVERY_OBSERVE_DELAY_MS, }); return { delivered: true, accepted: true, responsePending: true, responseState: ledgerRecord.responseState, ledgerStatus: ledgerRecord.status, ledgerRecordId: ledgerRecord.id, laneId: laneIdentity.laneId, visibleReplyMessageId: ledgerRecord.visibleReplyMessageId ?? undefined, visibleReplyCorrelation: ledgerRecord.visibleReplyCorrelation ?? undefined, reason: ledgerRecord.lastReason ?? 'opencode_delivery_response_pending', diagnostics: ledgerRecord.diagnostics, }; } if (ledgerRecord.status !== 'pending' && !adapter.observeMessageDelivery) { return { delivered: true, accepted: true, responsePending: true, responseState: ledgerRecord.responseState, ledgerStatus: ledgerRecord.status, ledgerRecordId: ledgerRecord.id, laneId: laneIdentity.laneId, reason: 'opencode_delivery_observe_bridge_unavailable', diagnostics: [ ...ledgerRecord.diagnostics, 'OpenCode message delivery observe bridge is unavailable.', ], }; } const retryDueBeforeObserve = isOpenCodePromptDeliveryRetryAttemptDue({ attemptDue, ledgerRecord, }); const retryShouldRefreshSessionBeforeObserve = retryDueBeforeObserve && ledgerRecord.status === 'retry_scheduled' && !this.hasOpenCodeAcceptedRuntimePrompt(ledgerRecord) && this.isOpenCodeSessionRefreshRetryRecord(ledgerRecord, ledgerRecord.lastReason); if ( ledgerRecord.status !== 'pending' && adapter.observeMessageDelivery && !retryShouldRefreshSessionBeforeObserve ) { const observed = await adapter.observeMessageDelivery({ ...(runtimeRunId ? { runId: runtimeRunId } : {}), teamName, laneId: laneIdentity.laneId, memberName: canonicalMemberName, cwd, text: input.text, messageId, replyRecipient: input.replyRecipient, actionMode: input.actionMode, messageKind: input.messageKind, workSyncIntent: input.workSyncIntent, workSyncReviewRequestEventIds: input.workSyncReviewRequestEventIds, taskRefs: input.taskRefs, prePromptCursor: ledgerRecord.prePromptCursor, sessionId: ledgerRecord.runtimeSessionId ?? undefined, runtimePromptMessageId: ledgerRecord.lastRuntimePromptMessageId ?? ledgerRecord.runtimePromptMessageId ?? undefined, }); await this.rememberOpenCodeRuntimePidFromBridge({ teamName, memberName: canonicalMemberName, laneId: laneIdentity.laneId, runId: runtimeRunId, runtimeSessionId: observed.sessionId, runtimePid: observed.runtimePid, reason: 'opencode_delivery_observe_runtime_pid_observed', }); const responseObservation = this.normalizeOpenCodeDeliveryResponseObservation( observed.responseObservation ); await this.maybeSyncOpenCodeRuntimePermissionsAfterDelivery({ teamName, runId: runtimeRunId, laneId: laneIdentity.laneId, memberName: canonicalMemberName, cwd, sessionId: observed.sessionId, responseState: responseObservation?.state, reason: responseObservation?.reason ?? observed.diagnostics[0], diagnostics: observed.diagnostics, teamColor: config?.color, teamDisplayName: config?.name, }); ledgerRecord = await ledger.applyObservation({ id: ledgerRecord.id, responseObservation: responseObservation ?? { state: observed.ok ? 'not_observed' : 'reconcile_failed', deliveredUserMessageId: null, assistantMessageId: null, toolCallNames: [], visibleMessageToolCallId: null, visibleReplyMessageId: null, visibleReplyCorrelation: null, latestAssistantPreview: null, reason: observed.diagnostics[0] ?? null, }, sessionId: observed.sessionId, runtimePromptMessageId: observed.runtimePromptMessageId, diagnostics: observed.diagnostics, observedAt: nowIso(), }); proof = await this.applyOpenCodeVisibleDestinationProof({ ledger, ledgerRecord, teamName, replyRecipient: input.replyRecipient, memberName: canonicalMemberName, }); ledgerRecord = proof.ledgerRecord; proof = await this.materializeOpenCodePlainTextReplyIfNeeded({ ledger, ledgerRecord, teamName, memberName: canonicalMemberName, visibleReply: proof.visibleReply, }); ledgerRecord = proof.ledgerRecord; readAllowed = await this.isOpenCodeDeliveryResponseReadCommitAllowed({ teamName, memberName: canonicalMemberName, responseState: ledgerRecord.responseState, actionMode: ledgerRecord.actionMode ?? undefined, taskRefs: ledgerRecord.taskRefs, visibleReply: proof.visibleReply, ledgerRecord, }); if (readAllowed) { this.logOpenCodePromptDeliveryEvent( 'opencode_prompt_delivery_response_observed', ledgerRecord, { visibleReplySemanticallySufficient: true } ); return { delivered: true, accepted: true, responsePending: false, responseState: ledgerRecord.responseState, ledgerStatus: ledgerRecord.status, ledgerRecordId: ledgerRecord.id, laneId: laneIdentity.laneId, visibleReplyMessageId: ledgerRecord.visibleReplyMessageId ?? undefined, visibleReplyCorrelation: ledgerRecord.visibleReplyCorrelation ?? undefined, diagnostics: ledgerRecord.diagnostics, }; } const pendingReason = this.getOpenCodeDeliveryPendingReason({ responseState: ledgerRecord.responseState, actionMode: ledgerRecord.actionMode, taskRefs: ledgerRecord.taskRefs, visibleReply: proof.visibleReply, ledgerRecord, }); const retryable = this.isOpenCodeDeliveryRetryablePendingResponse({ ledgerRecord, visibleReply: proof.visibleReply, readAllowed, }); const retryDue = retryDueBeforeObserve; if ( retryDue && retryable && this.isOpenCodeSessionRefreshRetryRecord(ledgerRecord, pendingReason) ) { ledgerRecord = await this.scheduleOpenCodePromptLedgerFollowUp({ ledger, ledgerRecord, teamName, memberName: canonicalMemberName, retry: true, reason: pendingReason, }); if (ledgerRecord.status === 'failed_terminal') { return { delivered: false, accepted: true, responsePending: false, responseState: ledgerRecord.responseState, ledgerStatus: ledgerRecord.status, ledgerRecordId: ledgerRecord.id, laneId: laneIdentity.laneId, visibleReplyMessageId: ledgerRecord.visibleReplyMessageId ?? undefined, visibleReplyCorrelation: ledgerRecord.visibleReplyCorrelation ?? undefined, reason: ledgerRecord.lastReason ?? 'opencode_prompt_delivery_failed_terminal', diagnostics: ledgerRecord.diagnostics.length ? ledgerRecord.diagnostics : [ledgerRecord.lastReason ?? 'opencode_prompt_delivery_failed_terminal'], }; } return { delivered: true, accepted: true, responsePending: true, responseState: ledgerRecord.responseState, ledgerStatus: ledgerRecord.status, ledgerRecordId: ledgerRecord.id, laneId: laneIdentity.laneId, visibleReplyMessageId: ledgerRecord.visibleReplyMessageId ?? undefined, visibleReplyCorrelation: ledgerRecord.visibleReplyCorrelation ?? undefined, reason: ledgerRecord.lastReason ?? 'opencode_delivery_response_pending', diagnostics: ledgerRecord.diagnostics, }; } if (!retryDue || !retryable) { ledgerRecord = await this.scheduleOpenCodePromptLedgerFollowUp({ ledger, ledgerRecord, teamName, memberName: canonicalMemberName, retry: retryable, reason: pendingReason, }); if (ledgerRecord.status === 'failed_terminal') { return { delivered: false, accepted: true, responsePending: false, responseState: ledgerRecord.responseState, ledgerStatus: ledgerRecord.status, ledgerRecordId: ledgerRecord.id, laneId: laneIdentity.laneId, visibleReplyMessageId: ledgerRecord.visibleReplyMessageId ?? undefined, visibleReplyCorrelation: ledgerRecord.visibleReplyCorrelation ?? undefined, reason: ledgerRecord.lastReason ?? 'opencode_prompt_delivery_failed_terminal', diagnostics: ledgerRecord.diagnostics.length ? ledgerRecord.diagnostics : [ledgerRecord.lastReason ?? 'opencode_prompt_delivery_failed_terminal'], }; } return { delivered: true, accepted: true, responsePending: true, responseState: ledgerRecord.responseState, ledgerStatus: ledgerRecord.status, ledgerRecordId: ledgerRecord.id, laneId: laneIdentity.laneId, visibleReplyMessageId: ledgerRecord.visibleReplyMessageId ?? undefined, visibleReplyCorrelation: ledgerRecord.visibleReplyCorrelation ?? undefined, reason: ledgerRecord.lastReason ?? 'opencode_delivery_response_pending', diagnostics: ledgerRecord.diagnostics, }; } } } const retryReadAllowed = ledgerRecord ? await this.isOpenCodeDeliveryResponseReadCommitAllowed({ teamName, memberName: canonicalMemberName, responseState: ledgerRecord.responseState, actionMode: ledgerRecord.actionMode ?? undefined, taskRefs: ledgerRecord.taskRefs, visibleReply: null, ledgerRecord, }) : false; const retryPendingReason = ledgerRecord ? this.getOpenCodeDeliveryPendingReason({ responseState: ledgerRecord.responseState, actionMode: ledgerRecord.actionMode, taskRefs: ledgerRecord.taskRefs, visibleReply: null, ledgerRecord, }) : 'opencode_delivery_response_pending'; const controlUrl = input.messageKind === 'member_work_sync_nudge' ? await this.resolveControlApiBaseUrl() : null; if ( !forceOpenCodeSessionRefreshReason && ledgerRecord?.status === 'retry_scheduled' && !this.hasOpenCodeAcceptedRuntimePrompt(ledgerRecord) && isOpenCodePromptDeliveryAttemptDue(ledgerRecord) && this.isOpenCodeSessionRefreshRetryRecord(ledgerRecord, ledgerRecord.lastReason) ) { forceOpenCodeSessionRefreshReason = ledgerRecord.lastSessionRefreshReason ?? ledgerRecord.lastReason ?? ledgerRecord.responseState ?? 'session_stale'; } const deliveryText = this.buildOpenCodePromptDeliveryAttemptText({ text: input.text, controlText: this.buildOpenCodePromptDeliveryRepairControlText({ ledgerRecord, readAllowed: retryReadAllowed, pendingReason: retryPendingReason, controlUrl, }), }); let result: OpenCodeTeamRuntimeMessageResult; try { result = await adapter.sendMessageToMember({ ...(runtimeRunId ? { runId: runtimeRunId } : {}), teamName, laneId: laneIdentity.laneId, memberName: canonicalMemberName, cwd, text: deliveryText, messageId: input.messageId, deliveryAttemptId, fileParts: openCodeFileParts, replyRecipient: input.replyRecipient, actionMode: input.actionMode, messageKind: input.messageKind, workSyncIntent: input.workSyncIntent, workSyncReviewRequestEventIds: input.workSyncReviewRequestEventIds, controlUrl: controlUrl ?? undefined, taskRefs: input.taskRefs, forceSessionRefreshReason: forceOpenCodeSessionRefreshReason, }); } catch (error) { const diagnostic = `opencode_message_delivery_exception: ${getErrorMessage(error)}`; await this.maybeSyncOpenCodeRuntimePermissionsAfterDelivery({ teamName, runId: runtimeRunId, laneId: laneIdentity.laneId, memberName: canonicalMemberName, cwd, reason: diagnostic, diagnostics: [diagnostic], teamColor: config?.color, teamDisplayName: config?.name, }); if (ledgerRecord && ledger) { ledgerRecord = await ledger.applyDeliveryResult({ id: ledgerRecord.id, accepted: false, attempted: true, responseObservation: { state: 'reconcile_failed', deliveredUserMessageId: null, assistantMessageId: null, toolCallNames: [], visibleMessageToolCallId: null, visibleReplyMessageId: null, visibleReplyCorrelation: null, latestAssistantPreview: null, reason: diagnostic, }, deliveryAttemptId, prePromptCursor: ledgerRecord.prePromptCursor, diagnostics: [diagnostic], reason: diagnostic, now: nowIso(), }); this.emitOpenCodePromptDeliveryTaskLogChange( ledgerRecord, 'opencode-prompt-delivery-send-exception' ); ledgerRecord = await this.scheduleOpenCodePromptLedgerFollowUp({ ledger, ledgerRecord, teamName, memberName: canonicalMemberName, retry: true, reason: diagnostic, }); return { delivered: false, accepted: false, responsePending: true, responseState: ledgerRecord.responseState, ledgerStatus: ledgerRecord.status, ledgerRecordId: ledgerRecord.id, laneId: laneIdentity.laneId, reason: diagnostic, diagnostics: ledgerRecord.diagnostics.length ? ledgerRecord.diagnostics : [diagnostic], }; } return { delivered: false, accepted: false, responsePending: false, reason: diagnostic, diagnostics: [diagnostic], }; } await this.rememberOpenCodeRuntimePidFromBridge({ teamName, memberName: canonicalMemberName, laneId: laneIdentity.laneId, runId: runtimeRunId, runtimeSessionId: result.sessionId, runtimePid: result.runtimePid, reason: 'opencode_delivery_runtime_pid_observed', }); if (result.ok && legacyOpenCodeBootstrapSessionToStamp) { await this.stampOpenCodeAppMcpTransportEvidenceIfMissing( legacyOpenCodeBootstrapSessionToStamp ); } if (result.ok && result.sessionId && refreshedOpenCodeBootstrapSessionToStamp) { await this.stampOpenCodeAppMcpTransportEvidenceIfMissing( refreshedOpenCodeBootstrapSessionToStamp, { overwriteExistingHash: true, runtimeSessionId: result.sessionId, } ); } const responseObservation = this.normalizeOpenCodeDeliveryResponseObservation( result.responseObservation ); await this.maybeSyncOpenCodeRuntimePermissionsAfterDelivery({ teamName, runId: runtimeRunId, laneId: laneIdentity.laneId, memberName: canonicalMemberName, cwd, sessionId: result.sessionId, responseState: responseObservation?.state, reason: responseObservation?.reason ?? result.diagnostics[0], diagnostics: result.diagnostics, teamColor: config?.color, teamDisplayName: config?.name, }); const promptAcceptedByRuntimeIdentity = Boolean( result.ok && result.runtimePromptMessageId?.trim() ); const promptAcceptedByObservation = this.isOpenCodePromptAcceptedByObservation(responseObservation); const promptAccepted = promptAcceptedByRuntimeIdentity || promptAcceptedByObservation; const promptAcceptanceMissingRuntimePromptId = result.ok && !promptAcceptedByRuntimeIdentity && !promptAcceptedByObservation; const deliveryDiagnostics = promptAcceptanceMissingRuntimePromptId ? [...result.diagnostics, 'opencode_prompt_acceptance_missing_runtime_prompt_id'] : result.diagnostics; if (ledgerRecord && ledger) { ledgerRecord = await ledger.applyDeliveryResult({ id: ledgerRecord.id, accepted: promptAccepted, attempted: true, responseObservation, sessionId: result.sessionId, runtimePromptMessageId: result.runtimePromptMessageId, deliveryAttemptId, prePromptCursor: result.prePromptCursor, diagnostics: deliveryDiagnostics, reason: promptAccepted ? responseObservation?.reason : deliveryDiagnostics[0], now: nowIso(), }); this.emitOpenCodePromptDeliveryTaskLogChange( ledgerRecord, 'opencode-prompt-delivery-session-evidence' ); let proof = await this.applyOpenCodeVisibleDestinationProof({ ledger, ledgerRecord, teamName, replyRecipient: input.replyRecipient, memberName: canonicalMemberName, }); ledgerRecord = proof.ledgerRecord; proof = await this.materializeOpenCodePlainTextReplyIfNeeded({ ledger, ledgerRecord, teamName, memberName: canonicalMemberName, visibleReply: proof.visibleReply, }); ledgerRecord = proof.ledgerRecord; proof = await this.observeOpenCodeDirectUserDeliveryInlineIfNeeded({ adapter, ledger, ledgerRecord, teamName, memberName: canonicalMemberName, laneId: laneIdentity.laneId, cwd, text: input.text, messageId: ledgerRecord.inboxMessageId, runtimeRunId, replyRecipient: input.replyRecipient, actionMode: input.actionMode, messageKind: input.messageKind, workSyncIntent: input.workSyncIntent, workSyncReviewRequestEventIds: input.workSyncReviewRequestEventIds, taskRefs: input.taskRefs, promptAccepted, visibleReply: proof.visibleReply, }); ledgerRecord = proof.ledgerRecord; this.logOpenCodePromptDeliveryEvent( promptAccepted ? ledgerRecord.status === 'unanswered' ? 'opencode_prompt_delivery_unanswered' : ledgerRecord.status === 'responded' ? 'opencode_prompt_delivery_response_observed' : 'opencode_prompt_delivery_prompt_accepted' : 'opencode_prompt_delivery_retry_scheduled', ledgerRecord, { accepted: promptAccepted, reason: ledgerRecord.lastReason ?? deliveryDiagnostics[0] ?? null, } ); } const responseState = ledgerRecord?.responseState ?? responseObservation?.state; const visibleReply = ledgerRecord ? await this.findOpenCodeVisibleReplyByRelayOfMessageId({ teamName, replyRecipient: input.replyRecipient ?? ledgerRecord.replyRecipient, from: canonicalMemberName, relayOfMessageId: ledgerRecord.inboxMessageId, expectedMessageId: ledgerRecord.visibleReplyCorrelation === 'relayOfMessageId' ? ledgerRecord.visibleReplyMessageId : null, allowUserFallbackForLeadRecipient: ledgerRecord.visibleReplyCorrelation === 'relayOfMessageId', }) : null; const readAllowed = await this.isOpenCodeDeliveryResponseReadCommitAllowed({ teamName, memberName: canonicalMemberName, responseState, actionMode: input.actionMode, taskRefs: input.taskRefs, visibleReply, ledgerRecord, }); if (ledgerRecord && promptAccepted && !readAllowed) { const retry = this.isOpenCodeDeliveryRetryablePendingResponse({ ledgerRecord, visibleReply, readAllowed, }); ledgerRecord = await this.scheduleOpenCodePromptLedgerFollowUp({ ledger: ledger!, ledgerRecord, teamName, memberName: canonicalMemberName, retry, reason: this.getOpenCodeDeliveryPendingReason({ responseState: ledgerRecord.responseState, actionMode: ledgerRecord.actionMode, taskRefs: ledgerRecord.taskRefs, visibleReply, ledgerRecord, }), }); if (ledgerRecord.status === 'failed_terminal') { return { delivered: false, accepted: true, responsePending: false, responseState: ledgerRecord.responseState, ledgerStatus: ledgerRecord.status, ledgerRecordId: ledgerRecord.id, laneId: laneIdentity.laneId, reason: ledgerRecord.lastReason ?? 'opencode_prompt_delivery_failed_terminal', diagnostics: ledgerRecord.diagnostics.length ? ledgerRecord.diagnostics : [ledgerRecord.lastReason ?? 'opencode_prompt_delivery_failed_terminal'], }; } } if (ledgerRecord && !promptAccepted) { const reason = promptAcceptanceMissingRuntimePromptId ? 'opencode_prompt_acceptance_unknown_missing_runtime_prompt_id' : this.isOpenCodePromptAcceptanceUnknownFailure(deliveryDiagnostics) ? 'opencode_prompt_acceptance_unknown_after_bridge_timeout' : (deliveryDiagnostics[0] ?? 'opencode_message_delivery_failed'); if ( reason === 'opencode_prompt_acceptance_unknown_after_bridge_timeout' || reason === 'opencode_prompt_acceptance_unknown_missing_runtime_prompt_id' ) { const delayMs = OPENCODE_PROMPT_DELIVERY_OBSERVE_DELAY_MS; ledgerRecord = await ledger!.markAcceptanceUnknown({ id: ledgerRecord.id, reason, nextAttemptAt: new Date(Date.now() + delayMs).toISOString(), diagnostics: deliveryDiagnostics, markedAt: nowIso(), }); this.scheduleOpenCodePromptDeliveryWatchdog({ teamName, memberName: canonicalMemberName, messageId: ledgerRecord.inboxMessageId, delayMs, }); this.logOpenCodePromptDeliveryEvent( 'opencode_prompt_delivery_retry_scheduled', ledgerRecord, { acceptanceUnknown: true, reason } ); } else { ledgerRecord = await this.scheduleOpenCodePromptLedgerFollowUp({ ledger: ledger!, ledgerRecord, teamName, memberName: canonicalMemberName, retry: true, reason, }); } } const responseVisibleReplyMessageId = ledgerRecord?.visibleReplyMessageId ?? responseObservation?.visibleReplyMessageId ?? undefined; const responseVisibleReplyCorrelation = ledgerRecord?.visibleReplyCorrelation ?? responseObservation?.visibleReplyCorrelation ?? undefined; const acceptanceUnknown = Boolean(ledgerRecord?.acceptanceUnknown && !promptAccepted); const responsePending = acceptanceUnknown || (promptAccepted && Boolean(ledgerRecord || responseObservation)) ? !readAllowed : false; const pendingReason = responsePending && ledgerRecord ? (ledgerRecord.lastReason ?? 'opencode_delivery_response_pending') : null; const diagnostics = pendingReason && result.diagnostics.length === 0 ? [pendingReason] : ledgerRecord?.diagnostics.length ? ledgerRecord.diagnostics : result.diagnostics; return { delivered: promptAccepted || acceptanceUnknown, ...(ledgerRecord || responseObservation ? { accepted: promptAccepted } : {}), ...(ledgerRecord || responseObservation ? { responsePending } : {}), ...(acceptanceUnknown ? { acceptanceUnknown: true } : {}), ...(ledgerRecord ? { ledgerStatus: ledgerRecord.status, ledgerRecordId: ledgerRecord.id, laneId: laneIdentity.laneId, } : {}), ...(responseState ? { responseState, ...(responseVisibleReplyMessageId ? { visibleReplyMessageId: responseVisibleReplyMessageId } : {}), ...(responseVisibleReplyCorrelation ? { visibleReplyCorrelation: responseVisibleReplyCorrelation } : {}), } : {}), ...(pendingReason ? { reason: pendingReason } : promptAccepted ? {} : { reason: result.diagnostics[0] ?? 'opencode_message_delivery_failed' }), diagnostics, }; } private shouldRouteOpenCodeToRuntimeAdapter(request: { providerId?: TeamProviderId; members?: readonly { providerId?: TeamProviderId; provider?: TeamProviderId }[]; }): boolean { return isPureOpenCodeProvisioningRequest(request) && this.getOpenCodeRuntimeAdapter() !== null; } private planRuntimeLanesOrThrow( leadProviderId: TeamProviderId | undefined, members: TeamCreateRequest['members'] ): TeamRuntimeLanePlan { return this.runtimeLaneCoordinator.planProvisioningMembers({ leadProviderId, members, hasOpenCodeRuntimeAdapter: this.getOpenCodeRuntimeAdapter() !== null, }); } private createMixedSecondaryLaneStates( plan: TeamRuntimeLanePlan ): MixedSecondaryRuntimeLaneState[] { if (!isMixedOpenCodeSideLanePlan(plan)) { return []; } return plan.sideLanes.map((sideLane) => ({ laneId: sideLane.laneId, providerId: 'opencode', member: { ...sideLane.member, }, runId: null, state: 'queued', result: null, warnings: [], diagnostics: [], })); } private createMixedSecondaryLaneStateForMember( run: Pick, member: TeamCreateRequest['members'][number] ): MixedSecondaryRuntimeLaneState { const laneIdentity = buildPlannedMemberLaneIdentity({ leadProviderId: resolveTeamProviderId(run.request.providerId), member: { name: member.name, providerId: normalizeOptionalTeamProviderId(member.providerId), }, }); if (laneIdentity.laneKind !== 'secondary' || laneIdentity.laneOwnerProviderId !== 'opencode') { throw new Error( `Member "${member.name}" is not eligible for an OpenCode secondary runtime lane` ); } return { laneId: laneIdentity.laneId, providerId: 'opencode', member: { ...member, }, runId: null, state: 'queued', result: null, warnings: [], diagnostics: [], }; } private getMixedSecondaryLaunchPhase(run: ProvisioningRun): PersistedTeamLaunchPhase { return (run.mixedSecondaryLanes ?? []).some( (lane) => (!lane.result && lane.state !== 'finished') || lane.result?.teamLaunchState === 'partial_pending' ) ? 'active' : 'finished'; } private upsertRunAllEffectiveMember( run: ProvisioningRun, member: TeamCreateRequest['members'][number] ): void { const normalizedName = member.name.trim().toLowerCase(); const currentMembers = Array.isArray(run.allEffectiveMembers) ? run.allEffectiveMembers : []; const nextMembers = currentMembers.filter( (candidate) => candidate.name.trim().toLowerCase() !== normalizedName ); nextMembers.push(member); run.allEffectiveMembers = nextMembers; run.request = { ...run.request, members: nextMembers, }; const laneIdentity = buildPlannedMemberLaneIdentity({ leadProviderId: resolveTeamProviderId(run.request.providerId), member: { name: member.name, providerId: normalizeOptionalTeamProviderId(member.providerId), }, }); const currentPrimaryMembers = Array.isArray(run.effectiveMembers) ? run.effectiveMembers : []; const nextPrimaryMembers = currentPrimaryMembers.filter( (candidate) => candidate.name.trim().toLowerCase() !== normalizedName ); const currentExpectedMembers = Array.isArray(run.expectedMembers) ? run.expectedMembers : []; const nextExpectedMembers = currentExpectedMembers.filter( (candidate) => candidate.trim().toLowerCase() !== normalizedName ); if (laneIdentity.laneKind === 'primary') { run.effectiveMembers = [...nextPrimaryMembers, member]; run.expectedMembers = [...nextExpectedMembers, member.name.trim()].filter(Boolean); } else { run.effectiveMembers = nextPrimaryMembers; run.expectedMembers = nextExpectedMembers; } } private removeRunAllEffectiveMember(run: ProvisioningRun, memberName: string): void { const normalizedName = memberName.trim().toLowerCase(); const currentMembers = Array.isArray(run.allEffectiveMembers) ? run.allEffectiveMembers : []; const nextMembers = currentMembers.filter( (candidate) => candidate.name.trim().toLowerCase() !== normalizedName ); run.allEffectiveMembers = nextMembers; run.request = { ...run.request, members: nextMembers, }; const currentPrimaryMembers = Array.isArray(run.effectiveMembers) ? run.effectiveMembers : []; run.effectiveMembers = currentPrimaryMembers.filter( (candidate) => candidate.name.trim().toLowerCase() !== normalizedName ); const currentExpectedMembers = Array.isArray(run.expectedMembers) ? run.expectedMembers : []; run.expectedMembers = currentExpectedMembers.filter( (candidate) => candidate.trim().toLowerCase() !== normalizedName ); } private hasSecondaryRuntimeRuns(teamName: string): boolean { const runs = this.secondaryRuntimeRunByTeam.get(teamName); return Boolean(runs && runs.size > 0); } private getSecondaryRuntimeRuns(teamName: string): { runId: string; providerId: 'opencode'; laneId: string; memberName: string; cwd?: string; }[] { return Array.from(this.secondaryRuntimeRunByTeam.get(teamName)?.values() ?? []); } private setSecondaryRuntimeRun(input: { teamName: string; runId: string; providerId: 'opencode'; laneId: string; memberName: string; cwd?: string; }): void { const runs = this.secondaryRuntimeRunByTeam.get(input.teamName) ?? new Map(); runs.set(input.laneId, { runId: input.runId, providerId: input.providerId, laneId: input.laneId, memberName: input.memberName, cwd: input.cwd, }); this.secondaryRuntimeRunByTeam.set(input.teamName, runs); } private deleteSecondaryRuntimeRun(teamName: string, laneId: string): void { this.clearOpenCodeRuntimeToolApprovals(teamName, { laneId, emitDismiss: true }); const runs = this.secondaryRuntimeRunByTeam.get(teamName); if (!runs) { return; } runs.delete(laneId); if (runs.size === 0) { this.secondaryRuntimeRunByTeam.delete(teamName); } } private clearSecondaryRuntimeRuns(teamName: string): void { this.clearOpenCodeRuntimeToolApprovals(teamName, { emitDismiss: true }); this.secondaryRuntimeRunByTeam.delete(teamName); } private getCurrentOpenCodeRuntimeRunId(teamName: string, laneId: string): string | null { if (laneId === 'primary') { const trackedRunId = this.getTrackedRunId(teamName); const trackedRun = trackedRunId ? this.runs.get(trackedRunId) : null; if (trackedRun && this.shouldRouteOpenCodeToRuntimeAdapter(trackedRun.request)) { return trackedRunId; } if ( trackedRunId && this.provisioningRunByTeam.get(teamName) === trackedRunId && this.runtimeAdapterProgressByRunId.has(trackedRunId) ) { const runtimeProgress = this.runtimeAdapterProgressByRunId.get(trackedRunId); if (runtimeProgress && this.isCancellableRuntimeAdapterProgress(runtimeProgress)) { return trackedRunId; } } const runtimeRun = this.runtimeAdapterRunByTeam.get(teamName); if (runtimeRun?.providerId === 'opencode') { return runtimeRun.runId; } return null; } const secondaryLaneRun = this.secondaryRuntimeRunByTeam.get(teamName)?.get(laneId); return secondaryLaneRun?.runId ?? null; } private async resolveCurrentOpenCodeRuntimeRunId( teamName: string, laneId: string ): Promise { const inMemoryRunId = this.getCurrentOpenCodeRuntimeRunId(teamName, laneId); if (inMemoryRunId) { return inMemoryRunId; } const laneIndex = await readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName).catch( () => null ); if (laneIndex?.lanes[laneId]?.state !== 'active') { return null; } const evidence = await new OpenCodeRuntimeManifestEvidenceReader({ teamsBasePath: getTeamsBasePath(), }) .read(teamName, laneId) .catch(() => null); const durableRunId = evidence?.activeRunId?.trim(); return durableRunId || null; } private async resolveOpenCodeMemberDeliveryIdentity( teamName: string, memberName: string ): Promise< | { ok: true; canonicalMemberName: string; laneId: string; } | { ok: false; reason: | 'recipient_is_not_opencode' | 'recipient_removed' | 'opencode_recipient_unavailable'; } > { const directory = await this.readOpenCodeMemberDirectory(teamName); const laneIdentity = this.resolveOpenCodeMemberIdentityFromDirectory( teamName, memberName, directory ); if (!laneIdentity.ok) { return laneIdentity; } return { ok: true, canonicalMemberName: laneIdentity.canonicalMemberName, laneId: laneIdentity.laneId, }; } private async resolveOpenCodeMembersForRuntimeLane( teamName: string, laneId: string ): Promise { const directory = await this.readOpenCodeMemberDirectory(teamName); const names = new Set(); for (const member of directory.config?.members ?? []) { if (member.name?.trim()) { names.add(member.name.trim()); } } for (const member of directory.metaMembers) { if (member.name?.trim()) { names.add(member.name.trim()); } } const resolved: string[] = []; for (const name of names) { const identity = this.resolveOpenCodeMemberIdentityFromDirectory(teamName, name, directory); if (identity.ok && identity.laneId === laneId) { resolved.push(identity.canonicalMemberName); } } if (resolved.length > 0) { return [...new Set(resolved)]; } const secondaryMatch = /^secondary:opencode:(.+)$/i.exec(laneId); const fallbackMember = secondaryMatch?.[1]?.trim(); return fallbackMember ? [fallbackMember] : []; } private async isOpenCodeRuntimeLaneIndexActive( teamName: string, laneId: string ): Promise { const laneIndex = await readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName).catch( () => null ); return laneIndex?.lanes[laneId]?.state === 'active'; } private async tryRecoverOpenCodeRuntimeLaneBeforeDelivery(input: { teamName: string; laneId: string; member: TeamMember; projectPath: string | null; }): Promise { if (!this.canDeliverToOpenCodeRuntimeForTeam(input.teamName)) { this.cleanupStoppedTeamOpenCodeRuntimeLanesInBackground(input.teamName); return false; } const snapshot = await this.launchStateStore.read(input.teamName).catch(() => null); const persistedMember = snapshot?.members?.[input.member.name] ?? Object.values(snapshot?.members ?? {}).find((member) => member.laneId === input.laneId); if (!persistedMember || !isRecoverablePersistedOpenCodeRuntimeCandidate(persistedMember)) { return false; } const runtimeEvidence = await this.tryRecoverMissingOpenCodeSecondaryLaneFromRuntime({ teamName: input.teamName, laneId: input.laneId, member: input.member, projectPath: input.projectPath, previousLaunchState: snapshot, persistedMember, }); if (!runtimeEvidence) { return false; } logger.info( `[${input.teamName}] Recovered OpenCode lane ${input.laneId} before message delivery.` ); return true; } private async tryRecoverOpenCodeRuntimeLaneFromCommittedSessionBeforeDelivery(input: { teamName: string; laneId: string; member: TeamMember; projectPath: string | null; previousLaunchState?: PersistedTeamLaunchSnapshot | null; }): Promise { if (!this.canAttemptCommittedOpenCodeSessionRecovery(input.teamName)) { this.cleanupStoppedTeamOpenCodeRuntimeLanesInBackground(input.teamName); return false; } const currentLaneIndex = await readOpenCodeRuntimeLaneIndex( getTeamsBasePath(), input.teamName ).catch(() => null); const currentEntry = currentLaneIndex?.lanes[input.laneId]; if (currentEntry?.state === 'active') { return true; } if (currentEntry?.state === 'degraded' || currentEntry?.state === 'stopped') { return false; } const committedSessionEvidence = await readCommittedOpenCodeBootstrapSessionEvidence({ teamsBasePath: getTeamsBasePath(), teamName: input.teamName, laneId: input.laneId, }).catch(() => null); if (!committedSessionEvidence?.committed || committedSessionEvidence.sessions.length === 0) { return false; } const expectedMemberName = input.member.name.trim().toLowerCase(); const matchingSession = committedSessionEvidence.sessions.find( (session) => session.memberName.trim().toLowerCase() === expectedMemberName ); if (!matchingSession) { return false; } const runtimeEvidence = await this.tryRecoverActiveOpenCodeSecondaryLaneFromRuntime({ teamName: input.teamName, laneId: input.laneId, member: input.member, projectPath: input.projectPath, previousLaunchState: input.previousLaunchState ?? null, }); if (!isRecoverableOpenCodeRuntimeEvidence(runtimeEvidence)) { return false; } const diagnostics = Array.from( new Set([ 'Recovered missing OpenCode runtime lane index from committed session evidence.', ...committedSessionEvidence.diagnostics, ...(runtimeEvidence.diagnostics ?? []), ]) ); await upsertOpenCodeRuntimeLaneIndexEntry({ teamsBasePath: getTeamsBasePath(), teamName: input.teamName, laneId: input.laneId, state: 'active', diagnostics, }).catch((error: unknown) => { logger.warn( `[${input.teamName}] Failed to recover missing OpenCode lane index ${input.laneId} from committed session evidence: ${getErrorMessage(error)}` ); }); await setOpenCodeRuntimeActiveRunManifest({ teamsBasePath: getTeamsBasePath(), teamName: input.teamName, laneId: input.laneId, runId: committedSessionEvidence.activeRunId ?? matchingSession.runId ?? null, }).catch((error: unknown) => { logger.warn( `[${input.teamName}] Failed to materialize committed-session recovered OpenCode lane manifest ${input.laneId}: ${getErrorMessage(error)}` ); }); logger.info( `[${input.teamName}] Recovered OpenCode lane ${input.laneId} from committed session evidence before message delivery.` ); return true; } private buildOpenCodeRecoveryMember(input: { canonicalMemberName: string; configMember?: TeamMember; metaMember?: TeamMember; }): TeamMember { return { ...(input.configMember ?? {}), ...(input.metaMember ?? {}), name: input.canonicalMemberName, providerId: 'opencode', model: input.metaMember?.model ?? input.configMember?.model, role: input.metaMember?.role ?? input.configMember?.role, workflow: input.metaMember?.workflow ?? input.configMember?.workflow, effort: input.metaMember?.effort ?? input.configMember?.effort, cwd: input.metaMember?.cwd ?? input.configMember?.cwd, isolation: input.metaMember?.isolation ?? input.configMember?.isolation, }; } private async tryRecoverOpenCodeRuntimeLaneForConfiguredMemberBeforeDelivery(input: { teamName: string; memberName: string; }): Promise { const directory = await this.readOpenCodeMemberDirectory(input.teamName).catch(() => null); if (!directory) { return false; } const identity = this.resolveOpenCodeMemberIdentityFromDirectory( input.teamName, input.memberName, directory ); if (!identity.ok) { return false; } const laneIndex = await readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), input.teamName).catch( () => null ); const currentEntry = laneIndex?.lanes[identity.laneId]; if (currentEntry?.state === 'active') { return true; } if (currentEntry?.state === 'degraded' || currentEntry?.state === 'stopped') { return false; } const previousLaunchState = await this.launchStateStore.read(input.teamName).catch(() => null); const projectPath = identity.memberRuntimeCwd ?? directory.config?.projectPath?.trim() ?? this.readPersistedTeamProjectPath(input.teamName); return this.tryRecoverOpenCodeRuntimeLaneFromCommittedSessionBeforeDelivery({ teamName: input.teamName, laneId: identity.laneId, member: this.buildOpenCodeRecoveryMember({ canonicalMemberName: identity.canonicalMemberName, configMember: identity.configMember, metaMember: identity.metaMember, }), projectPath, previousLaunchState, }); } private async tryRecoverOpenCodeRuntimeLaneForConfiguredMemberAndVerifyActive(input: { teamName: string; memberName: string; laneId: string; }): Promise { const recovered = await this.tryRecoverOpenCodeRuntimeLaneForConfiguredMemberBeforeDelivery({ teamName: input.teamName, memberName: input.memberName, }).catch(() => false); if (!recovered) { return false; } return this.isOpenCodeRuntimeLaneIndexActive(input.teamName, input.laneId).catch(() => false); } private async tryRecoverOpenCodeRuntimeLanesForDeliveryWatchdog( teamName: string, options: { allowCommittedSessionRecoveryWithoutTeamRuntime?: boolean } = {} ): Promise { const canDeliverToTeamRuntime = this.canDeliverToOpenCodeRuntimeForTeam(teamName); if (!canDeliverToTeamRuntime && !options.allowCommittedSessionRecoveryWithoutTeamRuntime) { this.cleanupStoppedTeamOpenCodeRuntimeLanesInBackground(teamName); return []; } if (!canDeliverToTeamRuntime && !this.canAttemptCommittedOpenCodeSessionRecovery(teamName)) { this.cleanupStoppedTeamOpenCodeRuntimeLanesInBackground(teamName); return []; } const snapshot = await this.launchStateStore.read(teamName).catch(() => null); const candidates = canDeliverToTeamRuntime ? Object.values(snapshot?.members ?? {}).filter( isRecoverablePersistedOpenCodeRuntimeCandidate ) : []; const [config, teamMeta, metaMembers, currentLaneIndex] = await Promise.all([ this.readConfigForObservation(teamName).catch(() => null), this.teamMetaStore.getMeta(teamName).catch(() => null), this.membersMetaStore.getMembers(teamName).catch(() => []), readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName).catch(() => null), ]); const projectPath = config?.projectPath?.trim() || this.readPersistedTeamProjectPath(teamName); const leadMember = config?.members?.find((member) => isLeadMember(member)); const leadProviderId = normalizeOptionalTeamProviderId(teamMeta?.launchIdentity?.providerId) ?? normalizeOptionalTeamProviderId(teamMeta?.providerId) ?? normalizeOptionalTeamProviderId(leadMember?.providerId); const recoveredLaneIds: string[] = []; for (const persistedMember of candidates) { const memberName = persistedMember.name.trim(); const configMember = config?.members?.find( (member) => member.name?.trim().toLowerCase() === memberName.toLowerCase() ); const metaMember = metaMembers.find( (member) => member.name?.trim().toLowerCase() === memberName.toLowerCase() ); if (metaMember?.removedAt != null || configMember?.removedAt != null) { continue; } const laneIdentity = buildPlannedMemberLaneIdentity({ leadProviderId, member: { name: memberName, providerId: 'opencode', }, }); if (laneIdentity.laneId !== persistedMember.laneId) { continue; } if (currentLaneIndex?.lanes[laneIdentity.laneId]) { continue; } const recovered = await this.tryRecoverOpenCodeRuntimeLaneBeforeDelivery({ teamName, laneId: laneIdentity.laneId, member: { ...(configMember ?? {}), ...(metaMember ?? {}), name: memberName, providerId: 'opencode', model: metaMember?.model ?? configMember?.model ?? persistedMember.model, role: metaMember?.role ?? configMember?.role, workflow: metaMember?.workflow ?? configMember?.workflow, effort: metaMember?.effort ?? configMember?.effort ?? persistedMember.effort, cwd: metaMember?.cwd ?? configMember?.cwd ?? persistedMember.cwd, isolation: metaMember?.isolation ?? configMember?.isolation, }, projectPath, }); if (recovered) { recoveredLaneIds.push(laneIdentity.laneId); } } const directory: OpenCodeMemberDirectory = { config, teamMeta, metaMembers }; const configuredNames = new Set(); for (const member of config?.members ?? []) { if (member.name?.trim()) { configuredNames.add(member.name.trim()); } } for (const member of metaMembers) { if (member.name?.trim()) { configuredNames.add(member.name.trim()); } } for (const memberName of configuredNames) { const identity = this.resolveOpenCodeMemberIdentityFromDirectory( teamName, memberName, directory ); if (!identity.ok) { continue; } if (currentLaneIndex?.lanes[identity.laneId] || recoveredLaneIds.includes(identity.laneId)) { continue; } const recovered = await this.tryRecoverOpenCodeRuntimeLaneFromCommittedSessionBeforeDelivery({ teamName, laneId: identity.laneId, member: this.buildOpenCodeRecoveryMember({ canonicalMemberName: identity.canonicalMemberName, configMember: identity.configMember, metaMember: identity.metaMember, }), projectPath: identity.memberRuntimeCwd ?? config?.projectPath?.trim() ?? this.readPersistedTeamProjectPath(teamName), previousLaunchState: snapshot, }); if (recovered) { recoveredLaneIds.push(identity.laneId); } } return [...new Set(recoveredLaneIds)]; } private async resolveOpenCodeRuntimeLaneId(params: { teamName: string; runId: string; memberName?: string; }): Promise { const runtimeRun = this.runtimeAdapterRunByTeam.get(params.teamName); if (runtimeRun?.providerId === 'opencode' && runtimeRun.runId === params.runId) { return 'primary'; } for (const lane of this.getSecondaryRuntimeRuns(params.teamName)) { if (lane.runId === params.runId) { return lane.laneId; } } if (params.memberName) { const trackedRunId = this.getTrackedRunId(params.teamName); const trackedRun = trackedRunId ? this.runs.get(trackedRunId) : null; const plannedLane = trackedRun?.mixedSecondaryLanes.find( (lane) => lane.member.name.trim() === params.memberName ); if (plannedLane) { return plannedLane.laneId; } const persisted = await this.launchStateStore.read(params.teamName).catch(() => null); const persistedMember = persisted?.members?.[params.memberName]; if ( persistedMember?.laneOwnerProviderId === 'opencode' && typeof persistedMember.laneId === 'string' && persistedMember.laneId.trim().length > 0 ) { return persistedMember.laneId.trim(); } } return 'primary'; } private buildConfiguredProvisioningMember( configuredMember: NonNullable< ReturnType > ): TeamCreateRequest['members'][number] { return { name: configuredMember.name, ...(configuredMember.role ? { role: configuredMember.role } : {}), ...(configuredMember.workflow ? { workflow: configuredMember.workflow } : {}), ...(configuredMember.isolation === 'worktree' ? { isolation: 'worktree' as const } : {}), ...(configuredMember.cwd ? { cwd: configuredMember.cwd } : {}), ...(configuredMember.providerId ? { providerId: configuredMember.providerId } : {}), ...(configuredMember.providerBackendId ? { providerBackendId: configuredMember.providerBackendId } : {}), ...(configuredMember.model ? { model: configuredMember.model } : {}), ...(configuredMember.effort ? { effort: configuredMember.effort } : {}), ...(configuredMember.fastMode ? { fastMode: configuredMember.fastMode } : {}), ...(configuredMember.mcpPolicy ? { mcpPolicy: normalizeTeamMemberMcpPolicy(configuredMember.mcpPolicy) } : {}), }; } private buildPrimaryOwnedMemberSpecForRuntime(input: { configuredMember: NonNullable< ReturnType >; run: ProvisioningRun; }): TeamCreateRequest['members'][number] { const configuredSpec = this.buildConfiguredProvisioningMember(input.configuredMember); const defaultProviderId = resolveTeamProviderId(input.run.request.providerId); const memberProviderId = normalizeTeamMemberProviderId(configuredSpec.providerId); const inheritsDefaultRuntime = memberProviderId == null || memberProviderId === defaultProviderId; const effectiveSpec = buildEffectiveTeamMemberSpec(configuredSpec, { providerId: defaultProviderId, model: input.run.request.model, effort: input.run.request.effort, }); const effectiveProviderId = resolveTeamProviderId(effectiveSpec.providerId); const providerBackendId = migrateProviderBackendId(effectiveProviderId, configuredSpec.providerBackendId) ?? (inheritsDefaultRuntime ? migrateProviderBackendId(effectiveProviderId, input.run.request.providerBackendId) : undefined); const fastMode = configuredSpec.fastMode ?? (inheritsDefaultRuntime ? input.run.request.fastMode : undefined); return { ...effectiveSpec, ...(providerBackendId ? { providerBackendId } : {}), ...(fastMode ? { fastMode } : {}), ...(input.configuredMember.agentType ? { agentType: input.configuredMember.agentType } : {}), }; } private async buildRuntimeBootstrapMemberMcpLaunchConfigs(input: { cwd: string; members: TeamCreateRequest['members']; run: ProvisioningRun; controlApiBaseUrl?: string | null; }): Promise> { const configs = new Map(); for (const member of input.members) { const mcpPolicy = normalizeTeamMemberMcpPolicy(member.mcpPolicy); if (!mcpPolicy) { continue; } const memberCwd = member.cwd?.trim() || input.cwd; const mcpConfigPath = await this.mcpConfigBuilder.writeConfigFile(memberCwd, { mcpPolicy, controlApiBaseUrl: input.controlApiBaseUrl, }); const memberMcpConfigPaths = input.run.memberMcpConfigPaths ?? []; input.run.memberMcpConfigPaths = memberMcpConfigPaths; memberMcpConfigPaths.push(mcpConfigPath); configs.set(member.name, { mcpConfigPath, mcpSettingSources: buildTeamMemberMcpSettingSources(mcpPolicy), strictMcpConfig: requiresStrictTeamMemberMcpConfig(mcpPolicy), }); } return configs; } private async buildTrackedMemberMcpLaunchConfig(input: { cwd: string; mcpPolicy: unknown; run: ProvisioningRun; controlApiBaseUrl?: string | null; }): Promise { const mcpPolicy = normalizeTeamMemberMcpPolicy(input.mcpPolicy); if (!mcpPolicy) { return null; } const mcpConfigPath = await this.mcpConfigBuilder.writeConfigFile(input.cwd, { mcpPolicy, controlApiBaseUrl: input.controlApiBaseUrl, }); const memberMcpConfigPaths = input.run.memberMcpConfigPaths ?? []; input.run.memberMcpConfigPaths = memberMcpConfigPaths; memberMcpConfigPaths.push(mcpConfigPath); return { mcpConfigPath, mcpSettingSources: buildTeamMemberMcpSettingSources(mcpPolicy), strictMcpConfig: requiresStrictTeamMemberMcpConfig(mcpPolicy), }; } private async removeTrackedMemberMcpLaunchConfig( run: ProvisioningRun, mcpLaunchConfig: RuntimeBootstrapMemberMcpLaunchConfig | null | undefined ): Promise { if (!mcpLaunchConfig?.mcpConfigPath) { return; } const memberMcpConfigPaths = (run.memberMcpConfigPaths ??= []); const index = memberMcpConfigPaths.indexOf(mcpLaunchConfig.mcpConfigPath); if (index >= 0) { memberMcpConfigPaths.splice(index, 1); } await this.mcpConfigBuilder.removeConfigFile(mcpLaunchConfig.mcpConfigPath); } async prepareLiveMemberMcpLaunchConfig(input: { teamName: string; cwd?: string; mcpPolicy?: unknown; }): Promise { const mcpPolicy = normalizeTeamMemberMcpPolicy(input.mcpPolicy); if (!mcpPolicy) { return null; } const runId = this.getAliveRunId(input.teamName); const run = runId ? this.runs.get(runId) : undefined; if (!run || run.processKilled || run.cancelRequested) { throw new Error(`Team "${input.teamName}" is not currently running`); } const cwd = input.cwd?.trim() || run.request.cwd?.trim(); if (!cwd) { throw new Error(`Team "${input.teamName}" project path is not available`); } await ensureCwdExists(cwd); return this.buildTrackedMemberMcpLaunchConfig({ cwd, mcpPolicy, run, controlApiBaseUrl: await this.resolveControlApiBaseUrl().catch(() => null), }); } async discardLiveMemberMcpLaunchConfig(input: { teamName: string; mcpLaunchConfig: RuntimeBootstrapMemberMcpLaunchConfig | null | undefined; }): Promise { const runId = this.getAliveRunId(input.teamName); const run = runId ? this.runs.get(runId) : undefined; if (!run) { if (input.mcpLaunchConfig?.mcpConfigPath) { await this.mcpConfigBuilder.removeConfigFile(input.mcpLaunchConfig.mcpConfigPath); } return; } await this.removeTrackedMemberMcpLaunchConfig(run, input.mcpLaunchConfig); } private async removeRunMemberMcpConfigFiles(run: ProvisioningRun): Promise { const paths = run.memberMcpConfigPaths?.splice(0) ?? []; await Promise.all( paths.map((configPath) => this.mcpConfigBuilder.removeConfigFile(configPath)) ); } private removeRunMemberMcpConfigFilesLater(run: ProvisioningRun): void { for (const configPath of run.memberMcpConfigPaths?.splice(0) ?? []) { void this.mcpConfigBuilder.removeConfigFile(configPath); } } private buildMembersMetaWritePayload(members: TeamCreateRequest['members']): TeamMember[] { return applyDistinctProvisioningMemberColors( members.map((member) => ({ name: member.name.trim(), role: member.role?.trim() || undefined, workflow: member.workflow?.trim() || undefined, isolation: member.isolation === 'worktree' ? ('worktree' as const) : undefined, cwd: member.cwd?.trim() || undefined, providerId: normalizeOptionalTeamProviderId(member.providerId), providerBackendId: migrateProviderBackendId(member.providerId, member.providerBackendId), model: member.model?.trim() || undefined, effort: isTeamEffortLevel(member.effort) ? member.effort : undefined, fastMode: member.fastMode === 'inherit' || member.fastMode === 'on' || member.fastMode === 'off' ? member.fastMode : undefined, mcpPolicy: normalizeTeamMemberMcpPolicy(member.mcpPolicy), agentType: 'general-purpose' as const, color: getMemberColorByName(member.name.trim()), joinedAt: typeof (member as { joinedAt?: unknown }).joinedAt === 'number' ? (member as { joinedAt?: number }).joinedAt! : Date.now(), })) ); } private enrichRuntimeAdapterProgressTrace( progress: TeamProvisioningProgress ): TeamProvisioningProgress { const detail = buildProvisioningTraceDetail(progress); const key = `${progress.state}\u0000${progress.message}\u0000${detail ?? ''}`; const lines = this.runtimeAdapterTraceLinesByRunId.get(progress.runId) ?? []; if (this.runtimeAdapterTraceKeyByRunId.get(progress.runId) !== key) { this.runtimeAdapterTraceKeyByRunId.set(progress.runId, key); lines.push( buildProgressTraceLine({ timestamp: progress.updatedAt, state: progress.state, message: progress.message, detail, }) ); if (lines.length > PROVISIONING_TRACE_STORAGE_LIMIT) { lines.splice(0, lines.length - PROVISIONING_TRACE_STORAGE_LIMIT); } this.runtimeAdapterTraceLinesByRunId.set(progress.runId, lines); } return { ...progress, assistantOutput: buildProgressLiveOutput(lines, []) ?? progress.assistantOutput, }; } private setRuntimeAdapterProgress( progress: TeamProvisioningProgress, onProgress?: (progress: TeamProvisioningProgress) => void ): TeamProvisioningProgress { const nextProgress = this.enrichRuntimeAdapterProgressTrace(progress); this.runtimeAdapterProgressByRunId.set(nextProgress.runId, nextProgress); onProgress?.(nextProgress); return nextProgress; } private async getPersistedTranscriptClaudeLogs( teamName: string ): Promise { const context = await this.transcriptProjectResolver.getContext(teamName); const leadSessionId = typeof context?.config.leadSessionId === 'string' ? context.config.leadSessionId.trim() : ''; if (!context || leadSessionId.length === 0) { this.persistedTranscriptClaudeLogsCache.delete(teamName); return null; } const transcriptPath = path.join(context.projectDir, `${leadSessionId}.jsonl`); let stat: fs.Stats; try { stat = await fs.promises.stat(transcriptPath); } catch { this.persistedTranscriptClaudeLogsCache.delete(teamName); return null; } if (!stat.isFile()) { this.persistedTranscriptClaudeLogsCache.delete(teamName); return null; } const cached = this.persistedTranscriptClaudeLogsCache.get(teamName); if ( cached?.transcriptPath === transcriptPath && cached.mtimeMs === stat.mtimeMs && cached.size === stat.size ) { return cached.snapshot; } const lines = await this.readTranscriptClaudeLogLines(transcriptPath); if (lines.length === 0) { this.persistedTranscriptClaudeLogsCache.delete(teamName); return null; } const snapshot = { lines, updatedAt: stat.mtime.toISOString(), }; this.persistedTranscriptClaudeLogsCache.set(teamName, { transcriptPath, mtimeMs: stat.mtimeMs, size: stat.size, snapshot, }); return snapshot; } private async readTranscriptClaudeLogLines(filePath: string): Promise { const lines: string[] = []; const stream = fs.createReadStream(filePath, { encoding: 'utf8' }); const rl = readline.createInterface({ input: stream, crlfDelay: Infinity }); try { for await (const rawLine of rl) { const line = rawLine.endsWith('\r') ? rawLine.slice(0, -1) : rawLine; if (!line.trim()) { continue; } lines.push(line); if (lines.length > TeamProvisioningService.CLAUDE_LOG_LINES_LIMIT) { lines.splice(0, lines.length - TeamProvisioningService.CLAUDE_LOG_LINES_LIMIT); } } } finally { rl.close(); stream.close(); } return lines; } private clearSameTeamRetryTimers(teamName: string): void { for (const suffix of ['deferred', 'persist']) { const key = `same-team-${suffix}:${teamName}`; const timer = this.pendingTimeouts.get(key); if (timer) { clearTimeout(timer); this.pendingTimeouts.delete(key); } } } private clearLeadInboxFollowUpRelayTimer(teamName: string): void { const key = `lead-inbox-follow-up:${teamName}`; const timer = this.pendingTimeouts.get(key); if (timer) { clearTimeout(timer); this.pendingTimeouts.delete(key); } } private scheduleLeadInboxFollowUpRelay(teamName: string): void { const key = `lead-inbox-follow-up:${teamName}`; if (this.pendingTimeouts.has(key)) return; const timer = setTimeout(() => { this.pendingTimeouts.delete(key); void this.relayLeadInboxMessages(teamName).catch((error: unknown) => logger.warn(`[${teamName}] lead inbox follow-up relay failed: ${String(error)}`) ); }, 50); timer.unref?.(); this.pendingTimeouts.set(key, timer); } private resetTeamScopedTransientStateForNewRun(teamName: string): void { peekAutoResumeService()?.cancelPendingAutoResume(teamName); this.clearOpenCodeRuntimeToolApprovals(teamName, { emitDismiss: true }); this.invalidateRuntimeSnapshotCaches(teamName); this.runtimeProcessRowsForUsageSnapshotByTeam.delete(teamName); this.retainedClaudeLogsByTeam.delete(teamName); this.persistedTranscriptClaudeLogsCache.delete(teamName); this.leadInboxRelayInFlight.delete(teamName); this.relayedLeadInboxMessageIds.delete(teamName); this.pendingCrossTeamFirstReplies.delete(teamName); this.recentCrossTeamLeadDeliveryMessageIds.delete(teamName); this.recentSameTeamNativeFingerprints.delete(teamName); this.clearSameTeamRetryTimers(teamName); this.clearLeadInboxFollowUpRelayTimer(teamName); for (const key of Array.from(this.memberInboxRelayInFlight.keys())) { if (key.startsWith(`${teamName}:`)) { this.memberInboxRelayInFlight.delete(key); } } for (const key of Array.from(this.openCodeMemberInboxRelayInFlight.keys())) { if (key.startsWith(`opencode:${teamName}:`)) { this.openCodeMemberInboxRelayInFlight.delete(key); } } for (const key of Array.from(this.openCodePromptDeliveryWatchdogTimers.keys())) { if (key.startsWith(`opencode-delivery:${teamName}:`)) { const timer = this.openCodePromptDeliveryWatchdogTimers.get(key); if (timer) clearTimeout(timer); this.openCodePromptDeliveryWatchdogTimers.delete(key); this.openCodePromptDeliveryWatchdogDeadlines.delete(key); } } for (let index = this.openCodePromptDeliveryWatchdogQueue.length - 1; index >= 0; index -= 1) { if (this.openCodePromptDeliveryWatchdogQueue[index]?.teamName === teamName) { this.openCodePromptDeliveryWatchdogQueue.splice(index, 1); } } for (const key of Array.from(this.relayedMemberInboxMessageIds.keys())) { if (key.startsWith(`${teamName}:`)) { this.relayedMemberInboxMessageIds.delete(key); } } this.liveLeadProcessMessages.delete(teamName); } private appendCliLogs(run: ProvisioningRun, stream: 'stdout' | 'stderr', text: string): void { const nowMs = Date.now(); run.claudeLogsUpdatedAt = new Date(nowMs).toISOString(); const marker = stream === 'stdout' ? '[stdout]' : '[stderr]'; if (run.lastClaudeLogStream !== stream) { run.lastClaudeLogStream = stream; run.claudeLogLines.push(marker); } if (stream === 'stdout') { run.stdoutLogLineBuf += text; const parts = run.stdoutLogLineBuf.split('\n'); run.stdoutLogLineBuf = parts.pop() ?? ''; for (const part of parts) { const normalized = part.endsWith('\r') ? part.slice(0, -1) : part; run.claudeLogLines.push(normalized); } } else { run.stderrLogLineBuf += text; const parts = run.stderrLogLineBuf.split('\n'); run.stderrLogLineBuf = parts.pop() ?? ''; for (const part of parts) { const normalized = part.endsWith('\r') ? part.slice(0, -1) : part; run.claudeLogLines.push(normalized); } } if (run.claudeLogLines.length > TeamProvisioningService.CLAUDE_LOG_LINES_LIMIT) { run.claudeLogLines.splice( 0, run.claudeLogLines.length - TeamProvisioningService.CLAUDE_LOG_LINES_LIMIT ); } } /** * Serializes operations per team name using promise-chaining. * Same pattern as withInboxLock / withTaskLock. * Prevents TOCTOU races between concurrent createTeam/launchTeam calls. */ private async withTeamLock(teamName: string, fn: () => Promise): Promise { const prev = this.teamOpLocks.get(teamName) ?? Promise.resolve(); let release!: () => void; const mine = new Promise((resolve) => { release = resolve; }); this.teamOpLocks.set(teamName, mine); await prev; try { return await fn(); } finally { release(); if (this.teamOpLocks.get(teamName) === mine) { this.teamOpLocks.delete(teamName); } } } setTeamChangeEmitter(emitter: ((event: TeamChangeEvent) => void) | null): void { this.teamChangeEmitter = emitter; } private parseCrossTeamRecipient( currentTeam: string, recipient: string, localRecipientNames: Set ): { teamName: string; memberName: string } | null { const trimmed = recipient.trim(); if (localRecipientNames.has(trimmed)) return null; const pseudoTeamName = this.extractCrossTeamPseudoTargetTeam(trimmed); if (pseudoTeamName) { if (pseudoTeamName === currentTeam) { return null; } return { teamName: pseudoTeamName, memberName: 'team-lead' }; } const dot = trimmed.indexOf('.'); if (dot <= 0 || dot === trimmed.length - 1) return null; const teamName = trimmed.slice(0, dot).trim(); const memberName = trimmed.slice(dot + 1).trim(); if (!TEAM_NAME_PATTERN.test(teamName) || !memberName || teamName === currentTeam) { return null; } return { teamName, memberName }; } private extractCrossTeamPseudoTargetTeam(value: string): string | null { const trimmed = value.trim(); const prefixes = [ 'cross_team::', 'cross_team--', 'cross-team:', 'cross-team-', 'cross_team:', 'cross_team-', ]; for (const prefix of prefixes) { if (!trimmed.startsWith(prefix)) continue; const teamName = trimmed.slice(prefix.length).trim(); if (TEAM_NAME_PATTERN.test(teamName)) { return teamName; } } return null; } private isCrossTeamToolRecipientName(name: string): boolean { return CROSS_TEAM_TOOL_RECIPIENT_NAMES.has(name.trim()); } private isCrossTeamPseudoRecipientName(name: string): boolean { return this.extractCrossTeamPseudoTargetTeam(name) !== null; } private resolveSingleActiveCrossTeamReplyHint( run: ProvisioningRun ): { toTeam: string; conversationId: string } | null { const uniqueHints = new Map(); for (const hint of run.activeCrossTeamReplyHints ?? []) { const toTeam = typeof hint?.toTeam === 'string' ? hint.toTeam.trim() : ''; const conversationId = typeof hint?.conversationId === 'string' ? hint.conversationId.trim() : ''; if (!toTeam || !conversationId) continue; uniqueHints.set(`${toTeam}\0${conversationId}`, { toTeam, conversationId }); } return uniqueHints.size === 1 ? (Array.from(uniqueHints.values())[0] ?? null) : null; } private looksLikeQualifiedExternalRecipientName(name: string): boolean { const trimmed = name.trim(); const dot = trimmed.indexOf('.'); if (dot <= 0 || dot === trimmed.length - 1) return false; const teamName = trimmed.slice(0, dot).trim(); const memberName = trimmed.slice(dot + 1).trim(); return TEAM_NAME_PATTERN.test(teamName) && memberName.length > 0; } private buildCrossTeamConversationKey(otherTeam: string, conversationId: string): string { return `${otherTeam.trim()}\0${conversationId.trim()}`; } registerPendingCrossTeamReplyExpectation( teamName: string, otherTeam: string, conversationId: string ): void { const normalizedTeam = teamName.trim(); const normalizedOtherTeam = otherTeam.trim(); const normalizedConversationId = conversationId.trim(); if (!normalizedTeam || !normalizedOtherTeam || !normalizedConversationId) return; const teamMap = this.pendingCrossTeamFirstReplies.get(normalizedTeam) ?? new Map(); teamMap.set( this.buildCrossTeamConversationKey(normalizedOtherTeam, normalizedConversationId), Date.now() ); this.pendingCrossTeamFirstReplies.set(normalizedTeam, teamMap); } clearPendingCrossTeamReplyExpectation( teamName: string, otherTeam: string, conversationId: string ): void { const teamMap = this.pendingCrossTeamFirstReplies.get(teamName.trim()); if (!teamMap) return; teamMap.delete(this.buildCrossTeamConversationKey(otherTeam, conversationId)); if (teamMap.size === 0) { this.pendingCrossTeamFirstReplies.delete(teamName.trim()); } } private getPendingCrossTeamReplyExpectationKeys(teamName: string): Set { const teamMap = this.pendingCrossTeamFirstReplies.get(teamName.trim()); if (!teamMap) return new Set(); const cutoff = Date.now() - TeamProvisioningService.RECENT_CROSS_TEAM_DELIVERY_TTL_MS; for (const [key, createdAt] of teamMap.entries()) { if (createdAt < cutoff) { teamMap.delete(key); } } if (teamMap.size === 0) { this.pendingCrossTeamFirstReplies.delete(teamName.trim()); return new Set(); } return new Set(teamMap.keys()); } private getRunLeadName(run: ProvisioningRun): string { const members = Array.isArray(run.request?.members) ? run.request.members : []; return members.find((m) => m.role?.toLowerCase().includes('lead'))?.name || 'team-lead'; } private rememberRecentCrossTeamLeadDeliveryMessageIds( teamName: string, messageIds: string[] ): void { const normalizedIds = messageIds.map((id) => id.trim()).filter((id) => id.length > 0); if (normalizedIds.length === 0) return; const teamKey = teamName.trim(); const current = this.recentCrossTeamLeadDeliveryMessageIds.get(teamKey) ?? new Map(); const now = Date.now(); const cutoff = now - TeamProvisioningService.RECENT_CROSS_TEAM_DELIVERY_TTL_MS; for (const [key, createdAt] of current.entries()) { if (createdAt < cutoff) current.delete(key); } for (const messageId of normalizedIds) { current.set(messageId, now); } if (current.size > 0) { this.recentCrossTeamLeadDeliveryMessageIds.set(teamKey, current); } } private wasRecentlyDeliveredToLead(teamName: string, messageId: string): boolean { const normalizedMessageId = messageId.trim(); if (!normalizedMessageId) return false; const teamKey = teamName.trim(); const current = this.recentCrossTeamLeadDeliveryMessageIds.get(teamKey); if (!current) return false; const cutoff = Date.now() - TeamProvisioningService.RECENT_CROSS_TEAM_DELIVERY_TTL_MS; for (const [key, createdAt] of current.entries()) { if (createdAt < cutoff) current.delete(key); } if (current.size === 0) { this.recentCrossTeamLeadDeliveryMessageIds.delete(teamKey); return false; } return current.has(normalizedMessageId); } private parseCrossTeamTargetTeam(value: string | undefined): string | null { if (typeof value !== 'string') return null; const trimmed = value.trim(); if (!trimmed) return null; if (trimmed.startsWith('cross-team:')) { const teamName = trimmed.slice('cross-team:'.length).trim(); return TEAM_NAME_PATTERN.test(teamName) ? teamName : null; } const dot = trimmed.indexOf('.'); if (dot <= 0) return null; const teamName = trimmed.slice(0, dot).trim(); return TEAM_NAME_PATTERN.test(teamName) ? teamName : null; } private getCrossTeamSourceTeam(value: string | undefined): string | null { if (typeof value !== 'string') return null; const trimmed = value.trim(); const dot = trimmed.indexOf('.'); if (dot <= 0) return null; const teamName = trimmed.slice(0, dot).trim(); return TEAM_NAME_PATTERN.test(teamName) ? teamName : null; } private extractStreamUserText(msg: Record): string | null { const topLevelContent = msg.content; if (typeof topLevelContent === 'string') { return topLevelContent; } if (Array.isArray(topLevelContent)) { const text = topLevelContent .filter( (part): part is Record => !!part && typeof part === 'object' && part.type === 'text' && typeof part.text === 'string' ) .map((part) => part.text as string) .join('\n') .trim(); if (text.length > 0) return text; } const message = msg.message; if (!message || typeof message !== 'object') return null; const innerContent = (message as Record).content; if (typeof innerContent === 'string') { const trimmed = innerContent.trim(); return trimmed.length > 0 ? trimmed : null; } if (!Array.isArray(innerContent)) return null; const text = innerContent .filter( (part): part is Record => !!part && typeof part === 'object' && part.type === 'text' && typeof part.text === 'string' ) .map((part) => part.text as string) .join('\n') .trim(); return text.length > 0 ? text : null; } private extractStreamContentBlocks(msg: Record): Record[] { const topLevelContent = msg.content; if (Array.isArray(topLevelContent)) { return topLevelContent as Record[]; } const message = msg.message; if (!message || typeof message !== 'object') return []; const innerContent = (message as Record).content; return Array.isArray(innerContent) ? (innerContent as Record[]) : []; } private hasCapturedVisibleSendMessage( content: Record[], teamName: string ): boolean { return content.some((part) => { if (!part || typeof part !== 'object') return false; if (part.type !== 'tool_use' || typeof part.name !== 'string') return false; const input = part.input; if (!input || typeof input !== 'object') return false; const inp = input as Record; if (part.name === 'SendMessage') { const target = (typeof inp.recipient === 'string' ? inp.recipient : '').trim(); const text = (typeof inp.content === 'string' ? inp.content : '').trim(); return target.length > 0 && text.length > 0; } const isTeamMessageSendTool = isAgentTeamsToolUse({ rawName: part.name, canonicalName: 'message_send', toolInput: inp, currentTeamName: teamName, }); const isDirectCrossTeamSendTool = isAgentTeamsToolUse({ rawName: part.name, canonicalName: 'cross_team_send', toolInput: inp, currentTeamName: teamName, }); if (!isTeamMessageSendTool && !isDirectCrossTeamSendTool) return false; const target = isTeamMessageSendTool ? typeof inp.to === 'string' ? inp.to : '' : typeof inp.toTeam === 'string' ? inp.toTeam : ''; const text = typeof inp.text === 'string' ? inp.text : ''; return target.trim().length > 0 && text.trim().length > 0; }); } private hasCapturedUserVisibleSendMessage( content: Record[], teamName: string ): boolean { return content.some((part) => { if (!part || typeof part !== 'object') return false; if (part.type !== 'tool_use' || typeof part.name !== 'string') return false; const input = part.input; if (!input || typeof input !== 'object') return false; const inp = input as Record; if (part.name === 'SendMessage') { const target = (typeof inp.recipient === 'string' ? inp.recipient : '') .trim() .toLowerCase(); const text = (typeof inp.content === 'string' ? inp.content : '').trim(); return target === 'user' && text.length > 0; } const isTeamMessageSendTool = isAgentTeamsToolUse({ rawName: part.name, canonicalName: 'message_send', toolInput: inp, currentTeamName: teamName, }); if (!isTeamMessageSendTool) return false; const target = typeof inp.to === 'string' ? inp.to.trim().toLowerCase() : ''; const text = typeof inp.text === 'string' ? inp.text.trim() : ''; return target === 'user' && text.length > 0; }); } private async matchCrossTeamLeadInboxMessages( teamName: string, leadName: string, deliveredBlocks: { teammateId: string; content: string; toTeam: string; conversationId: string; }[] ): Promise< { teammateId: string; content: string; toTeam: string; conversationId: string; messageId: string; wasRead: boolean; }[] > { if (deliveredBlocks.length === 0) return []; let leadInboxMessages: Awaited> = []; try { leadInboxMessages = await this.inboxReader.getMessagesFor(teamName, leadName); } catch { return []; } const usedMessageIds = new Set(); const matches: { teammateId: string; content: string; toTeam: string; conversationId: string; messageId: string; wasRead: boolean; }[] = []; for (const block of deliveredBlocks) { const matchesBlock = (message: InboxMessage, requireExactText: boolean): boolean => { if (message.source !== CROSS_TEAM_SOURCE) return false; if (!this.hasStableMessageId(message)) return false; if (usedMessageIds.has(message.messageId)) return false; if (message.from.trim() !== block.teammateId.trim()) return false; const messageConversationId = message.replyToConversationId?.trim() ?? message.conversationId?.trim() ?? parseCrossTeamPrefix(message.text)?.conversationId; if (messageConversationId !== block.conversationId) return false; return !requireExactText || message.text.trim() === block.content.trim(); }; const matched = leadInboxMessages.find((message) => matchesBlock(message, true)) ?? leadInboxMessages.find((message) => matchesBlock(message, false)); if (!matched || !this.hasStableMessageId(matched)) continue; usedMessageIds.add(matched.messageId); matches.push({ teammateId: block.teammateId, content: block.content, toTeam: block.toTeam, conversationId: block.conversationId, messageId: matched.messageId, wasRead: matched.read === true, }); } return matches; } private handleNativeTeammateUserMessage( run: ProvisioningRun, msg: Record ): void { const rawText = this.extractStreamUserText(msg); if (!rawText) return; const blocks = parseAllTeammateMessages(rawText); if (blocks.length === 0) return; // Intercept teammate permission_request messages delivered natively via stdout. // This runs even during provisioning (unlike relayLeadInboxMessages which waits // for provisioningComplete). The lead already received the message — we can't // prevent that — but we create a ToolApprovalRequest so the user sees the dialog. for (const block of blocks) { const perm = parsePermissionRequest(block.content); if (perm) { this.handleTeammatePermissionRequest(run, perm, new Date().toISOString()); } } const crossTeamBlocks = blocks.flatMap((block) => { const origin = parseCrossTeamPrefix(block.content); const sourceTeam = origin?.from.includes('.') ? origin.from.split('.', 1)[0] : null; const conversationId = origin?.conversationId?.trim() || origin?.replyToConversationId?.trim(); if (!sourceTeam || !conversationId) return []; return [ { teammateId: block.teammateId, content: block.content, toTeam: sourceTeam, conversationId, }, ]; }); // Cross-team reconciliation (existing logic) if (crossTeamBlocks.length > 0) { const leadName = this.getRunLeadName(run); void (async () => { const matches = await this.matchCrossTeamLeadInboxMessages( run.teamName, leadName, crossTeamBlocks ); const unreadMatches = matches.filter((match) => !match.wasRead); if (unreadMatches.length > 0) { try { await this.markInboxMessagesRead(run.teamName, leadName, unreadMatches); } catch { // best-effort } } const freshMatches = matches.filter( (match) => !this.wasRecentlyDeliveredToLead(run.teamName, match.messageId) ); this.rememberRecentCrossTeamLeadDeliveryMessageIds( run.teamName, freshMatches.map((match) => match.messageId) ); run.activeCrossTeamReplyHints = freshMatches.map((match) => ({ toTeam: match.toTeam, conversationId: match.conversationId, })); })(); } // Same-team teammate messages are the canonical heartbeat signal: they prove the // runtime produced a real post-spawn message, unlike writes to inboxes/.json // which may simply be user/lead messages addressed TO the teammate. const sameTeamBlocks = blocks.filter((block) => !parseCrossTeamPrefix(block.content)); const meaningfulSameTeamBlocks = sameTeamBlocks.filter((block) => isMeaningfulBootstrapCheckInMessage(block.content) ); for (const block of meaningfulSameTeamBlocks) { this.setMemberSpawnStatus(run, block.teammateId, 'online', undefined, 'heartbeat'); } for (const block of sameTeamBlocks) { const bootstrapFailureReason = extractBootstrapFailureReason(block.content); if (!bootstrapFailureReason) continue; this.setMemberSpawnStatus(run, block.teammateId, 'error', bootstrapFailureReason); } if (sameTeamBlocks.length > 0) { this.rememberSameTeamNativeFingerprints(run.teamName, sameTeamBlocks); const leadName = this.getRunLeadName(run); void this.reconcileSameTeamNativeDeliveries(run.teamName, leadName); } } private async refreshMemberSpawnStatusesFromLeadInbox(run: ProvisioningRun): Promise { const leadName = this.getRunLeadName(run); let leadInboxMessages: Awaited> = []; try { leadInboxMessages = await this.inboxReader.getMessagesFor(run.teamName, leadName); } catch { return; } const runStartedAtMs = Date.parse(run.startedAt); const expectedMembers = Array.isArray(run.expectedMembers) ? run.expectedMembers : []; const teammateMessages = leadInboxMessages .filter((message): message is LeadInboxMemberSpawnMessage => { const from = typeof message.from === 'string' ? message.from.trim() : ''; if (!from || from === leadName || from === 'user' || from === 'system') return false; if (!this.resolveExpectedLaunchMemberName(expectedMembers, from)) return false; if (typeof message.messageId !== 'string' || message.messageId.trim().length === 0) { return false; } const messageTs = Date.parse(message.timestamp); if ( Number.isFinite(messageTs) && Number.isFinite(runStartedAtMs) && messageTs < runStartedAtMs ) { return false; } return typeof message.text === 'string' && message.text.trim().length > 0; }) .sort((left, right) => compareMemberSpawnInboxCursor( { timestamp: left.timestamp, messageId: left.messageId }, { timestamp: right.timestamp, messageId: right.messageId } ) ); const messagesByMember = new Map(); for (const message of teammateMessages) { const memberName = this.resolveExpectedLaunchMemberName(expectedMembers, message.from); if (!memberName) { continue; } const bucket = messagesByMember.get(memberName) ?? []; bucket.push(message); messagesByMember.set(memberName, bucket); } for (const [memberName, messages] of messagesByMember.entries()) { const currentCursor = run.memberSpawnLeadInboxCursorByMember.get(memberName); let nextCursor = currentCursor; for (const message of messages) { const messageCursor = toMemberSpawnInboxCursor(message); const effectiveCursor = nextCursor ?? currentCursor; if (messageCursor && effectiveCursor) { if (compareMemberSpawnInboxCursor(messageCursor, effectiveCursor) <= 0) { continue; } } this.applyLeadInboxSpawnSignal(run, memberName, message); if (messageCursor) { nextCursor = maxMemberSpawnInboxCursor(nextCursor, messageCursor); } } if ( nextCursor && (currentCursor == null || compareMemberSpawnInboxCursor(nextCursor, currentCursor) > 0) ) { run.memberSpawnLeadInboxCursorByMember.set(memberName, nextCursor); } } } private applyLeadInboxSpawnSignal( run: ProvisioningRun, memberName: string, message: LeadInboxMemberSpawnMessage ): void { const reason = extractBootstrapFailureReason(message.text); if (reason) { this.setMemberSpawnStatus(run, memberName, 'error', reason); return; } this.setMemberSpawnStatus( run, memberName, 'online', undefined, 'heartbeat', extractHeartbeatTimestamp(message.text, message.timestamp) ); } private resolveExpectedLaunchMemberName( expectedMembers: readonly string[] | undefined, candidateName: string ): string | null { const trimmedCandidate = candidateName.trim(); if (!trimmedCandidate || !Array.isArray(expectedMembers) || expectedMembers.length === 0) { return null; } const exact = expectedMembers.find((memberName) => matchesExactTeamMemberName(memberName, trimmedCandidate) ); if (exact) { return exact; } const matches = expectedMembers.filter((memberName) => matchesObservedMemberNameForExpected(trimmedCandidate, memberName) ); return matches.length === 1 ? (matches[0] ?? null) : null; } private persistSentMessage(teamName: string, message: InboxMessage): void { try { createController({ teamName, claudeDir: getClaudeBasePath(), }).messages.appendSentMessage({ from: message.from, to: message.to, text: message.text, timestamp: message.timestamp, summary: message.summary, messageId: message.messageId, relayOfMessageId: message.relayOfMessageId, source: message.source, leadSessionId: message.leadSessionId, conversationId: message.conversationId, replyToConversationId: message.replyToConversationId, taskRefs: message.taskRefs, attachments: message.attachments, color: message.color, toolSummary: message.toolSummary, toolCalls: message.toolCalls, messageKind: message.messageKind, workSyncIntent: message.workSyncIntent, workSyncIntentKey: message.workSyncIntentKey, workSyncReviewRequestEventIds: message.workSyncReviewRequestEventIds, slashCommand: message.slashCommand, commandOutput: message.commandOutput, }); } catch (error) { logger.warn(`[${teamName}] sent-message persist failed: ${String(error)}`); } } private persistInboxMessage(teamName: string, recipient: string, message: InboxMessage): void { try { createController({ teamName, claudeDir: getClaudeBasePath(), }).messages.sendMessage({ member: recipient, from: message.from, text: message.text, timestamp: message.timestamp, summary: message.summary, messageId: message.messageId, relayOfMessageId: message.relayOfMessageId, source: message.source, leadSessionId: message.leadSessionId, conversationId: message.conversationId, replyToConversationId: message.replyToConversationId, taskRefs: message.taskRefs, attachments: message.attachments, color: message.color, toolSummary: message.toolSummary, toolCalls: message.toolCalls, messageKind: message.messageKind, workSyncIntent: message.workSyncIntent, workSyncIntentKey: message.workSyncIntentKey, workSyncReviewRequestEventIds: message.workSyncReviewRequestEventIds, slashCommand: message.slashCommand, commandOutput: message.commandOutput, }); this.emitRuntimeDeliveryReplyAdvisoryRefresh(teamName, message); } catch (error) { logger.warn(`[${teamName}] inbox-message persist for ${recipient} failed: ${String(error)}`); } } private getMemberRelayKey(teamName: string, memberName: string): string { return `${teamName}:${memberName.trim()}`; } private getOpenCodeMemberRelayKey(teamName: string, memberName: string): string { return `opencode:${this.getMemberRelayKey(teamName, memberName)}`; } private async waitForInboxRelayInFlight(input: { promise: Promise; relayName: string; relayKey: string; }): Promise { let timer: ReturnType | null = null; try { return await Promise.race([ input.promise, new Promise((_, reject) => { timer = setTimeout(() => { reject( new InboxRelayInFlightTimeoutError( `${input.relayName} timed out after ${TeamProvisioningService.INBOX_RELAY_IN_FLIGHT_TIMEOUT_MS}ms: ${input.relayKey}` ) ); }, TeamProvisioningService.INBOX_RELAY_IN_FLIGHT_TIMEOUT_MS); timer.unref?.(); }), ]); } finally { if (timer) { clearTimeout(timer); } } } private normalizeRelayCandidateText(text: string): string { return stripAgentBlocks(String(text)).trim().replace(/\r\n/g, '\n'); } private normalizeRelayCandidateSummary(summary?: string): string { return typeof summary === 'string' ? summary.trim() : ''; } private prunePendingInboxRelayCandidates(run: ProvisioningRun): PendingInboxRelayCandidate[] { const cutoff = Date.now() - TeamProvisioningService.PENDING_INBOX_RELAY_TTL_MS; run.pendingInboxRelayCandidates = (run.pendingInboxRelayCandidates ?? []).filter( (candidate) => candidate.queuedAtMs >= cutoff ); return run.pendingInboxRelayCandidates; } private rememberPendingInboxRelayCandidates( run: ProvisioningRun, recipient: string, messages: Pick[] ): string[] { const candidates = this.prunePendingInboxRelayCandidates(run); const queuedAtMs = Date.now(); const rememberedIds: string[] = []; for (const message of messages) { const sourceMessageId = typeof message.messageId === 'string' ? message.messageId.trim() : ''; const normalizedText = this.normalizeRelayCandidateText(message.text); if (!sourceMessageId || !normalizedText) { continue; } candidates.push({ recipient, sourceMessageId, normalizedText, normalizedSummary: this.normalizeRelayCandidateSummary(message.summary), queuedAtMs, }); rememberedIds.push(sourceMessageId); } return rememberedIds; } private forgetPendingInboxRelayCandidates( run: ProvisioningRun, recipient: string, sourceMessageIds: readonly string[] ): void { if (sourceMessageIds.length === 0) { return; } const idSet = new Set(sourceMessageIds); run.pendingInboxRelayCandidates = this.prunePendingInboxRelayCandidates(run).filter( (candidate) => !(candidate.recipient === recipient && idSet.has(candidate.sourceMessageId)) ); } private consumePendingInboxRelayCandidate( run: ProvisioningRun, recipient: string, text: string, summary?: string ): string | undefined { const normalizedText = this.normalizeRelayCandidateText(text); if (!normalizedText) { return undefined; } const normalizedSummary = this.normalizeRelayCandidateSummary(summary); const candidates = this.prunePendingInboxRelayCandidates(run); const exactSummaryIdx = candidates.findIndex( (candidate) => candidate.recipient === recipient && candidate.normalizedText === normalizedText && candidate.normalizedSummary === normalizedSummary ); const fallbackIdx = exactSummaryIdx >= 0 ? exactSummaryIdx : candidates.findIndex( (candidate) => candidate.recipient === recipient && candidate.normalizedText === normalizedText ); if (fallbackIdx < 0) { return undefined; } const [matched] = candidates.splice(fallbackIdx, 1); return matched?.sourceMessageId; } private armSilentTeammateForward( run: ProvisioningRun, teammateName: string, mode: 'user_dm' | 'member_inbox_relay' ): void { run.silentUserDmForward = { target: teammateName, startedAt: nowIso(), mode }; if (run.silentUserDmForwardClearHandle) { clearTimeout(run.silentUserDmForwardClearHandle); run.silentUserDmForwardClearHandle = null; } run.silentUserDmForwardClearHandle = setTimeout(() => { run.silentUserDmForward = null; run.silentUserDmForwardClearHandle = null; }, 60_000); run.silentUserDmForwardClearHandle.unref(); } private toolApprovalEventEmitter: ((event: ToolApprovalEvent) => void) | null = null; private mainWindowRef: import('electron').BrowserWindow | null = null; private activeApprovalNotifications = new Map(); setToolApprovalEventEmitter(emitter: (event: ToolApprovalEvent) => void): void { this.toolApprovalEventEmitter = emitter; } setMainWindow(win: import('electron').BrowserWindow | null): void { this.mainWindowRef = win; } private getToolApprovalSettings(teamName: string): ToolApprovalSettings { return this.toolApprovalSettingsByTeam.get(teamName) ?? DEFAULT_TOOL_APPROVAL_SETTINGS; } updateToolApprovalSettings(teamName: string, settings: ToolApprovalSettings): void { this.toolApprovalSettingsByTeam.set(teamName, settings); this.reEvaluatePendingApprovals(); } private emitToolApprovalEvent(event: ToolApprovalEvent): void { this.toolApprovalEventEmitter?.(event); } getLiveLeadProcessMessages(teamName: string): InboxMessage[] { const runId = this.getTrackedRunId(teamName); const detectedSessionId = runId ? (this.runs.get(runId)?.detectedSessionId ?? null) : null; return (this.liveLeadProcessMessages.get(teamName) ?? []).map((message) => !message.leadSessionId && detectedSessionId ? { ...message, leadSessionId: detectedSessionId } : { ...message } ); } private pruneLiveLeadMessagesForCleanedRun(run: ProvisioningRun): void { const list = this.liveLeadProcessMessages.get(run.teamName); if (!list || list.length === 0) { return; } const runMessageIdPrefixes = [ `lead-turn-${run.runId}-`, `lead-sendmsg-${run.runId}-`, `lead-process-${run.runId}-`, `compact-${run.runId}-`, ]; const filtered = list.filter((message) => { const messageId = typeof message.messageId === 'string' ? message.messageId.trim() : ''; if (messageId && runMessageIdPrefixes.some((prefix) => messageId.startsWith(prefix))) { return false; } if (run.detectedSessionId && message.leadSessionId === run.detectedSessionId) { return false; } return true; }); if (filtered.length === 0) { this.liveLeadProcessMessages.delete(run.teamName); return; } this.liveLeadProcessMessages.set(run.teamName, filtered); } getCurrentLeadSessionId(teamName: string): string | null { const runId = this.getTrackedRunId(teamName); if (!runId) return null; return this.runs.get(runId)?.detectedSessionId ?? null; } getCurrentRunId(teamName: string): string | null { return this.getAliveRunId(teamName); } async recordOpenCodeRuntimeBootstrapCheckin(raw: unknown): Promise { const payload = asRuntimeRecord(raw); const teamName = requireRuntimeString(payload.teamName, 'teamName'); const runId = requireRuntimeString(payload.runId, 'runId'); const memberName = requireRuntimeString(payload.memberName, 'memberName'); const runtimeSessionId = requireRuntimeString(payload.runtimeSessionId, 'runtimeSessionId'); const observedAt = normalizeRuntimeIso(payload.observedAt); const laneId = await this.resolveOpenCodeRuntimeLaneId({ teamName, runId, memberName }); await this.assertOpenCodeRuntimeEvidenceAccepted({ teamName, runId, laneId, evidenceKind: 'bootstrap_checkin', }); const idempotent = await this.resolveOpenCodeRuntimeBootstrapCheckinIdempotency({ teamName, runId, memberName, runtimeSessionId, }); await this.assertOpenCodeRuntimeMemberCheckinAllowed({ teamName, memberName, previousMember: idempotent.previousMember, }); if (idempotent.state === 'duplicate') { const committed = await this.hasCommittedOpenCodeRuntimeBootstrapSessionEvidence({ teamName, runId, laneId, memberName, runtimeSessionId, }); if (!committed) { await this.commitOpenCodeRuntimeBootstrapSessionEvidence({ teamName, runId, laneId, memberName, runtimeSessionId, observedAt, }); } await this.updateOpenCodeRuntimeMemberLiveness({ teamName, runId, memberName, runtimeSessionId, observedAt, diagnostics: payload.diagnostics, metadata: parseRuntimeToolMetadata(payload.metadata), reason: 'OpenCode runtime bootstrap check-in accepted', }); return { ok: true, providerId: 'opencode', teamName, runId, state: 'accepted', memberName, runtimeSessionId, diagnostics: ['opencode_bootstrap_checkin_duplicate_accepted'], observedAt, }; } if (idempotent.state === 'conflict') { throw new RuntimeStaleEvidenceError( `opencode_bootstrap_checkin_session_conflict: existing runtime session ${idempotent.existingRuntimeSessionId}, received ${runtimeSessionId} for ${memberName}`, 'run_mismatch', 'bootstrap_checkin', runId ); } await this.commitOpenCodeRuntimeBootstrapSessionEvidence({ teamName, runId, laneId, memberName, runtimeSessionId, observedAt, }); await this.updateOpenCodeRuntimeMemberLiveness({ teamName, runId, memberName, runtimeSessionId, observedAt, diagnostics: payload.diagnostics, metadata: parseRuntimeToolMetadata(payload.metadata), reason: 'OpenCode runtime bootstrap check-in accepted', }); return { ok: true, providerId: 'opencode', teamName, runId, state: 'accepted', memberName, runtimeSessionId, diagnostics: [], observedAt, }; } private async commitOpenCodeRuntimeBootstrapSessionEvidence(input: { teamName: string; runId: string; laneId: string; memberName: string; runtimeSessionId: string; observedAt: string; source?: OpenCodeBootstrapEvidenceSource; appManagedBootstrapCandidate?: OpenCodeAppManagedBootstrapCandidate; }): Promise { const descriptor = OPENCODE_RUNTIME_STORE_DESCRIPTORS.find( (candidate) => candidate.schemaName === 'opencode.sessionStore' ); if (!descriptor) { throw new Error('OpenCode runtime session store descriptor is not registered'); } const manifestPath = getOpenCodeRuntimeManifestPath( getTeamsBasePath(), input.teamName, input.laneId ); const runtimeDirectory = path.dirname(manifestPath); await fs.promises.mkdir(runtimeDirectory, { recursive: true }); const sessionStorePath = path.join(runtimeDirectory, descriptor.relativePath); const existingSessions = await this.readOpenCodeRuntimeSessionStore(sessionStorePath); const source = input.source ?? 'runtime_bootstrap_checkin'; const appMcpTransportEvidence = source === 'app_managed_bootstrap' ? getCurrentAgentTeamsMcpHttpTransportEvidence() : null; const session = { id: input.runtimeSessionId, teamName: input.teamName, memberName: input.memberName, runId: input.runId, laneId: input.laneId, providerId: 'opencode', observedAt: input.observedAt, source, ...(source === 'app_managed_bootstrap' && input.appManagedBootstrapCandidate ? { appManagedBootstrapCandidate: input.appManagedBootstrapCandidate } : {}), ...(appMcpTransportEvidence ? { appMcpTransportHash: appMcpTransportEvidence.urlHash, appMcpTransportEvidence, } : {}), }; const sessions = this.mergeOpenCodeRuntimeSessionRecords(existingSessions, session); const manifestStore = createRuntimeStoreManifestStore({ filePath: manifestPath, teamName: input.teamName, lockOptions: OPENCODE_BOOTSTRAP_EVIDENCE_LOCK_OPTIONS, }); const receiptStore = createRuntimeStoreReceiptStore({ filePath: path.join(runtimeDirectory, 'opencode-runtime-receipts.json'), lockOptions: OPENCODE_BOOTSTRAP_EVIDENCE_LOCK_OPTIONS, }); const writer = new RuntimeStoreBatchWriter(runtimeDirectory, manifestStore, receiptStore); try { await writer.writeBatch({ teamName: input.teamName, runId: input.runId, capabilitySnapshotId: null, behaviorFingerprint: null, reason: 'launch_checkpoint', writes: [ { descriptor, data: { sessions }, }, ], }); } catch (error) { if ( isFileLockTimeoutError(error) && (await this.hasCommittedOpenCodeRuntimeBootstrapSessionEvidence(input)) ) { return; } throw error; } if (!(await this.hasCommittedOpenCodeRuntimeBootstrapSessionEvidence(input))) { throw new Error( `OpenCode bootstrap session evidence write did not verify for ${input.memberName}` ); } } private async hasCommittedOpenCodeRuntimeBootstrapSessionEvidence(input: { teamName: string; runId: string; laneId: string; memberName: string; runtimeSessionId: string; source?: OpenCodeBootstrapEvidenceSource; appManagedBootstrapCandidate?: OpenCodeAppManagedBootstrapCandidate; }): Promise { const evidence = await readCommittedOpenCodeBootstrapSessionEvidence({ teamsBasePath: getTeamsBasePath(), teamName: input.teamName, laneId: input.laneId, }).catch(() => null); if (!evidence?.committed) { return false; } if (evidence.activeRunId && evidence.activeRunId.trim() !== input.runId) { return false; } return evidence.sessions.some((session) => { if ( session.id !== input.runtimeSessionId || session.runId !== input.runId || !namesMatchCaseInsensitive(session.memberName, input.memberName) ) { return false; } if (input.source && session.source !== input.source) { return false; } if (input.source === 'app_managed_bootstrap' && input.appManagedBootstrapCandidate) { const candidate = session.appManagedBootstrapCandidate; return ( candidate?.runtimeSessionId === input.appManagedBootstrapCandidate.runtimeSessionId && candidate.messageID === input.appManagedBootstrapCandidate.messageID && candidate.contextHash === input.appManagedBootstrapCandidate.contextHash && candidate.briefingHash === input.appManagedBootstrapCandidate.briefingHash ); } return true; }); } private getOpenCodeAppMcpTransportMismatchDiagnostic( session: OpenCodeCommittedBootstrapSessionRecord ): string | null { const committedHash = session.appMcpTransportHash?.trim(); const currentHash = getCurrentAgentTeamsMcpHttpTransportEvidence()?.urlHash?.trim(); if (!committedHash || !currentHash || committedHash === currentHash) { return null; } return `opencode_app_mcp_transport_changed:${committedHash}->${currentHash}`; } private async stampOpenCodeAppMcpTransportEvidenceIfMissing( session: OpenCodeCommittedBootstrapSessionRecord, options: { overwriteExistingHash?: boolean; runtimeSessionId?: string | null; } = {} ): Promise { const overwriteExistingHash = options.overwriteExistingHash === true; const runtimeSessionId = options.runtimeSessionId?.trim() || null; if (session.appMcpTransportHash?.trim() && !overwriteExistingHash) { return; } const appMcpTransportEvidence = getCurrentAgentTeamsMcpHttpTransportEvidence(); if (!appMcpTransportEvidence) { return; } const descriptor = OPENCODE_RUNTIME_STORE_DESCRIPTORS.find( (candidate) => candidate.schemaName === 'opencode.sessionStore' ); if (!descriptor) { return; } try { const manifestPath = getOpenCodeRuntimeManifestPath( getTeamsBasePath(), session.teamName, session.laneId ); const runtimeDirectory = path.dirname(manifestPath); const sessionStorePath = path.join(runtimeDirectory, descriptor.relativePath); const existingSessions = await this.readOpenCodeRuntimeSessionStore(sessionStorePath); let changed = false; const sessions = existingSessions .filter((record) => { if (!runtimeSessionId || runtimeSessionId === session.id) { return true; } const recordId = typeof record.id === 'string' ? record.id : ''; const recordRunId = typeof record.runId === 'string' ? record.runId : null; const recordLaneId = typeof record.laneId === 'string' ? record.laneId : ''; const recordMemberName = typeof record.memberName === 'string' ? record.memberName : ''; return !( recordId === runtimeSessionId && recordRunId === session.runId && recordLaneId === session.laneId && namesMatchCaseInsensitive(recordMemberName, session.memberName) ); }) .map((record) => { const recordId = typeof record.id === 'string' ? record.id : ''; const recordRunId = typeof record.runId === 'string' ? record.runId : null; const recordLaneId = typeof record.laneId === 'string' ? record.laneId : ''; const recordMemberName = typeof record.memberName === 'string' ? record.memberName : ''; const hasTransportHash = typeof record.appMcpTransportHash === 'string' && record.appMcpTransportHash.trim().length > 0; if ( recordId !== session.id || recordRunId !== session.runId || recordLaneId !== session.laneId || !namesMatchCaseInsensitive(recordMemberName, session.memberName) || (hasTransportHash && !overwriteExistingHash) ) { return record; } changed = true; return { ...record, ...(runtimeSessionId ? { id: runtimeSessionId } : {}), appMcpTransportHash: appMcpTransportEvidence.urlHash, appMcpTransportEvidence, }; }); if (!changed) { return; } const manifestStore = createRuntimeStoreManifestStore({ filePath: manifestPath, teamName: session.teamName, lockOptions: OPENCODE_BOOTSTRAP_EVIDENCE_LOCK_OPTIONS, }); const receiptStore = createRuntimeStoreReceiptStore({ filePath: path.join(runtimeDirectory, 'opencode-runtime-receipts.json'), lockOptions: OPENCODE_BOOTSTRAP_EVIDENCE_LOCK_OPTIONS, }); const writer = new RuntimeStoreBatchWriter(runtimeDirectory, manifestStore, receiptStore); await writer.writeBatch({ teamName: session.teamName, runId: session.runId, capabilitySnapshotId: null, behaviorFingerprint: null, reason: 'delivery_commit', writes: [ { descriptor, data: { sessions }, }, ], }); } catch (error) { logger.warn( `[${session.teamName}] Failed to stamp OpenCode app MCP transport evidence for ${session.memberName}: ${getErrorMessage(error)}` ); } } private async findDeliverableOpenCodeRuntimeBootstrapSessionEvidence(input: { teamName: string; runId: string | null; laneId: string; memberName: string; }): Promise { const evidence = await readCommittedOpenCodeBootstrapSessionEvidence({ teamsBasePath: getTeamsBasePath(), teamName: input.teamName, laneId: input.laneId, }).catch(() => null); if (!evidence?.committed) { return null; } const activeRunId = evidence.activeRunId?.trim() || null; if (activeRunId !== input.runId) { return null; } return ( evidence.sessions.find( (session) => session.runId === input.runId && namesMatchCaseInsensitive(session.memberName, input.memberName) ) ?? null ); } private async hasDeliverableOpenCodeRuntimeBootstrapSessionEvidence(input: { teamName: string; runId: string | null; laneId: string; memberName: string; }): Promise { return (await this.findDeliverableOpenCodeRuntimeBootstrapSessionEvidence(input)) != null; } private async readOpenCodeRuntimeSessionStore( filePath: string ): Promise[]> { let raw: string; try { raw = await fs.promises.readFile(filePath, 'utf8'); } catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') { return []; } throw error; } try { const parsed = JSON.parse(raw) as unknown; const record = parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? (parsed as Record) : null; const data = record && Object.prototype.hasOwnProperty.call(record, 'data') ? record.data : record; const sessions = data && typeof data === 'object' && !Array.isArray(data) ? (data as Record).sessions : null; return Array.isArray(sessions) ? sessions.filter( (session): session is Record => Boolean(session) && typeof session === 'object' && !Array.isArray(session) ) : []; } catch { return []; } } private mergeOpenCodeRuntimeSessionRecords( existingSessions: Record[], session: Record ): Record[] { const sessionId = typeof session.id === 'string' ? session.id.trim() : ''; const memberName = typeof session.memberName === 'string' ? session.memberName.trim() : ''; const runId = typeof session.runId === 'string' ? session.runId.trim() : ''; const laneId = typeof session.laneId === 'string' ? session.laneId.trim() : ''; const filtered = existingSessions.filter((candidate) => { const candidateId = typeof candidate.id === 'string' ? candidate.id.trim() : ''; if (sessionId && candidateId === sessionId) { return false; } const sameMember = memberName && runId && laneId && candidate.memberName === memberName && candidate.runId === runId && candidate.laneId === laneId; return !sameMember; }); return [...filtered, session]; } private async resolveOpenCodeRuntimeBootstrapCheckinIdempotency(input: { teamName: string; runId: string; memberName: string; runtimeSessionId: string; }): Promise< | { state: 'new'; previousMember?: PersistedTeamLaunchMemberState; } | { state: 'duplicate'; previousMember: PersistedTeamLaunchMemberState; } | { state: 'conflict'; previousMember: PersistedTeamLaunchMemberState; existingRuntimeSessionId: string; } > { const snapshot = await this.launchStateStore.read(input.teamName); const previousMember = snapshot?.members[input.memberName]; if (!previousMember) { return { state: 'new' }; } const existingRuntimeSessionId = previousMember.runtimeSessionId?.trim(); const existingRuntimeRunId = typeof previousMember.runtimeRunId === 'string' ? previousMember.runtimeRunId.trim() : ''; const hasAcceptedBootstrap = previousMember.bootstrapConfirmed === true || previousMember.livenessKind === 'confirmed_bootstrap' || previousMember.launchState === 'confirmed_alive'; if (!hasAcceptedBootstrap || !existingRuntimeSessionId) { return { state: 'new', previousMember }; } if (existingRuntimeRunId && existingRuntimeRunId !== input.runId) { return { state: 'new', previousMember }; } if (existingRuntimeSessionId === input.runtimeSessionId) { return { state: 'duplicate', previousMember }; } if (!existingRuntimeRunId) { return { state: 'new', previousMember }; } return { state: 'conflict', previousMember, existingRuntimeSessionId, }; } private async assertOpenCodeRuntimeMemberCheckinAllowed(input: { teamName: string; memberName: string; previousMember?: PersistedTeamLaunchMemberState; }): Promise { const config = await this.readConfigForStrictDecision(input.teamName).catch(() => null); const metaMembers = await this.membersMetaStore.getMembers(input.teamName).catch(() => []); const configuredMember = this.resolveEffectiveConfiguredMember( config?.members ?? [], metaMembers, input.memberName ); if (configuredMember?.removedAt != null) { throw new RuntimeStaleEvidenceError( `Rejected OpenCode bootstrap check-in for removed member "${input.memberName}"`, 'run_mismatch', 'bootstrap_checkin', null ); } if (!configuredMember && !input.previousMember) { throw new RuntimeStaleEvidenceError( `Rejected OpenCode bootstrap check-in for unconfigured member "${input.memberName}"`, 'run_mismatch', 'bootstrap_checkin', null ); } } async deliverOpenCodeRuntimeMessage(raw: unknown): Promise { const payload = asRuntimeRecord(raw); const teamName = requireRuntimeString(payload.teamName, 'teamName'); const runId = requireRuntimeString(payload.runId, 'runId'); const fromMemberName = requireRuntimeString(payload.fromMemberName, 'fromMemberName'); const laneId = await this.resolveOpenCodeRuntimeLaneId({ teamName, runId, memberName: fromMemberName, }); await this.assertOpenCodeRuntimeEvidenceAccepted({ teamName, runId, laneId, evidenceKind: 'delivery_call', }); const delivery = this.createOpenCodeRuntimeDeliveryService(teamName, laneId); const ack = await delivery.deliver({ ...payload, teamName, runId, providerId: 'opencode', createdAt: normalizeRuntimeIso(payload.createdAt), }); if (!ack.ok) { throw new Error(`OpenCode runtime delivery rejected: ${ack.reason}`); } return { ok: true, providerId: 'opencode', teamName, runId, state: ack.delivered ? 'delivered' : 'duplicate', idempotencyKey: ack.idempotencyKey, location: ack.location, diagnostics: ack.reason ? [ack.reason] : [], observedAt: normalizeRuntimeIso(payload.createdAt), }; } async recordOpenCodeRuntimeTaskEvent(raw: unknown): Promise { const payload = asRuntimeRecord(raw); const teamName = requireRuntimeString(payload.teamName, 'teamName'); const runId = requireRuntimeString(payload.runId, 'runId'); const memberName = requireRuntimeString(payload.memberName, 'memberName'); const taskId = requireRuntimeString(payload.taskId, 'taskId'); const event = requireRuntimeString(payload.event, 'event'); const idempotencyKey = requireRuntimeString(payload.idempotencyKey, 'idempotencyKey'); const runtimeSessionId = optionalRuntimeString(payload.runtimeSessionId); const observedAt = normalizeRuntimeIso(payload.createdAt); const laneId = await this.resolveOpenCodeRuntimeLaneId({ teamName, runId, memberName }); await this.assertOpenCodeRuntimeEvidenceAccepted({ teamName, runId, laneId, evidenceKind: 'delivery_call', }); const writeResult = await this.openCodeTaskLogAttributionStore.upsertTaskRecord(teamName, { taskId, memberName, scope: 'member_session_window', ...(runtimeSessionId ? { sessionId: runtimeSessionId } : {}), since: observedAt, source: 'launch_runtime', }); this.teamChangeEmitter?.({ type: 'task-log-change', teamName, runId, taskId, detail: `opencode-runtime-task-event:${event}`, taskSignalKind: 'log', }); return { ok: true, providerId: 'opencode', teamName, runId, state: 'recorded', memberName, ...(runtimeSessionId ? { runtimeSessionId } : {}), idempotencyKey, diagnostics: [writeResult], observedAt, }; } async recordOpenCodeRuntimeHeartbeat(raw: unknown): Promise { const payload = asRuntimeRecord(raw); const teamName = requireRuntimeString(payload.teamName, 'teamName'); const runId = requireRuntimeString(payload.runId, 'runId'); const memberName = requireRuntimeString(payload.memberName, 'memberName'); const runtimeSessionId = requireRuntimeString(payload.runtimeSessionId, 'runtimeSessionId'); const observedAt = normalizeRuntimeIso(payload.observedAt); const laneId = await this.resolveOpenCodeRuntimeLaneId({ teamName, runId, memberName }); await this.assertOpenCodeRuntimeEvidenceAccepted({ teamName, runId, laneId, evidenceKind: 'heartbeat', }); await this.updateOpenCodeRuntimeMemberLiveness({ teamName, runId, memberName, runtimeSessionId, observedAt, diagnostics: undefined, metadata: parseRuntimeToolMetadata(payload.metadata), reason: `OpenCode runtime heartbeat accepted${optionalRuntimeString(payload.status) ? ` (${optionalRuntimeString(payload.status)})` : ''}`, }); return { ok: true, providerId: 'opencode', teamName, runId, state: 'accepted', memberName, runtimeSessionId, diagnostics: [], observedAt, }; } private async assertOpenCodeRuntimeEvidenceAccepted(input: { teamName: string; runId: string; laneId: string; evidenceKind: RuntimeEvidenceKind; }): Promise { const store = createRuntimeRunTombstoneStore({ filePath: getOpenCodeRuntimeRunTombstonesPath( getTeamsBasePath(), input.teamName, input.laneId ), }); await store.assertEvidenceAccepted({ teamName: input.teamName, runId: input.runId, currentRunId: await this.resolveCurrentOpenCodeRuntimeRunId(input.teamName, input.laneId), evidenceKind: input.evidenceKind, }); } private async updateOpenCodeRuntimeMemberLiveness(input: { teamName: string; runId: string; memberName: string; runtimeSessionId: string; observedAt: string; diagnostics: unknown; metadata?: RuntimeToolMetadata; reason: string; }): Promise { const trackedUpdate = this.applyOpenCodeRuntimeBootstrapCheckinToTrackedRun(input); if (trackedUpdate) { await this.persistLaunchStateSnapshot( trackedUpdate.run, this.getMixedSecondaryLaunchPhase(trackedUpdate.run) ); this.invalidateRuntimeSnapshotCaches(input.teamName); if (trackedUpdate.changed) { this.emitMemberSpawnChange(trackedUpdate.run, input.memberName); } return; } const previous = await this.launchStateStore.read(input.teamName); const expectedMembers = previous ? this.getPersistedLaunchMemberNames(previous) : this.readPersistedRuntimeMembers(input.teamName) .map((member) => (typeof member.name === 'string' ? member.name.trim() : '')) .filter((name) => name.length > 0 && name !== 'user' && !isLeadMember({ name })); const previousMember = previous?.members[input.memberName]; const previousRuntimeRunId = typeof previousMember?.runtimeRunId === 'string' ? previousMember.runtimeRunId.trim() : ''; const sameRuntimeRun = previousRuntimeRunId.length > 0 && previousRuntimeRunId === input.runId; const shouldEmitMemberSpawnChange = this.shouldEmitOpenCodeRuntimeLivenessMemberSpawnChange({ previousMember, runtimeRunId: input.runId, runtimeSessionId: input.runtimeSessionId, runtimePid: input.metadata?.runtimePid, }); const runtimePid = input.metadata?.runtimePid ?? (sameRuntimeRun ? previousMember?.runtimePid : undefined); const pidSource = input.metadata?.runtimePid ? ('runtime_bootstrap' as const) : sameRuntimeRun ? previousMember?.pidSource : undefined; const persistedIdentity = this.resolvePersistedRuntimeMemberIdentity({ teamName: input.teamName, memberName: input.memberName, previousMember, }); const nextMember: PersistedTeamLaunchMemberState = { ...persistedIdentity, ...(previousMember ?? {}), name: input.memberName, launchState: 'confirmed_alive', agentToolAccepted: true, runtimeAlive: true, bootstrapConfirmed: true, hardFailure: false, bootstrapStalled: undefined, runtimePid, runtimeRunId: input.runId, runtimeSessionId: input.runtimeSessionId, livenessKind: 'confirmed_bootstrap', pidSource, runtimeDiagnostic: input.reason, runtimeDiagnosticSeverity: 'info', runtimeLastSeenAt: input.observedAt, firstSpawnAcceptedAt: previousMember?.firstSpawnAcceptedAt ?? input.observedAt, lastHeartbeatAt: input.observedAt, lastRuntimeAliveAt: input.observedAt, lastEvaluatedAt: input.observedAt, sources: { ...(previousMember?.sources ?? {}), nativeHeartbeat: true, processAlive: true, }, diagnostics: mergeRuntimeDiagnostics( previousMember?.diagnostics, [ ...normalizeRuntimeStringArray(input.diagnostics), ...buildRuntimeToolMetadataDiagnostics(input.metadata), ], input.reason ), }; const snapshot = createPersistedLaunchSnapshot({ teamName: input.teamName, expectedMembers: [...new Set([...expectedMembers, input.memberName])], leadSessionId: previous?.leadSessionId, launchPhase: previous?.launchPhase ?? 'active', members: { ...(previous?.members ?? {}), [input.memberName]: nextMember, }, updatedAt: input.observedAt, }); await this.writeLaunchStateSnapshot(input.teamName, snapshot); if (shouldEmitMemberSpawnChange) { this.teamChangeEmitter?.({ type: 'member-spawn', teamName: input.teamName, runId: input.runId, detail: input.memberName, }); } } private applyOpenCodeRuntimeBootstrapCheckinToTrackedRun(input: { teamName: string; runId: string; memberName: string; runtimeSessionId: string; observedAt: string; diagnostics: unknown; metadata?: RuntimeToolMetadata; reason: string; }): { run: ProvisioningRun; changed: boolean } | null { const trackedRunId = this.getTrackedRunId(input.teamName); const run = trackedRunId ? this.runs.get(trackedRunId) : undefined; if (!run || run.processKilled || run.cancelRequested) { return null; } const lane = (run.mixedSecondaryLanes ?? []).find((candidate) => { if (candidate.providerId !== 'opencode') { return false; } if (!matchesTeamMemberIdentity(candidate.member.name, input.memberName)) { return false; } return !candidate.runId || candidate.runId === input.runId; }); if (!lane) { return null; } const runtimePid = input.metadata?.runtimePid; const runtimeDiagnostics = mergeRuntimeDiagnostics( lane.result?.members[input.memberName]?.diagnostics ?? lane.diagnostics, [ ...normalizeRuntimeStringArray(input.diagnostics), ...buildRuntimeToolMetadataDiagnostics(input.metadata), 'opencode_bootstrap_evidence_committed', ], input.reason ); const evidence: TeamRuntimeMemberLaunchEvidence = { memberName: input.memberName, providerId: 'opencode', launchState: 'confirmed_alive', agentToolAccepted: true, runtimeAlive: true, bootstrapConfirmed: true, hardFailure: false, sessionId: input.runtimeSessionId, backendType: 'process', ...(runtimePid ? { runtimePid, pidSource: 'runtime_bootstrap' as const } : {}), livenessKind: 'confirmed_bootstrap', runtimeDiagnostic: input.reason, runtimeDiagnosticSeverity: 'info', diagnostics: runtimeDiagnostics ?? [input.reason], }; const previousLaneState = lane.state; const previousLaneRunId = lane.runId; const previousLaneMember = lane.result?.members[input.memberName]; lane.runId = input.runId; lane.state = 'finished'; lane.diagnostics = runtimeDiagnostics ?? lane.diagnostics; lane.result = { ...(lane.result ?? { runId: input.runId, teamName: input.teamName, launchPhase: 'finished' as const, teamLaunchState: 'partial_pending' as const, members: {}, warnings: lane.warnings, diagnostics: [], }), runId: input.runId, teamName: input.teamName, launchPhase: 'finished', members: { ...(lane.result?.members ?? {}), [input.memberName]: evidence, }, warnings: lane.result?.warnings ?? lane.warnings, diagnostics: runtimeDiagnostics ?? lane.result?.diagnostics ?? lane.diagnostics, }; lane.result.teamLaunchState = summarizeRuntimeLaunchResultMembers(lane.result.members); const previousStatus = run.memberSpawnStatuses.get(input.memberName) ?? createInitialMemberSpawnStatusEntry(); const nextStatus: MemberSpawnStatusEntry = { ...previousStatus, status: 'online', launchState: 'confirmed_alive', error: undefined, hardFailureReason: undefined, skippedForLaunch: undefined, skipReason: undefined, skippedAt: undefined, livenessSource: 'heartbeat', agentToolAccepted: true, runtimeAlive: true, bootstrapConfirmed: true, hardFailure: false, bootstrapStalled: undefined, pendingPermissionRequestIds: undefined, firstSpawnAcceptedAt: previousStatus.firstSpawnAcceptedAt ?? input.observedAt, lastHeartbeatAt: input.observedAt, runtimeModel: lane.member.model, livenessKind: 'confirmed_bootstrap', runtimeDiagnostic: input.reason, runtimeDiagnosticSeverity: 'info', livenessLastCheckedAt: input.observedAt, updatedAt: input.observedAt, }; this.syncMemberTaskActivityForRuntimeTransition( run, input.memberName, previousStatus, nextStatus, input.observedAt ); run.memberSpawnStatuses.set(input.memberName, nextStatus); run.pendingMemberRestarts?.delete(input.memberName); this.syncMemberLaunchGraceCheck(run, input.memberName, nextStatus); const statusChanged = previousStatus.status !== nextStatus.status || previousStatus.launchState !== nextStatus.launchState || previousStatus.bootstrapConfirmed !== nextStatus.bootstrapConfirmed || previousStatus.runtimeAlive !== nextStatus.runtimeAlive || previousStatus.hardFailure !== nextStatus.hardFailure || previousStatus.livenessKind !== nextStatus.livenessKind; const laneChanged = previousLaneState !== lane.state || previousLaneRunId !== lane.runId || previousLaneMember?.sessionId !== evidence.sessionId || previousLaneMember?.launchState !== evidence.launchState || previousLaneMember?.bootstrapConfirmed !== evidence.bootstrapConfirmed; return { run, changed: statusChanged || laneChanged }; } private shouldEmitOpenCodeRuntimeLivenessMemberSpawnChange(input: { previousMember?: PersistedTeamLaunchMemberState; runtimeRunId: string; runtimeSessionId: string; runtimePid?: number; }): boolean { const previous = input.previousMember; if (!previous) { return true; } const previousRuntimeRunId = typeof previous.runtimeRunId === 'string' ? previous.runtimeRunId.trim() : ''; const previousRuntimeSessionId = typeof previous.runtimeSessionId === 'string' ? previous.runtimeSessionId.trim() : ''; if ( previousRuntimeRunId !== input.runtimeRunId || previousRuntimeSessionId !== input.runtimeSessionId ) { return true; } if ( input.runtimePid !== undefined && (previous.runtimePid === undefined || previous.runtimePid !== input.runtimePid) ) { return true; } return ( previous.launchState !== 'confirmed_alive' || previous.runtimeAlive !== true || previous.bootstrapConfirmed !== true || previous.hardFailure === true ); } private resolvePersistedRuntimeMemberIdentity(params: { teamName: string; memberName: string; previousMember?: PersistedTeamLaunchMemberState; }): Partial { if (params.previousMember) { return { providerId: params.previousMember.providerId, providerBackendId: params.previousMember.providerBackendId, model: params.previousMember.model, effort: params.previousMember.effort, selectedFastMode: params.previousMember.selectedFastMode, resolvedFastMode: params.previousMember.resolvedFastMode, laneId: params.previousMember.laneId, laneKind: params.previousMember.laneKind, laneOwnerProviderId: params.previousMember.laneOwnerProviderId, launchIdentity: params.previousMember.launchIdentity, }; } const trackedRunId = this.getTrackedRunId(params.teamName); const trackedRun = trackedRunId ? this.runs.get(trackedRunId) : null; const secondaryLane = trackedRun?.mixedSecondaryLanes?.find( (lane) => lane.member.name.trim() === params.memberName ); if (secondaryLane) { return { providerId: 'opencode', model: secondaryLane.member.model, effort: secondaryLane.member.effort, laneId: secondaryLane.laneId, laneKind: 'secondary', laneOwnerProviderId: 'opencode', }; } const primaryMember = trackedRun?.effectiveMembers?.find( (member) => member.name.trim() === params.memberName ); if (!primaryMember) { return {}; } const laneIdentity = buildPlannedMemberLaneIdentity({ leadProviderId: resolveTeamProviderId(trackedRun?.request.providerId), member: { name: primaryMember.name, providerId: normalizeOptionalTeamProviderId(primaryMember.providerId), }, }); const providerId = normalizeOptionalTeamProviderId(primaryMember.providerId) ?? resolveTeamProviderId(trackedRun?.request.providerId); return { providerId, providerBackendId: migrateProviderBackendId( providerId, primaryMember.providerBackendId ?? trackedRun?.request.providerBackendId ), model: primaryMember.model, effort: primaryMember.effort, selectedFastMode: primaryMember.fastMode ?? trackedRun?.request.fastMode, laneId: laneIdentity.laneId, laneKind: laneIdentity.laneKind, laneOwnerProviderId: laneIdentity.laneOwnerProviderId, }; } private createOpenCodeRuntimeDeliveryService( teamName: string, laneId: string ): RuntimeDeliveryService { const journal = createRuntimeDeliveryJournalStore({ filePath: getOpenCodeLaneScopedRuntimeFilePath({ teamsBasePath: getTeamsBasePath(), teamName, laneId, fileName: 'opencode-delivery-journal.json', }), }); return new RuntimeDeliveryService( { getCurrentRunId: async (candidateTeamName) => this.resolveCurrentOpenCodeRuntimeRunId(candidateTeamName, laneId), }, journal, new RuntimeDeliveryDestinationRegistry(this.createOpenCodeRuntimeDeliveryPorts()), { append: async (event) => { logger.warn(`[${event.teamName}] ${event.message}`); }, }, { emit: (event) => { this.teamChangeEmitter?.({ type: event.type as TeamChangeEvent['type'], teamName: event.teamName, detail: typeof event.data?.detail === 'string' ? event.data.detail : undefined, }); }, } ); } private createOpenCodePromptDeliveryLedger(teamName: string, laneId: string) { return createOpenCodePromptDeliveryLedgerStore({ filePath: getOpenCodeLaneScopedRuntimeFilePath({ teamsBasePath: getTeamsBasePath(), teamName, laneId, fileName: 'opencode-prompt-delivery-ledger.json', }), }); } async getOpenCodeRuntimeDeliveryStatus( teamName: string, messageId: string ): Promise { const normalizedMessageId = messageId.trim(); if (!normalizedMessageId) { return null; } const laneIndex = await readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName).catch( () => null ); const laneIds = [ ...new Set( Object.values(laneIndex?.lanes ?? {}) .map((entry) => entry.laneId.trim()) .filter(Boolean) ), ]; for (const laneId of laneIds) { const records = await this.createOpenCodePromptDeliveryLedger(teamName, laneId) .list() .catch(() => []); const record = records.find((candidate) => candidate.inboxMessageId === normalizedMessageId); if (record) { const { record: latestRecord, decision } = await this.decideOpenCodeRuntimeDeliveryUserFacingAdvisory(record); return this.toOpenCodeRuntimeDeliveryStatus(latestRecord, decision); } } return null; } private buildOpenCodePromptDeliveryActiveBusyStatus(input: { teamName: string; memberName: string; retryAfterIso: string; activeRecord: OpenCodePromptDeliveryLedgerRecord; }): { busy: true; reason: string; retryAfterIso: string; activeMessageId: string; activeMessageKind: string | null; } { const nextAttemptMs = input.activeRecord.nextAttemptAt ? Date.parse(input.activeRecord.nextAttemptAt) : NaN; this.scheduleOpenCodeMemberInboxDeliveryWake({ teamName: input.teamName, memberName: input.memberName, messageId: input.activeRecord.inboxMessageId, delayMs: Number.isFinite(nextAttemptMs) ? Math.max(500, nextAttemptMs - Date.now()) : 500, }); return { busy: true, reason: `opencode_prompt_delivery_active:${input.activeRecord.messageKind ?? 'default'}`, retryAfterIso: input.activeRecord.nextAttemptAt ?? input.retryAfterIso, activeMessageId: input.activeRecord.inboxMessageId, activeMessageKind: input.activeRecord.messageKind, }; } private async tryGetActiveOpenCodePromptDeliveryRecord(input: { teamName: string; memberName: string; }): Promise { const identity = await this.resolveOpenCodeMemberDeliveryIdentity( input.teamName, input.memberName ).catch(() => null); if (!identity?.ok) { return null; } const laneIndex = await readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), input.teamName).catch( () => null ); if (laneIndex?.lanes[identity.laneId]?.state !== 'active') { const recovered = await this.tryRecoverOpenCodeRuntimeLaneForConfiguredMemberAndVerifyActive({ teamName: input.teamName, memberName: identity.canonicalMemberName, laneId: identity.laneId, }); if (!recovered) { return null; } } return await this.createOpenCodePromptDeliveryLedger(input.teamName, identity.laneId) .getActiveForMember({ teamName: input.teamName, memberName: identity.canonicalMemberName, laneId: identity.laneId, }) .catch(() => null); } async getOpenCodeMemberDeliveryBusyStatus(input: { teamName: string; memberName: string; nowIso: string; workSyncIntent?: 'agenda_sync' | 'review_pickup'; workSyncIntentKey?: string; taskRefs?: TaskRef[]; }): Promise<{ busy: boolean; reason?: string; retryAfterIso?: string; activeMessageId?: string; activeMessageKind?: string | null; }> { if (!(await this.isOpenCodeRuntimeRecipient(input.teamName, input.memberName))) { return { busy: false }; } const nowMs = Date.parse(input.nowIso); const retryAfterIso = new Date( (Number.isFinite(nowMs) ? nowMs : Date.now()) + 60_000 ).toISOString(); let inboxMessages: Awaited>; try { inboxMessages = await this.inboxReader.getMessagesFor(input.teamName, input.memberName); } catch { return { busy: true, reason: 'opencode_inbox_read_failed', retryAfterIso, }; } const foregroundMessages = inboxMessages.filter( (message) => message.messageKind !== 'member_work_sync_nudge' ); const agendaSyncRecoveryBypassMessageIds = await this.getOpenCodeAgendaSyncRecoveryBypassMessageIds({ teamName: input.teamName, memberName: input.memberName, workSyncIntent: input.workSyncIntent, taskRefs: input.taskRefs, foregroundMessages, }); const blockingForegroundMessages = foregroundMessages.filter((message) => { const messageId = typeof message.messageId === 'string' ? message.messageId.trim() : ''; return ( !agendaSyncRecoveryBypassMessageIds.has(messageId) && !this.isCurrentReviewPickupRequestForegroundMessage(message, input) && !this.isCurrentProofMissingRecoveryForegroundMessage(message, input) ); }); const unreadForeground = blockingForegroundMessages.find( (message) => !message.read && typeof message.text === 'string' && message.text.trim().length > 0 && this.hasStableMessageId(message) ); if (unreadForeground?.messageId) { const activeRecord = await this.tryGetActiveOpenCodePromptDeliveryRecord({ teamName: input.teamName, memberName: input.memberName, }); if (activeRecord) { return this.buildOpenCodePromptDeliveryActiveBusyStatus({ teamName: input.teamName, memberName: input.memberName, retryAfterIso, activeRecord, }); } this.scheduleOpenCodeMemberInboxDeliveryWake({ teamName: input.teamName, memberName: input.memberName, messageId: unreadForeground.messageId, delayMs: 500, }); return { busy: true, reason: 'opencode_foreground_inbox_unread', retryAfterIso, activeMessageId: unreadForeground.messageId, activeMessageKind: unreadForeground.messageKind ?? null, }; } const recentForeground = blockingForegroundMessages.find((message) => { const timestampMs = Date.parse(message.timestamp); return Number.isFinite(timestampMs) && Number.isFinite(nowMs) && nowMs - timestampMs < 60_000; }); if (recentForeground?.messageId) { return { busy: true, reason: 'opencode_foreground_inbox_recent', retryAfterIso, activeMessageId: recentForeground.messageId, activeMessageKind: recentForeground.messageKind ?? null, }; } const identity = await this.resolveOpenCodeMemberDeliveryIdentity( input.teamName, input.memberName ); if (!identity.ok) { return { busy: true, reason: identity.reason, retryAfterIso }; } const laneIndex = await readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), input.teamName).catch( () => null ); if (!laneIndex) { return { busy: true, reason: 'opencode_lane_index_unavailable', retryAfterIso }; } if ( laneIndex.lanes[identity.laneId]?.state !== 'active' && !(await this.tryRecoverOpenCodeRuntimeLaneForConfiguredMemberAndVerifyActive({ teamName: input.teamName, memberName: identity.canonicalMemberName, laneId: identity.laneId, }).catch(() => false)) ) { return { busy: true, reason: 'opencode_no_active_lane', retryAfterIso }; } let activeRecord: OpenCodePromptDeliveryLedgerRecord | null; try { activeRecord = await this.createOpenCodePromptDeliveryLedger( input.teamName, identity.laneId ).getActiveForMember({ teamName: input.teamName, memberName: identity.canonicalMemberName, laneId: identity.laneId, }); } catch { return { busy: true, reason: 'opencode_prompt_ledger_unavailable', retryAfterIso, }; } if (activeRecord) { return this.buildOpenCodePromptDeliveryActiveBusyStatus({ teamName: input.teamName, memberName: input.memberName, retryAfterIso, activeRecord, }); } return { busy: false }; } scheduleOpenCodeMemberInboxDeliveryWake(input: { teamName: string; memberName: string; messageId: string; delayMs?: number; }): void { const teamName = input.teamName.trim(); const memberName = input.memberName.trim(); const messageId = input.messageId.trim(); if (!teamName || !memberName || !messageId || !this.isOpenCodePromptDeliveryWatchdogEnabled()) { return; } this.scheduleOpenCodePromptDeliveryWatchdog({ teamName, memberName, messageId, delayMs: Math.max(0, input.delayMs ?? 500), }); } buildOpenCodeRuntimeDeliveryUserVisibleImpact(input: { delivered?: boolean; responsePending?: boolean; acceptanceUnknown?: boolean; responseState?: OpenCodeMemberInboxDelivery['responseState']; ledgerStatus?: OpenCodePromptDeliveryStatus; reason?: string; diagnostics?: string[]; queuedBehindMessageId?: string; policyImpact?: OpenCodeRuntimeDeliveryUserVisibleImpact; }): OpenCodeRuntimeDeliveryUserVisibleImpact { if (input.policyImpact) { return input.policyImpact; } if ( input.responsePending === true || input.acceptanceUnknown === true || Boolean(input.queuedBehindMessageId) ) { return { state: 'checking', reasonCode: input.reason ? classifyOpenCodeRuntimeDeliveryReasonCode(input.reason) : undefined, message: this.selectOpenCodeRuntimeDeliveryUserVisibleMessage(input), }; } if (input.delivered === false) { const reason = input.reason ?? input.diagnostics?.find((diagnostic) => diagnostic.trim()); if ( input.ledgerStatus === 'failed_terminal' && isDeferredGenericOpenCodeRuntimeDeliveryReason(reason) ) { return { state: 'checking', reasonCode: classifyOpenCodeRuntimeDeliveryReasonCode(reason), message: this.selectOpenCodeRuntimeDeliveryUserVisibleMessage(input), }; } return { state: 'error', reasonCode: classifyOpenCodeRuntimeDeliveryReasonCode(reason), message: this.selectOpenCodeRuntimeDeliveryUserVisibleMessage(input), }; } return input.policyImpact ?? { state: 'none' }; } private selectOpenCodeRuntimeDeliveryUserVisibleMessage(input: { reason?: string; diagnostics?: string[]; }): string | undefined { const attachmentMessage = this.selectOpenCodeAttachmentDeliveryUserVisibleMessage(input); if (attachmentMessage) { return attachmentMessage; } return input.reason; } private selectOpenCodeAttachmentDeliveryUserVisibleMessage(input: { reason?: string; diagnostics?: string[]; }): string | undefined { const reason = input.reason?.trim(); const isAttachmentFailure = this.isOpenCodeAttachmentDeliveryFailureReason(reason) || input.diagnostics?.some((diagnostic) => diagnostic.trim().startsWith('opencode_attachment_delivery_prepare_failed:') ) === true; if (!isAttachmentFailure) { return undefined; } const diagnosticMessage = input.diagnostics ?.map((diagnostic) => diagnostic.trim()) .find((diagnostic) => diagnostic.startsWith('opencode_attachment_delivery_prepare_failed:')); const strippedDiagnostic = diagnosticMessage ?.slice('opencode_attachment_delivery_prepare_failed:'.length) .trim(); if (strippedDiagnostic) { return strippedDiagnostic; } if (reason === 'attachment_model_unsupported') { return 'This OpenCode model is not verified for image attachments. Choose a vision-capable model or remove the image.'; } if (reason === 'attachment_type_unsupported') { return 'This OpenCode model cannot receive this attachment type. Remove the attachment or choose a supported image model.'; } if (reason === 'attachment_too_large') { return 'The attachment is too large for live OpenCode delivery. Reduce the image size or remove the attachment.'; } if (reason === 'attachment_artifact_missing' || reason === 'attachment_artifact_path_unsafe') { return 'The attachment file is not available for live OpenCode delivery. Reattach the file and try again.'; } if (reason === 'attachment_optimization_failed') { return 'The attachment could not be optimized for live OpenCode delivery. Try a smaller image or remove the attachment.'; } if (reason === 'attachment_provider_rejected') { return 'The OpenCode provider rejected the attachment. Choose a different model or remove the attachment.'; } if (reason === 'attachment_runtime_transport_failed') { return 'OpenCode could not transport the attachment to the runtime. Try again or remove the attachment.'; } return undefined; } private isOpenCodeAttachmentDeliveryFailureReason(reason: string | undefined): boolean { return ( reason === 'opencode_attachment_delivery_prepare_failed' || reason?.startsWith('attachment_') === true ); } private toOpenCodeRuntimeDeliveryStatus( record: OpenCodePromptDeliveryLedgerRecord, decision?: OpenCodeRuntimeDeliveryAdvisoryDecision ): OpenCodeRuntimeDeliveryStatus { const failed = record.status === 'failed_terminal'; const responded = record.status === 'responded' && Boolean(record.inboxReadCommittedAt || record.visibleReplyMessageId); const policyImpact = decision ? toOpenCodeRuntimeDeliveryUserVisibleImpact(decision) : undefined; const userVisibleImpact = this.buildOpenCodeRuntimeDeliveryUserVisibleImpact({ delivered: !failed, responsePending: !failed && !responded, acceptanceUnknown: record.acceptanceUnknown, responseState: record.responseState, ledgerStatus: record.status, reason: record.lastReason ?? undefined, diagnostics: record.diagnostics, policyImpact, }); return { messageId: record.inboxMessageId, providerId: 'opencode', attempted: true, delivered: !failed, responsePending: !failed && !responded, responseState: record.responseState, ledgerStatus: record.status, visibleReplyMessageId: record.visibleReplyMessageId ?? undefined, visibleReplyCorrelation: record.visibleReplyCorrelation ?? undefined, acceptanceUnknown: record.acceptanceUnknown, reason: record.lastReason ?? undefined, diagnostics: record.diagnostics, userVisibleImpact, }; } private createOpenCodeRuntimeDeliveryPorts(): RuntimeDeliveryDestinationPort[] { const userMessagesPort: RuntimeDeliveryDestinationPort = { kind: 'user_sent_messages', write: async ({ envelope, destinationMessageId }) => { await this.sentMessagesStore.appendMessage(envelope.teamName, { from: envelope.fromMemberName, to: 'user', text: envelope.text, timestamp: envelope.createdAt, read: true, summary: envelope.summary ?? undefined, messageId: destinationMessageId, source: 'lead_process', leadSessionId: envelope.runtimeSessionId, taskRefs: runtimeTaskRefs(envelope.teamName, envelope.taskRefs), }); return { kind: 'user_sent_messages', teamName: envelope.teamName, messageId: destinationMessageId, }; }, verify: async ({ destination, destinationMessageId }) => { if (destination.kind !== 'user_sent_messages') { return { found: false, location: null, diagnostics: ['destination kind mismatch'] }; } const messages = await this.sentMessagesStore.readMessages(destination.teamName); const found = messages.some((message) => message.messageId === destinationMessageId); return { found, location: found ? { kind: 'user_sent_messages', teamName: destination.teamName, messageId: destinationMessageId, } : null, diagnostics: [], }; }, buildChangeEvent: ({ teamName }) => ({ type: 'lead-message', teamName, data: { detail: 'opencode-runtime-delivery' }, }), }; const memberInboxPort: RuntimeDeliveryDestinationPort = { kind: 'member_inbox', write: async ({ envelope, destinationMessageId }) => { if (typeof envelope.to !== 'object' || !('memberName' in envelope.to)) { throw new Error('Runtime delivery member destination missing memberName'); } const memberName = envelope.to.memberName; await this.inboxWriter.sendMessage(envelope.teamName, { member: memberName, from: envelope.fromMemberName, to: memberName, text: envelope.text, timestamp: envelope.createdAt, messageId: destinationMessageId, summary: envelope.summary ?? undefined, source: 'inbox', leadSessionId: envelope.runtimeSessionId, taskRefs: runtimeTaskRefs(envelope.teamName, envelope.taskRefs), }); return { kind: 'member_inbox', teamName: envelope.teamName, memberName, messageId: destinationMessageId, }; }, verify: async ({ destination, destinationMessageId }) => { if (destination.kind !== 'member_inbox') { return { found: false, location: null, diagnostics: ['destination kind mismatch'] }; } const messages = await this.inboxReader.getMessagesFor( destination.teamName, destination.memberName ); const found = messages.some((message) => message.messageId === destinationMessageId); return { found, location: found ? { kind: 'member_inbox', teamName: destination.teamName, memberName: destination.memberName, messageId: destinationMessageId, } : null, diagnostics: [], }; }, buildChangeEvent: ({ teamName, location }) => ({ type: 'inbox', teamName, data: { detail: location.kind === 'member_inbox' ? `inboxes/${location.memberName}.json` : 'inboxes', }, }), }; const crossTeamPort: RuntimeDeliveryDestinationPort = { kind: 'cross_team_outbox', write: async ({ envelope, destinationMessageId }) => { if (typeof envelope.to !== 'object' || !('teamName' in envelope.to)) { throw new Error('Runtime delivery cross-team destination missing teamName'); } if (!this.crossTeamSender) { throw new Error('Cross-team sender is not configured'); } const taskRefs = runtimeTaskRefs(envelope.teamName, envelope.taskRefs); await this.crossTeamSender({ fromTeam: envelope.teamName, fromMember: envelope.fromMemberName, toTeam: envelope.to.teamName, text: envelope.text, summary: envelope.summary ?? undefined, ...(taskRefs ? { taskRefs } : {}), messageId: destinationMessageId, timestamp: envelope.createdAt, conversationId: envelope.idempotencyKey, }); return { kind: 'cross_team_outbox', fromTeamName: envelope.teamName, toTeamName: envelope.to.teamName, toMemberName: envelope.to.memberName, messageId: destinationMessageId, }; }, verify: async ({ destination, destinationMessageId }) => { if (destination.kind !== 'cross_team_outbox') { return { found: false, location: null, diagnostics: ['destination kind mismatch'] }; } const messages = await this.sentMessagesStore.readMessages(destination.fromTeamName); const found = messages.some((message) => message.messageId === destinationMessageId); return { found, location: found ? { kind: 'cross_team_outbox', fromTeamName: destination.fromTeamName, toTeamName: destination.toTeamName, toMemberName: destination.toMemberName, messageId: destinationMessageId, } : null, diagnostics: [], }; }, buildChangeEvent: ({ teamName }) => ({ type: 'inbox', teamName, data: { detail: 'cross-team-outbox' }, }), }; return [userMessagesPort, memberInboxPort, crossTeamPort]; } async recoverOpenCodeRuntimeDeliveryJournal(teamName: string): Promise<{ recovered: true }> { const laneIndex = await readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName).catch( () => ({ version: 1 as const, updatedAt: nowIso(), lanes: {}, }) ); const recoveryLaneIds = await this.getOpenCodeRuntimeRecoveryLaneIds(teamName, laneIndex.lanes); for (const laneId of recoveryLaneIds) { const journal = createRuntimeDeliveryJournalStore({ filePath: getOpenCodeLaneScopedRuntimeFilePath({ teamsBasePath: getTeamsBasePath(), teamName, laneId, fileName: 'opencode-delivery-journal.json', }), }); const reconciler = new RuntimeDeliveryReconciler( journal, new RuntimeDeliveryDestinationRegistry(this.createOpenCodeRuntimeDeliveryPorts()), { append: async (event) => { logger.warn(`[${event.teamName}] ${event.message}`); }, } ); await reconciler.reconcileTeam(teamName); } return { recovered: true }; } private async getOpenCodeRuntimeRecoveryLaneIds( teamName: string, laneIndexEntries?: Record ): Promise { const laneIds = Object.keys(laneIndexEntries ?? {}); if (laneIds.length > 0) { return laneIds; } const snapshot = await this.launchStateStore.read(teamName).catch(() => null); const snapshotLaneIds = Array.from( new Set( Object.values(snapshot?.members ?? {}) .map((member) => member?.laneOwnerProviderId === 'opencode' && typeof member.laneId === 'string' ? member.laneId.trim() : '' ) .filter((laneId) => laneId.length > 0) ) ); return snapshotLaneIds.length > 0 ? snapshotLaneIds : ['primary']; } getLeadActivityState(teamName: string): { state: 'active' | 'idle' | 'offline'; runId: string | null; } { const runId = this.getTrackedRunId(teamName); if (!runId) return { state: 'offline', runId: null }; const run = this.runs.get(runId); if (!run) { const runtimeAdapterRun = this.runtimeAdapterRunByTeam.get(teamName); const runtimeProgress = this.runtimeAdapterProgressByRunId.get(runId); if ( runtimeAdapterRun?.runId === runId && !['cancelled', 'disconnected', 'failed'].includes(runtimeProgress?.state ?? '') ) { return { state: 'idle', runId }; } return { state: 'offline', runId: null }; } if (run.processKilled || run.cancelRequested) return { state: 'offline', runId: null }; // Read-repair active lead task intervals for runs that were already active // before interval tracking was introduced or before the renderer polled state. this.syncLeadTaskActivityForState(run, run.leadActivityState, run.leadActivityState); return { state: run.leadActivityState, runId }; } getLeadContextUsage(teamName: string): { usage: LeadContextUsage | null; runId: string | null } { const runId = this.getTrackedRunId(teamName); if (!runId) return { usage: null, runId: null }; const run = this.runs.get(runId); if (!run?.leadContextUsage || run.processKilled || run.cancelRequested) { return { usage: null, runId: null }; } return { usage: this.buildLeadContextUsagePayload(run), runId, }; } private getInitialLeadContextWindowTokens(run: ProvisioningRun): number | null { const providerId = normalizeOptionalTeamProviderId(run.request.providerId); const modelName = typeof run.request.model === 'string' && run.request.model.trim().length > 0 ? run.request.model.trim() : providerId === 'anthropic' ? getAnthropicDefaultTeamModel(run.request.limitContext === true) : undefined; return inferContextWindowTokens({ providerId, modelName, limitContext: run.request.limitContext === true, }); } private buildLeadContextUsagePayload(run: ProvisioningRun): LeadContextUsage { const usage = run.leadContextUsage; if (!usage) { return { promptInputTokens: null, outputTokens: null, contextUsedTokens: null, contextWindowTokens: null, contextUsedPercent: null, promptInputSource: 'unavailable', updatedAt: new Date().toISOString(), }; } const { contextUsedTokens, contextWindowTokens } = usage; const percentRaw = contextUsedTokens !== null && contextWindowTokens !== null && contextWindowTokens > 0 ? Math.round((contextUsedTokens / contextWindowTokens) * 100) : null; return { promptInputTokens: usage.promptInputTokens, outputTokens: usage.outputTokens, contextUsedTokens: usage.contextUsedTokens, contextWindowTokens: usage.contextWindowTokens, contextUsedPercent: percentRaw === null ? null : Math.max(0, Math.min(100, percentRaw)), promptInputSource: usage.promptInputSource, updatedAt: new Date().toISOString(), }; } private updateLeadContextUsageFromUsage( run: ProvisioningRun, usage: Record, modelName: string | undefined ): void { const existingContextWindowTokens = run.leadContextUsage?.contextWindowTokens ?? this.getInitialLeadContextWindowTokens(run); const metrics = deriveContextMetrics({ usage, providerId: normalizeOptionalTeamProviderId(run.request.providerId), modelName, contextWindowTokens: existingContextWindowTokens, limitContext: run.request.limitContext === true, }); if (!run.leadContextUsage) { run.leadContextUsage = { promptInputTokens: metrics.promptInputTokens, outputTokens: metrics.outputTokens, contextUsedTokens: metrics.contextUsedTokens, contextWindowTokens: metrics.contextWindowTokens, promptInputSource: metrics.promptInputSource, lastUsageMessageId: null, lastEmittedAt: 0, }; return; } run.leadContextUsage.promptInputTokens = metrics.promptInputTokens; run.leadContextUsage.outputTokens = metrics.outputTokens; run.leadContextUsage.contextUsedTokens = metrics.contextUsedTokens; run.leadContextUsage.contextWindowTokens = metrics.contextWindowTokens ?? run.leadContextUsage.contextWindowTokens; run.leadContextUsage.promptInputSource = metrics.promptInputSource; } private isCurrentTrackedRun(run: ProvisioningRun): boolean { return this.getTrackedRunId(run.teamName) === run.runId; } private getRunTrackedCwd(run: ProvisioningRun | null | undefined): string | null { const requestCwd = typeof run?.request?.cwd === 'string' ? run.request.cwd.trim() : ''; if (requestCwd) return path.resolve(requestCwd); const spawnCwd = typeof run?.spawnContext?.cwd === 'string' ? run.spawnContext.cwd.trim() : ''; if (spawnCwd) return path.resolve(spawnCwd); return null; } private getPreCompleteCliErrorText(run: ProvisioningRun): string { const parts: string[] = []; const stderrText = run.stderrBuffer.trim(); if (stderrText) { parts.push(stderrText); } // Re-check only the parser-owned stdout carry that never became a newline-delimited message. // If it is complete JSON or clearly looks like Claude stream-json structure, ignore it here. // Otherwise treat it as trailing plaintext CLI output that should still participate in the // final auth/API failure guard. const trailingStdout = run.stdoutParserCarry.trim(); if ( trailingStdout && !run.stdoutParserCarryIsCompleteJson && !run.stdoutParserCarryLooksLikeClaudeJson ) { parts.push(trailingStdout); } return parts.join('\n').trim(); } private getLeadTaskActivityRunKey(run: ProvisioningRun): string { return `${run.teamName}\u0000${run.runId}`; } private syncLeadTaskActivityForState( run: ProvisioningRun, state: 'active' | 'idle' | 'offline', previousState: 'active' | 'idle' | 'offline', at = nowIso() ): void { const key = this.getLeadTaskActivityRunKey(run); if (state === 'active') { if (this.leadTaskActivitySyncedRunKeys.has(key)) return; const result = this.taskActivityIntervalService.resumeActiveIntervalsForMember( run.teamName, this.getRunLeadName(run), at ); if (result.failed) return; this.leadTaskActivitySyncedRunKeys.add(key); return; } const wasSynced = this.leadTaskActivitySyncedRunKeys.has(key); if (previousState !== 'active' && !wasSynced) return; const result = this.taskActivityIntervalService.pauseActiveIntervalsForMember( run.teamName, this.getRunLeadName(run), at ); if (result.failed) { this.leadTaskActivitySyncedRunKeys.add(key); return; } this.leadTaskActivitySyncedRunKeys.delete(key); } private setLeadActivity(run: ProvisioningRun, state: 'active' | 'idle' | 'offline'): void { const previousState = run.leadActivityState; const isCurrentRun = this.isCurrentTrackedRun(run); if (isCurrentRun) { this.syncLeadTaskActivityForState(run, state, previousState); } else { this.leadTaskActivitySyncedRunKeys.delete(this.getLeadTaskActivityRunKey(run)); } if (previousState === state) return; run.leadActivityState = state; if (!isCurrentRun) return; this.teamChangeEmitter?.({ type: 'lead-activity', teamName: run.teamName, runId: run.runId, detail: state, }); } private emitToolActivity(run: ProvisioningRun, payload: ToolActivityEventPayload): void { if (!this.isCurrentTrackedRun(run)) return; this.teamChangeEmitter?.({ type: 'tool-activity', teamName: run.teamName, runId: run.runId, detail: JSON.stringify(payload), }); } private startRuntimeToolActivity( run: ProvisioningRun, memberName: string, block: Record ): void { const rawId = typeof block.id === 'string' ? block.id.trim() : ''; if (!rawId) return; const toolUseId = rawId; if (run.activeToolCalls.has(toolUseId)) return; const toolName = typeof block.name === 'string' ? block.name : 'unknown'; const input = (block.input ?? {}) as Record; const activity: ActiveToolCall = { memberName, toolUseId, toolName, preview: extractToolPreview(toolName, input), startedAt: nowIso(), state: 'running', source: 'runtime', }; run.activeToolCalls.set(toolUseId, activity); this.emitToolActivity(run, { action: 'start', activity: { memberName: activity.memberName, toolUseId: activity.toolUseId, toolName: activity.toolName, preview: activity.preview, startedAt: activity.startedAt, source: activity.source, }, }); } private finishRuntimeToolActivity( run: ProvisioningRun, toolUseId: string, resultContent: unknown, isError: boolean ): void { const active = run.activeToolCalls.get(toolUseId); if (!active) return; run.activeToolCalls.delete(toolUseId); this.emitToolActivity(run, { action: 'finish', memberName: active.memberName, toolUseId, finishedAt: nowIso(), resultPreview: extractToolResultPreview(resultContent), isError, }); const spawnedMemberName = run.memberSpawnToolUseIds.get(toolUseId); if (spawnedMemberName) { run.memberSpawnToolUseIds.delete(toolUseId); const pendingRestart = run.pendingMemberRestarts.get(spawnedMemberName); if (isError) { const resultPreview = extractToolResultPreview(resultContent); this.handleMemberSpawnFailure(run, spawnedMemberName, resultPreview); } else if (active.toolName === 'Agent') { const parsedStatus = parseAgentToolResultStatus(resultContent); if (parsedStatus?.status === 'duplicate_skipped') { const detail = parsedStatus.reason === 'already_running' ? 'duplicate spawn skipped - already running' : parsedStatus.reason === 'bootstrap_pending' ? 'duplicate spawn skipped - teammate bootstrap still pending' : parsedStatus.rawReason ? `duplicate spawn skipped - unrecognized reason: ${parsedStatus.rawReason}` : 'duplicate spawn skipped - reason unavailable'; this.appendMemberBootstrapDiagnostic(run, spawnedMemberName, detail); if (pendingRestart && !parsedStatus.reason) { logger.warn( `[${run.teamName}] Restart for teammate "${spawnedMemberName}" returned duplicate_skipped without a recognized reason` ); run.pendingMemberRestarts.delete(spawnedMemberName); this.setMemberSpawnStatus( run, spawnedMemberName, 'error', buildRestartDuplicateUnconfirmedReason(spawnedMemberName, parsedStatus.rawReason) ); return; } if (parsedStatus.reason === 'already_running') { if (pendingRestart) { run.pendingMemberRestarts.delete(spawnedMemberName); this.setMemberSpawnStatus( run, spawnedMemberName, 'error', buildRestartStillRunningReason(spawnedMemberName) ); return; } this.invalidateRuntimeSnapshotCaches(run.teamName); this.setMemberSpawnStatus(run, spawnedMemberName, 'waiting'); this.appendMemberBootstrapDiagnostic( run, spawnedMemberName, 'already_running requires strong runtime verification' ); void this.reevaluateMemberLaunchStatus(run, spawnedMemberName); } else { this.setMemberSpawnStatus(run, spawnedMemberName, 'waiting'); } return; } // Agent tool_result only confirms that the runtime accepted the spawn. // The teammate becomes truly "online" only after the first inbox heartbeat. this.setMemberSpawnStatus(run, spawnedMemberName, 'waiting'); } else { this.setMemberSpawnStatus(run, spawnedMemberName, 'waiting'); } } } private handleMemberSpawnFailure( run: ProvisioningRun, memberName: string, resultPreview?: string ): void { const pendingRestart = run.pendingMemberRestarts.get(memberName); const reason = (typeof resultPreview === 'string' && resultPreview.trim().length > 0 ? resultPreview.trim() : 'Teammate spawn failed immediately after launch.') || 'Teammate spawn failed.'; const message = pendingRestart ? `Failed to restart teammate "${memberName}": ${reason}` : `Teammate "${memberName}" failed to start: ${reason}`; run.pendingMemberRestarts.delete(memberName); this.setMemberSpawnStatus(run, memberName, 'error', message); const lastIndex = run.provisioningOutputParts.length - 1; if (lastIndex < 0 || run.provisioningOutputParts[lastIndex]?.trim() !== message) { run.provisioningOutputParts.push(message); } if ( !run.provisioningComplete && (run.progress.state === 'assembling' || run.progress.state === 'configuring') ) { const progress = updateProgress(run, 'assembling', `Failed to start member ${memberName}`); run.onProgress(progress); } } private appendMemberBootstrapDiagnostic( run: ProvisioningRun, memberName: string, text: string ): void { const line = normalizeMemberDiagnosticText(memberName, text); const lastIndex = run.provisioningOutputParts.length - 1; if (lastIndex >= 0 && run.provisioningOutputParts[lastIndex]?.trim() === line) { return; } run.provisioningOutputParts.push(line); logger.info(`[${run.teamName}] [bootstrap] ${line}`); } private resetRuntimeToolActivity(run: ProvisioningRun, memberName?: string): void { if (run.activeToolCalls.size === 0) return; if (!memberName) { run.activeToolCalls.clear(); this.emitToolActivity(run, { action: 'reset' }); return; } let removed = false; for (const [toolUseId, active] of run.activeToolCalls.entries()) { if (active.memberName !== memberName) continue; run.activeToolCalls.delete(toolUseId); removed = true; } if (removed) { this.emitToolActivity(run, { action: 'reset', memberName }); } } private clearMemberSpawnToolTracking(run: ProvisioningRun, memberName: string): void { let removed = false; for (const [toolUseId, trackedMemberName] of run.memberSpawnToolUseIds.entries()) { if (trackedMemberName !== memberName) continue; run.memberSpawnToolUseIds.delete(toolUseId); removed = true; } if (removed) { this.appendMemberBootstrapDiagnostic( run, memberName, 'cleared stale spawn tool tracking before manual restart' ); } } private pauseMemberTaskActivityForRuntimeLoss( run: ProvisioningRun, memberName: string, previous: MemberSpawnStatusEntry, observedAt: string ): void { if (previous.runtimeAlive !== true) return; this.taskActivityIntervalService.pauseActiveIntervalsForMember( run.teamName, memberName, deriveTaskActivityPauseAt(previous, observedAt) ); } private syncMemberTaskActivityForRuntimeTransition( run: ProvisioningRun, memberName: string, previous: MemberSpawnStatusEntry, next: MemberSpawnStatusEntry, observedAt: string ): void { if (previous.runtimeAlive === true && next.runtimeAlive !== true) { this.pauseMemberTaskActivityForRuntimeLoss(run, memberName, previous, observedAt); } else if (previous.runtimeAlive !== true && next.runtimeAlive === true) { const nextUpdatedMs = parseOptionalIsoMs(next.updatedAt); const previousUpdatedMs = parseOptionalIsoMs(previous.updatedAt); const resumeFallbackAt = nextUpdatedMs > 0 && (previousUpdatedMs <= 0 || nextUpdatedMs > previousUpdatedMs) ? next.updatedAt : nowIso(); this.taskActivityIntervalService.resumeActiveIntervalsForMember( run.teamName, memberName, deriveTaskActivityResumeAt(previous, observedAt, resumeFallbackAt) ); } } /** * Update spawn status for a specific team member and emit a change event. */ private setMemberSpawnStatus( run: ProvisioningRun, memberName: string, status: MemberSpawnStatus, error?: string, livenessSource?: MemberSpawnLivenessSource, heartbeatAt?: string ): void { const prev = run.memberSpawnStatuses.get(memberName) ?? createInitialMemberSpawnStatusEntry(); if ( status === 'waiting' && !prev.hardFailure && (prev.bootstrapConfirmed || prev.runtimeAlive) ) { this.setMemberSpawnStatus( run, memberName, 'online', undefined, prev.livenessSource, prev.lastHeartbeatAt ); return; } const updatedAt = nowIso(); const next: MemberSpawnStatusEntry = { ...prev, status, updatedAt, }; if (status === 'spawning') { const pendingRestart = run.pendingMemberRestarts?.get(memberName); next.skippedForLaunch = false; next.skipReason = undefined; next.skippedAt = undefined; next.agentToolAccepted = false; next.runtimeAlive = false; next.bootstrapConfirmed = false; next.hardFailure = false; next.bootstrapStalled = undefined; next.error = undefined; next.hardFailureReason = undefined; next.livenessSource = undefined; next.livenessKind = undefined; next.runtimeDiagnostic = undefined; next.runtimeDiagnosticSeverity = undefined; next.livenessLastCheckedAt = undefined; next.firstSpawnAcceptedAt = pendingRestart?.requestedAt; next.lastHeartbeatAt = undefined; if (pendingRestart) { next.runtimeDiagnostic = 'Manual restart is already in progress; waiting for teammate bootstrap.'; next.runtimeDiagnosticSeverity = 'info'; } next.launchState = 'starting'; } else if (status === 'waiting') { next.skippedForLaunch = false; next.skipReason = undefined; next.skippedAt = undefined; next.agentToolAccepted = true; next.runtimeAlive = false; next.bootstrapConfirmed = false; next.hardFailure = false; next.bootstrapStalled = undefined; next.error = undefined; next.hardFailureReason = undefined; next.livenessSource = undefined; next.livenessKind = undefined; next.runtimeDiagnostic = undefined; next.runtimeDiagnosticSeverity = undefined; next.livenessLastCheckedAt = undefined; next.firstSpawnAcceptedAt = prev.firstSpawnAcceptedAt ?? updatedAt; next.lastHeartbeatAt = undefined; next.launchState = 'runtime_pending_bootstrap'; } else if (status === 'online') { next.skippedForLaunch = false; next.skipReason = undefined; next.skippedAt = undefined; next.agentToolAccepted = true; next.runtimeAlive = true; next.livenessSource = livenessSource; next.firstSpawnAcceptedAt = prev.firstSpawnAcceptedAt ?? updatedAt; if (livenessSource === 'heartbeat') { const incomingHeartbeatAt = heartbeatAt?.trim() || updatedAt; next.bootstrapConfirmed = true; next.lastHeartbeatAt = isMemberSpawnHeartbeatTimestampNewer( prev.lastHeartbeatAt, incomingHeartbeatAt ) ? incomingHeartbeatAt : prev.lastHeartbeatAt; } next.hardFailure = false; next.bootstrapStalled = undefined; next.error = undefined; next.hardFailureReason = undefined; next.launchState = deriveMemberLaunchState(next); } else if (status === 'error') { next.skippedForLaunch = false; next.skipReason = undefined; next.skippedAt = undefined; next.error = error; next.hardFailure = true; next.bootstrapStalled = undefined; next.hardFailureReason = error; next.launchState = 'failed_to_start'; } else if (status === 'skipped') { next.skippedForLaunch = true; next.skipReason = error?.trim() || prev.hardFailureReason || prev.error || 'Skipped for this launch'; next.skippedAt = updatedAt; next.agentToolAccepted = false; next.runtimeAlive = false; next.bootstrapConfirmed = false; next.hardFailure = false; next.bootstrapStalled = undefined; next.error = undefined; next.hardFailureReason = undefined; next.livenessSource = undefined; next.livenessKind = undefined; next.runtimeDiagnostic = undefined; next.runtimeDiagnosticSeverity = undefined; next.livenessLastCheckedAt = undefined; next.firstSpawnAcceptedAt = undefined; next.lastHeartbeatAt = undefined; next.launchState = 'skipped_for_launch'; } else if (status === 'offline') { Object.assign(next, createInitialMemberSpawnStatusEntry(), { updatedAt }); next.error = undefined; next.hardFailureReason = undefined; next.skippedForLaunch = false; next.skipReason = undefined; next.skippedAt = undefined; next.livenessSource = undefined; next.livenessKind = undefined; next.runtimeDiagnostic = undefined; next.runtimeDiagnosticSeverity = undefined; next.livenessLastCheckedAt = undefined; next.firstSpawnAcceptedAt = undefined; next.lastHeartbeatAt = undefined; } next.launchState = deriveMemberLaunchState(next); if ( prev.status === next.status && prev.launchState === next.launchState && prev.error === next.error && prev.hardFailureReason === next.hardFailureReason && (prev.skippedForLaunch === true) === (next.skippedForLaunch === true) && prev.skipReason === next.skipReason && prev.skippedAt === next.skippedAt && prev.livenessSource === next.livenessSource && prev.agentToolAccepted === next.agentToolAccepted && prev.runtimeAlive === next.runtimeAlive && prev.bootstrapConfirmed === next.bootstrapConfirmed && prev.hardFailure === next.hardFailure && prev.livenessKind === next.livenessKind && prev.runtimeDiagnostic === next.runtimeDiagnostic && prev.runtimeDiagnosticSeverity === next.runtimeDiagnosticSeverity && prev.bootstrapStalled === next.bootstrapStalled && prev.firstSpawnAcceptedAt === next.firstSpawnAcceptedAt && prev.lastHeartbeatAt === next.lastHeartbeatAt ) { return; } const runtimeTransitionAt = status === 'online' && livenessSource === 'heartbeat' ? (normalizeIsoTimestamp(heartbeatAt) ?? updatedAt) : updatedAt; this.syncMemberTaskActivityForRuntimeTransition( run, memberName, prev, next, runtimeTransitionAt ); run.memberSpawnStatuses.set(memberName, next); if ( (status === 'online' && (next.bootstrapConfirmed || livenessSource === 'process')) || status === 'offline' || status === 'error' || status === 'skipped' ) { run.pendingMemberRestarts?.delete(memberName); } this.syncMemberLaunchGraceCheck(run, memberName, next); const launchDiagnostics = boundLaunchDiagnostics(buildLaunchDiagnosticsFromRun(run)); if (launchDiagnostics) { run.progress = { ...run.progress, updatedAt: nowIso(), launchDiagnostics, }; run.onProgress(run.progress); } if (status === 'spawning') { this.appendMemberBootstrapDiagnostic(run, memberName, 'Agent tool invoked'); } else if (status === 'waiting') { this.appendMemberBootstrapDiagnostic( run, memberName, 'spawn accepted, waiting for teammate check-in' ); } else if (status === 'online' && livenessSource === 'heartbeat' && !prev.bootstrapConfirmed) { this.appendMemberBootstrapDiagnostic( run, memberName, 'bootstrap confirmed via first heartbeat' ); } else if (status === 'online' && livenessSource === 'process') { this.appendMemberBootstrapDiagnostic( run, memberName, 'runtime process is alive, teammate check-in not yet received' ); } else if (status === 'error') { this.appendMemberBootstrapDiagnostic( run, memberName, error?.trim().length ? error.trim() : 'bootstrap failed' ); } else if (status === 'skipped') { this.appendMemberBootstrapDiagnostic( run, memberName, error?.trim().length ? `skipped for this launch: ${error.trim()}` : 'skipped for this launch' ); } if (!this.isCurrentTrackedRun(run)) return; this.emitMemberSpawnChange(run, memberName); if (run.isLaunch) { void this.persistLaunchStateSnapshot(run, run.provisioningComplete ? 'finished' : 'active'); } } private confirmMemberSpawnStatusFromTranscript( run: ProvisioningRun, memberName: string, observedAt: string, source: 'transcript' | 'runtime-proof' = 'transcript' ): void { const prev = run.memberSpawnStatuses.get(memberName) ?? createInitialMemberSpawnStatusEntry(); const updatedAt = nowIso(); const next: MemberSpawnStatusEntry = { ...prev, status: 'online', updatedAt, agentToolAccepted: true, runtimeAlive: source === 'runtime-proof' ? true : prev.runtimeAlive, bootstrapConfirmed: true, hardFailure: false, bootstrapStalled: undefined, error: undefined, hardFailureReason: undefined, livenessSource: source === 'runtime-proof' ? (prev.livenessSource ?? 'process') : prev.livenessSource, firstSpawnAcceptedAt: prev.firstSpawnAcceptedAt ?? observedAt, lastHeartbeatAt: isMemberSpawnHeartbeatTimestampNewer(prev.lastHeartbeatAt, observedAt) ? observedAt : prev.lastHeartbeatAt, }; next.launchState = deriveMemberLaunchState(next); if ( prev.status === next.status && prev.launchState === next.launchState && prev.error === next.error && prev.hardFailureReason === next.hardFailureReason && prev.livenessSource === next.livenessSource && prev.agentToolAccepted === next.agentToolAccepted && prev.runtimeAlive === next.runtimeAlive && prev.bootstrapConfirmed === next.bootstrapConfirmed && prev.hardFailure === next.hardFailure && prev.bootstrapStalled === next.bootstrapStalled && prev.firstSpawnAcceptedAt === next.firstSpawnAcceptedAt && prev.lastHeartbeatAt === next.lastHeartbeatAt ) { return; } const runtimeTransitionAt = source === 'runtime-proof' ? observedAt : updatedAt; this.syncMemberTaskActivityForRuntimeTransition( run, memberName, prev, next, runtimeTransitionAt ); run.memberSpawnStatuses.set(memberName, next); run.pendingMemberRestarts?.delete(memberName); this.syncMemberLaunchGraceCheck(run, memberName, next); this.appendMemberBootstrapDiagnostic( run, memberName, source === 'runtime-proof' ? 'bootstrap confirmed via runtime proof' : 'bootstrap confirmed via transcript' ); if (!this.isCurrentTrackedRun(run)) return; this.emitMemberSpawnChange(run, memberName); if (run.isLaunch) { void this.persistLaunchStateSnapshot(run, run.provisioningComplete ? 'finished' : 'active'); } } /** * Get current member spawn statuses for a team. * Returns a map of memberName → MemberSpawnStatusEntry. */ async getMemberSpawnStatuses(teamName: string): Promise<{ statuses: Record; runId: string | null; teamLaunchState?: TeamLaunchAggregateState; launchPhase?: PersistedTeamLaunchPhase; expectedMembers?: string[]; updatedAt?: string; summary?: PersistedTeamLaunchSummary; source?: 'live' | 'persisted' | 'merged'; }> { const readPersistedStatuses = async (resolvedRunId: string | null) => { const generationAtStart = this.getMemberSpawnStatusesCacheGeneration(teamName); const cached = this.memberSpawnStatusesSnapshotCache.get(teamName); if ( cached && cached.expiresAtMs > Date.now() && cached.runId === resolvedRunId && cached.generation === generationAtStart ) { return this.cloneMemberSpawnStatusesSnapshot(cached.snapshot); } const repairSnapshot = await this.readTaskActivityRepairLaunchSnapshot(teamName); this.repairStaleTaskActivityIntervalsOnce(teamName, repairSnapshot); const { snapshot, statuses } = await this.reconcilePersistedLaunchState(teamName); const nextStatuses = await this.attachLiveRuntimeMetadataToStatuses(teamName, statuses, { openCodeSecondaryBootstrapPendingMembers: this.getOpenCodeSecondaryBootstrapPendingMemberNames(snapshot), }); const runtimeObservedAt = nowIso(); const aliveMemberNames = Object.entries(nextStatuses) .filter(([, entry]) => entry.runtimeAlive === true) .map(([memberName]) => memberName); if (aliveMemberNames.length > 0) { // Resume all alive members in a single locked task-file pass per cycle // instead of one synchronous lock + full task read per member. this.taskActivityIntervalService.resumeActiveIntervalsForMembers( teamName, aliveMemberNames, runtimeObservedAt ); } const expectedMembers = snapshot ? this.getPersistedLaunchMemberNames(snapshot) : undefined; const summary = expectedMembers ? summarizeMemberSpawnStatusRecord(expectedMembers, nextStatuses) : undefined; const persistedSnapshot = { statuses: nextStatuses, runId: resolvedRunId, teamLaunchState: summary ? deriveTeamLaunchAggregateState(summary) : snapshot?.teamLaunchState, launchPhase: snapshot?.launchPhase, expectedMembers, updatedAt: snapshot?.updatedAt, summary: summary ?? snapshot?.summary, source: 'persisted' as const, }; if ( this.getMemberSpawnStatusesCacheGeneration(teamName) === generationAtStart && this.getTrackedRunId(teamName) === resolvedRunId ) { this.memberSpawnStatusesSnapshotCache.set(teamName, { expiresAtMs: Date.now() + TeamProvisioningService.PERSISTED_MEMBER_SPAWN_STATUS_SNAPSHOT_CACHE_TTL_MS, generation: generationAtStart, runId: resolvedRunId, snapshot: this.cloneMemberSpawnStatusesSnapshot(persistedSnapshot), }); } return persistedSnapshot; }; const runId = this.getTrackedRunId(teamName); if (!runId) { return readPersistedStatuses(null); } const run = this.runs.get(runId); if (!run) { return readPersistedStatuses(runId); } if (!this.shouldCacheMemberSpawnStatusesSnapshot(run)) { return this.buildMemberSpawnStatusesSnapshotForRun(run); } const generationAtStart = this.getMemberSpawnStatusesCacheGeneration(teamName); const cached = this.memberSpawnStatusesSnapshotCache.get(teamName); if ( cached && cached.expiresAtMs > Date.now() && cached.runId === run.runId && cached.generation === generationAtStart ) { return this.cloneMemberSpawnStatusesSnapshot(cached.snapshot); } const existingRequest = this.memberSpawnStatusesInFlightByTeam.get(teamName); if ( existingRequest?.generationAtStart === generationAtStart && existingRequest.runIdAtStart === run.runId ) { const snapshot = await existingRequest.promise; if ( this.getMemberSpawnStatusesCacheGeneration(teamName) === generationAtStart && this.getTrackedRunId(teamName) === run.runId ) { return this.cloneMemberSpawnStatusesSnapshot(snapshot); } return this.getMemberSpawnStatuses(teamName); } const request = this.buildMemberSpawnStatusesSnapshotForRun(run, generationAtStart).finally( () => { if (this.memberSpawnStatusesInFlightByTeam.get(teamName)?.promise === request) { this.memberSpawnStatusesInFlightByTeam.delete(teamName); } } ); this.memberSpawnStatusesInFlightByTeam.set(teamName, { generationAtStart, runIdAtStart: run.runId, promise: request, }); const snapshot = await request; if ( this.getMemberSpawnStatusesCacheGeneration(teamName) === generationAtStart && this.getTrackedRunId(teamName) === run.runId ) { return this.cloneMemberSpawnStatusesSnapshot(snapshot); } return this.getMemberSpawnStatuses(teamName); } private shouldCacheMemberSpawnStatusesSnapshot(run: ProvisioningRun): boolean { return run.isLaunch === true && run.provisioningComplete !== true; } private async buildMemberSpawnStatusesSnapshotForRun( run: ProvisioningRun, generationAtStart?: number ): Promise { const teamName = run.teamName; await this.refreshMemberSpawnStatusesFromLeadInbox(run); await this.maybeAuditMemberSpawnStatuses(run); await this.persistLaunchStateSnapshot(run, run.provisioningComplete ? 'finished' : 'active'); const persisted = await this.launchStateStore.read(teamName); if (persisted) { this.syncRunMemberSpawnStatusesFromSnapshot(run, persisted); } const liveSnapshot = this.buildLiveLaunchSnapshotForRun(run, run.provisioningComplete ? 'finished' : 'active') ?? snapshotFromRuntimeMemberStatuses({ teamName: run.teamName, expectedMembers: run.expectedMembers, leadSessionId: run.detectedSessionId ?? undefined, launchPhase: run.provisioningComplete ? 'finished' : 'active', statuses: this.buildRuntimeSpawnStatusRecord(run), }); const rawSnapshot = liveSnapshot ?? persisted; const metaMembers = await this.membersMetaStore.getMembers(teamName).catch(() => []); const launchSnapshot = this.filterRemovedMembersFromLaunchSnapshot(rawSnapshot, metaMembers); const statuses = await this.attachLiveRuntimeMetadataToStatuses( teamName, snapshotToMemberSpawnStatuses(launchSnapshot), { openCodeSecondaryBootstrapPendingMembers: this.getOpenCodeSecondaryBootstrapPendingMemberNames(launchSnapshot), } ); const expectedMembers = this.getPersistedLaunchMemberNames(launchSnapshot); const summary = summarizeMemberSpawnStatusRecord(expectedMembers, statuses); const spawnSnapshot: MemberSpawnStatusesSnapshot = { statuses, runId: run.runId, teamLaunchState: deriveTeamLaunchAggregateState(summary), launchPhase: launchSnapshot.launchPhase, expectedMembers, updatedAt: launchSnapshot.updatedAt, summary, source: persisted ? 'merged' : 'live', }; if ( generationAtStart != null && this.shouldCacheMemberSpawnStatusesSnapshot(run) && this.getMemberSpawnStatusesCacheGeneration(teamName) === generationAtStart && this.getTrackedRunId(teamName) === run.runId ) { this.memberSpawnStatusesSnapshotCache.set(teamName, { expiresAtMs: Date.now() + TeamProvisioningService.MEMBER_SPAWN_STATUS_SNAPSHOT_CACHE_TTL_MS, generation: generationAtStart, runId: run.runId, snapshot: this.cloneMemberSpawnStatusesSnapshot(spawnSnapshot), }); } return spawnSnapshot; } async getTeamAgentRuntimeSnapshot(teamName: string): Promise { const runId = this.getTrackedRunId(teamName); const cached = this.agentRuntimeSnapshotCache.get(teamName); if (cached && cached.expiresAtMs > Date.now() && cached.snapshot.runId === runId) { return cached.snapshot; } const generationAtStart = this.getRuntimeSnapshotCacheGeneration(teamName); const existingRequest = this.agentRuntimeSnapshotInFlightByTeam.get(teamName); if ( existingRequest?.generationAtStart === generationAtStart && existingRequest.runIdAtStart === runId ) { return existingRequest.promise; } const request = this.buildTeamAgentRuntimeSnapshot(teamName, runId, generationAtStart).finally( () => { if (this.agentRuntimeSnapshotInFlightByTeam.get(teamName)?.promise === request) { this.agentRuntimeSnapshotInFlightByTeam.delete(teamName); } } ); this.agentRuntimeSnapshotInFlightByTeam.set(teamName, { generationAtStart, runIdAtStart: runId, promise: request, }); return request; } private async buildTeamAgentRuntimeSnapshot( teamName: string, runId: string | null, generationAtStart: number ): Promise { const updatedAt = nowIso(); const run = runId ? (this.runs.get(runId) ?? null) : null; const currentRuntimeAdapterRun = this.runtimeAdapterRunByTeam.get(teamName); const persistedTeamMeta = await this.teamMetaStore.getMeta(teamName).catch(() => null); let configuredMembers: TeamConfig['members'] = []; try { const config = await this.readConfigSnapshot(teamName); configuredMembers = config?.members ?? []; } catch { configuredMembers = []; } const metaMembers = await this.membersMetaStore.getMembers(teamName).catch(() => []); const launchSnapshot = choosePreferredLaunchSnapshot( await readBootstrapLaunchSnapshot(teamName), await this.launchStateStore.read(teamName) ); const spawnStatusSnapshot = await this.getMemberSpawnStatuses(teamName).catch(() => null); const liveRuntimeByMember = await this.getLiveTeamAgentRuntimeMetadata(teamName); const activeRuntimeRunId = run?.runId?.trim() || currentRuntimeAdapterRun?.runId?.trim() || runId?.trim() || ''; const spawnStatusRunId = spawnStatusSnapshot?.runId?.trim() ?? ''; const canUseLiveSpawnStatusRuntimeTruth = spawnStatusSnapshot?.source === 'live' && activeRuntimeRunId.length > 0 && spawnStatusRunId === activeRuntimeRunId; const runtimeRootOwnersByPid = new Map>(); const runtimeUsageRootPids = new Set(); const addRuntimeRootPid = ( pid: unknown, ownerKey: string, options: { sampleUsage?: boolean } = {} ): void => { if (typeof pid !== 'number' || !Number.isFinite(pid) || pid <= 0) { return; } const owners = runtimeRootOwnersByPid.get(pid) ?? new Set(); owners.add(ownerKey); runtimeRootOwnersByPid.set(pid, owners); if (options.sampleUsage !== false) { runtimeUsageRootPids.add(pid); } }; const canSampleRuntimeMetadataPid = ( metadata: LiveTeamAgentRuntimeMetadata, pid: unknown ): boolean => { if (typeof pid !== 'number' || !Number.isFinite(pid) || pid <= 0) { return false; } if (process.platform !== 'win32') { return true; } const paneId = metadata.tmuxPaneId?.trim() ?? ''; if (metadata.backendType === 'tmux' || (paneId && !paneId.startsWith('process:'))) { return false; } return ( metadata.pidSource !== 'tmux_child' && metadata.pidSource !== 'tmux_pane' && metadata.pidSource !== 'persisted_metadata' ); }; const leadPid = run?.child?.pid; addRuntimeRootPid(leadPid, '__lead__'); for (const [memberName, metadata] of liveRuntimeByMember.entries()) { const memberPids = [metadata.pid, metadata.metricsPid]; for (const memberPid of memberPids) { addRuntimeRootPid(memberPid, memberName, { sampleUsage: canSampleRuntimeMetadataPid(metadata, memberPid), }); } } let runtimeUsageTreesByRootPid = new Map(); let usageStatsByPid = new Map(); try { const runtimeProcessRows = runtimeRootOwnersByPid.size > 0 ? await this.readRuntimeProcessRowsForUsageSnapshot(teamName, { includeWindowsHostRows: process.platform === 'win32', }) : null; this.addRuntimeRootOwnersFromProcessRows( teamName, runtimeProcessRows, runtimeRootOwnersByPid ); runtimeUsageTreesByRootPid = this.buildRuntimeUsageProcessTrees( [...runtimeUsageRootPids], runtimeProcessRows, runtimeRootOwnersByPid ); const runtimeUsagePids = [ ...new Set([...runtimeUsageTreesByRootPid.values()].flatMap((tree) => tree.pids)), ]; usageStatsByPid = this.buildProcessUsageStatsFromRows(runtimeProcessRows, runtimeUsagePids); const pidsMissingUsageStats = runtimeUsagePids.filter((pid) => !usageStatsByPid.has(pid)); if ( pidsMissingUsageStats.length > 0 && this.shouldSampleMissingRuntimeUsageStatsWithPidusage() ) { const sampledUsageStats = await this.readProcessUsageStatsByPid(pidsMissingUsageStats); for (const [pid, stats] of sampledUsageStats) { usageStatsByPid.set(pid, stats); } } } catch (error) { logger.debug( `[${teamName}] Runtime telemetry sampling failed; continuing without resource metrics: ${ error instanceof Error ? error.message : String(error) }` ); } const persistedRuntimeMembers = this.readPersistedRuntimeMembers(teamName); const snapshotMembers: Record = {}; const activeResourceHistoryKeys = new Set(); const getPersistedRuntimeMember = ( memberName: string ): PersistedRuntimeMemberLike | undefined => { return persistedRuntimeMembers.find((member) => { const candidateName = typeof member.name === 'string' ? member.name.trim() : ''; return candidateName.length > 0 && matchesMemberNameOrBase(candidateName, memberName); }); }; const getLiveRuntimeMember = (memberName: string): LiveTeamAgentRuntimeMetadata | undefined => { let fallback: LiveTeamAgentRuntimeMetadata | undefined; for (const [candidateName, metadata] of liveRuntimeByMember.entries()) { if (candidateName === memberName) { return metadata; } if (matchesMemberNameOrBase(candidateName, memberName)) { fallback = metadata; } } return fallback; }; const getSpawnStatusMember = (memberName: string): MemberSpawnStatusEntry | undefined => { const statuses = spawnStatusSnapshot?.statuses; if (!statuses) { return undefined; } const direct = statuses[memberName]; if (direct) { return direct; } let fallback: MemberSpawnStatusEntry | undefined; for (const [candidateName, status] of Object.entries(statuses)) { if (matchesMemberNameOrBase(candidateName, memberName)) { fallback = status; } } return fallback; }; const activeRunMemberByName = new Map(); const runAllEffectiveMembers = run?.allEffectiveMembers ?? []; const activeRunMembers = runAllEffectiveMembers.length > 0 ? runAllEffectiveMembers : (run?.effectiveMembers ?? []); for (const member of activeRunMembers) { const memberName = typeof member?.name === 'string' ? member.name.trim() : ''; if (!memberName) continue; activeRunMemberByName.set(memberName, member); } const candidateMembers = new Map(); for (const member of configuredMembers) { const memberName = typeof member?.name === 'string' ? member.name.trim() : ''; if (!memberName || this.isMemberRemovedInMeta(metaMembers, memberName)) continue; candidateMembers.set(memberName, member); } for (const member of metaMembers) { const memberName = typeof member?.name === 'string' ? member.name.trim() : ''; if (!memberName || member.removedAt || candidateMembers.has(memberName)) continue; candidateMembers.set(memberName, member); } for (const memberName of launchSnapshot ? this.getPersistedLaunchMemberNames(launchSnapshot) : []) { if (candidateMembers.has(memberName) || this.isMemberRemovedInMeta(metaMembers, memberName)) { continue; } const launchMember = launchSnapshot?.members[memberName]; candidateMembers.set(memberName, { name: memberName, agentType: 'general-purpose', providerId: launchMember?.providerId, providerBackendId: launchMember?.providerBackendId, model: launchMember?.model, effort: launchMember?.effort, fastMode: launchMember?.selectedFastMode, }); } for (const member of activeRunMemberByName.values()) { const memberName = typeof member?.name === 'string' ? member.name.trim() : ''; if (!memberName || this.isMemberRemovedInMeta(metaMembers, memberName)) continue; candidateMembers.set(memberName, member); } for (const member of candidateMembers.values()) { const memberName = typeof member?.name === 'string' ? member.name.trim() : ''; if (!memberName) continue; const isLead = isLeadMember({ name: memberName, agentType: member.agentType }); if (isLead) { const pid = run?.child?.pid; const usageStats = pid ? this.buildRuntimeProcessLoadStatsSafely(teamName, memberName, { rootPid: pid, usageStatsByPid, processTree: runtimeUsageTreesByRootPid.get(pid), }) : undefined; const runtimeModel = run?.request.model?.trim() || (run?.spawnContext ? extractCliFlagValue(run.spawnContext.args.join(' '), '--model') : undefined) || member.model?.trim() || undefined; const resourceHistory = pid ? this.recordAgentRuntimeResourceSampleSafely({ teamName, memberName, timestamp: updatedAt, cpuPercent: usageStats?.cpuPercent, rssBytes: usageStats?.rssBytes, primaryCpuPercent: usageStats?.primaryCpuPercent, primaryRssBytes: usageStats?.primaryRssBytes, childCpuPercent: usageStats?.childCpuPercent, childRssBytes: usageStats?.childRssBytes, processCount: usageStats?.processCount, runtimeLoadScope: usageStats?.runtimeLoadScope, runtimeLoadTruncated: usageStats?.runtimeLoadTruncated, pidSource: 'lead_process', pid, activeKeys: activeResourceHistoryKeys, }) : undefined; snapshotMembers[memberName] = { memberName, alive: Boolean(pid && !run?.processKilled && !run?.cancelRequested), restartable: false, backendType: 'lead', ...(pid ? { pid } : {}), ...(runtimeModel ? { runtimeModel } : {}), ...(usageStats?.rssBytes != null ? { rssBytes: usageStats.rssBytes } : {}), ...(usageStats?.cpuPercent != null ? { cpuPercent: usageStats.cpuPercent } : {}), ...(usageStats?.primaryRssBytes != null ? { primaryRssBytes: usageStats.primaryRssBytes } : {}), ...(usageStats?.primaryCpuPercent != null ? { primaryCpuPercent: usageStats.primaryCpuPercent } : {}), ...(usageStats?.childRssBytes != null ? { childRssBytes: usageStats.childRssBytes } : {}), ...(usageStats?.childCpuPercent != null ? { childCpuPercent: usageStats.childCpuPercent } : {}), ...(usageStats?.processCount != null ? { processCount: usageStats.processCount } : {}), ...(usageStats?.runtimeLoadScope ? { runtimeLoadScope: usageStats.runtimeLoadScope } : {}), ...(usageStats?.runtimeLoadTruncated ? { runtimeLoadTruncated: true } : {}), ...(pid ? { pidSource: 'lead_process' as const } : {}), ...(resourceHistory && resourceHistory.length > 0 ? { resourceHistory } : {}), updatedAt, }; continue; } const persistedRuntimeMember = getPersistedRuntimeMember(memberName); const liveRuntimeMember = getLiveRuntimeMember(memberName); const spawnStatusMember = getSpawnStatusMember(memberName); const launchMember = launchSnapshot?.members[memberName]; const activeRunMember = activeRunMemberByName.get(memberName); const activeRunModel = activeRunMember?.model?.trim(); const activeRunProviderId = normalizeOptionalTeamProviderId(activeRunMember?.providerId) ?? inferTeamProviderIdFromModel(activeRunModel); const liveRuntimeModel = liveRuntimeMember?.model?.trim(); const liveRuntimeModelProviderId = inferTeamProviderIdFromModel(liveRuntimeModel); const explicitLiveRuntimeProviderId = normalizeOptionalTeamProviderId( liveRuntimeMember?.providerId ); const liveRuntimeProviderConflictsWithActive = activeRunProviderId != null && ((explicitLiveRuntimeProviderId != null && explicitLiveRuntimeProviderId !== activeRunProviderId) || (liveRuntimeModelProviderId != null && liveRuntimeModelProviderId !== activeRunProviderId)); const canUseLiveRuntimeModel = !!liveRuntimeModel && !liveRuntimeProviderConflictsWithActive; const backendType = liveRuntimeMember?.backendType ?? normalizeTeamAgentRuntimeBackendType(persistedRuntimeMember?.backendType, false); const runtimeModel = (canUseLiveRuntimeModel ? liveRuntimeModel : undefined) ?? activeRunModel ?? launchMember?.model?.trim() ?? member.model?.trim() ?? undefined; const memberProviderId = activeRunProviderId ?? normalizeOptionalTeamProviderId(launchMember?.providerId) ?? normalizeOptionalTeamProviderId(member.providerId) ?? inferTeamProviderIdFromModel(runtimeModel) ?? inferTeamProviderIdFromModel(launchMember?.model) ?? inferTeamProviderIdFromModel(member.model); const memberProviderBackendId = migrateProviderBackendId( memberProviderId, activeRunMember?.providerBackendId ?? launchMember?.providerBackendId ?? member.providerBackendId ); const isOpenCodeMember = memberProviderId === 'opencode'; const configuredCwd = typeof activeRunMember?.cwd === 'string' ? activeRunMember.cwd.trim() : typeof member.cwd === 'string' ? member.cwd.trim() : ''; const runtimeCwd = liveRuntimeMember?.cwd ?? (configuredCwd || (isOpenCodeMember ? currentRuntimeAdapterRun?.cwd : undefined)); const metricsPid = liveRuntimeMember?.metricsPid; const isSharedOpenCodeHost = isOpenCodeMember && typeof metricsPid === 'number' && metricsPid > 0 && liveRuntimeMember?.pidSource !== 'agent_process_table'; const rssPid = isSharedOpenCodeHost ? metricsPid : (liveRuntimeMember?.pid ?? metricsPid); const displayPid = isSharedOpenCodeHost ? rssPid : liveRuntimeMember?.pid; const restartable = isOpenCodeMember ? !isSharedOpenCodeHost && Boolean(liveRuntimeMember?.pid) : isSharedOpenCodeHost ? false : backendType !== 'in-process'; const historicalBootstrapConfirmed = launchMember?.bootstrapConfirmed === true || launchMember?.launchState === 'confirmed_alive' || spawnStatusMember?.bootstrapConfirmed === true || spawnStatusMember?.launchState === 'confirmed_alive'; const spawnStatusConfirmsBootstrap = spawnStatusMember?.bootstrapConfirmed === true || spawnStatusMember?.launchState === 'confirmed_alive'; const hasOpenCodeRuntimeHandle = isOpenCodeMember && (typeof liveRuntimeMember?.pid === 'number' || typeof liveRuntimeMember?.metricsPid === 'number' || typeof liveRuntimeMember?.runtimeSessionId === 'string'); const confirmedOpenCodeRuntimeAlive = isOpenCodeMember && canUseLiveSpawnStatusRuntimeTruth && historicalBootstrapConfirmed && hasOpenCodeRuntimeHandle && spawnStatusMember?.hardFailure !== true && spawnStatusMember?.launchState !== 'failed_to_start' && spawnStatusMember?.launchState !== 'runtime_pending_permission'; const confirmedSpawnRuntimeFallback = !isOpenCodeMember && spawnStatusConfirmsBootstrap && spawnStatusMember?.hardFailure !== true && spawnStatusMember?.launchState !== 'failed_to_start' && !isStrongRuntimeEvidence(liveRuntimeMember); const confirmedSpawnRuntimeDiagnostic = spawnStatusMember?.runtimeDiagnostic ?? liveRuntimeMember?.runtimeDiagnostic; const shouldKeepConfirmedSpawnRuntimeDiagnostic = !!confirmedSpawnRuntimeDiagnostic && !shouldClearRuntimeDiagnosticAfterBootstrapConfirmation(confirmedSpawnRuntimeDiagnostic); const effectiveAlive = liveRuntimeMember?.alive === true || confirmedOpenCodeRuntimeAlive || confirmedSpawnRuntimeFallback; const effectiveLivenessKind = confirmedOpenCodeRuntimeAlive && liveRuntimeMember?.livenessKind === 'runtime_process_candidate' ? 'confirmed_bootstrap' : confirmedSpawnRuntimeFallback ? 'confirmed_bootstrap' : liveRuntimeMember?.livenessKind; const effectivePidSource = confirmedSpawnRuntimeFallback && (liveRuntimeMember?.pidSource === 'persisted_metadata' || liveRuntimeMember?.pidSource == null) ? 'runtime_bootstrap' : liveRuntimeMember?.pidSource; const effectiveRuntimeDiagnostic = confirmedOpenCodeRuntimeAlive && liveRuntimeMember?.livenessKind === 'runtime_process_candidate' ? 'OpenCode bootstrap confirmed; runtime host/session evidence present.' : confirmedSpawnRuntimeFallback ? shouldKeepConfirmedSpawnRuntimeDiagnostic ? confirmedSpawnRuntimeDiagnostic : 'bootstrap confirmed' : liveRuntimeMember?.runtimeDiagnostic; const effectiveRuntimeDiagnosticSeverity = confirmedOpenCodeRuntimeAlive && liveRuntimeMember?.livenessKind === 'runtime_process_candidate' ? 'info' : confirmedSpawnRuntimeFallback ? shouldKeepConfirmedSpawnRuntimeDiagnostic ? (spawnStatusMember?.runtimeDiagnosticSeverity ?? liveRuntimeMember?.runtimeDiagnosticSeverity ?? 'info') : 'info' : liveRuntimeMember?.runtimeDiagnosticSeverity; if ( rssPid && !usageStatsByPid.has(rssPid) && isSharedOpenCodeHost && typeof rssPid === 'number' && rssPid > 0 && this.isRuntimePidusageTelemetryEnabled() ) { try { const refreshedUsageStats = ( await this.readProcessUsageStatsByPid([rssPid], { ignoreCachedMisses: true }) ).get(rssPid); if (refreshedUsageStats) { usageStatsByPid.set(rssPid, refreshedUsageStats); } } catch (error) { logger.debug( `[${teamName}] Shared OpenCode host runtime usage refresh failed for pid ${rssPid}; continuing without refreshed metrics: ${ error instanceof Error ? error.message : String(error) }` ); // Shared OpenCode host can exit between discovery and the targeted RSS refresh. } } const usageStats = rssPid ? this.buildRuntimeProcessLoadStatsSafely(teamName, memberName, { rootPid: rssPid, usageStatsByPid, processTree: runtimeUsageTreesByRootPid.get(rssPid), scope: isSharedOpenCodeHost ? 'shared-host' : undefined, }) : undefined; const resourceHistory = rssPid ? this.recordAgentRuntimeResourceSampleSafely({ teamName, memberName, timestamp: updatedAt, cpuPercent: usageStats?.cpuPercent, rssBytes: usageStats?.rssBytes, primaryCpuPercent: usageStats?.primaryCpuPercent, primaryRssBytes: usageStats?.primaryRssBytes, childCpuPercent: usageStats?.childCpuPercent, childRssBytes: usageStats?.childRssBytes, processCount: usageStats?.processCount, runtimeLoadScope: usageStats?.runtimeLoadScope, runtimeLoadTruncated: usageStats?.runtimeLoadTruncated, pidSource: liveRuntimeMember?.pidSource, pid: rssPid, runtimePid: liveRuntimeMember?.metricsPid, activeKeys: activeResourceHistoryKeys, }) : undefined; snapshotMembers[memberName] = { memberName, alive: effectiveAlive, restartable, ...(backendType ? { backendType } : {}), ...(memberProviderId ? { providerId: memberProviderId } : {}), ...(memberProviderBackendId ? { providerBackendId: memberProviderBackendId } : {}), ...(launchMember?.laneId ? { laneId: launchMember.laneId } : {}), ...(launchMember?.laneKind ? { laneKind: launchMember.laneKind } : {}), ...(displayPid ? { pid: displayPid } : {}), ...(runtimeModel ? { runtimeModel } : {}), ...(runtimeCwd ? { cwd: runtimeCwd } : {}), ...(usageStats?.rssBytes != null ? { rssBytes: usageStats.rssBytes } : {}), ...(usageStats?.cpuPercent != null ? { cpuPercent: usageStats.cpuPercent } : {}), ...(usageStats?.primaryRssBytes != null ? { primaryRssBytes: usageStats.primaryRssBytes } : {}), ...(usageStats?.primaryCpuPercent != null ? { primaryCpuPercent: usageStats.primaryCpuPercent } : {}), ...(usageStats?.childRssBytes != null ? { childRssBytes: usageStats.childRssBytes } : {}), ...(usageStats?.childCpuPercent != null ? { childCpuPercent: usageStats.childCpuPercent } : {}), ...(usageStats?.processCount != null ? { processCount: usageStats.processCount } : {}), ...(usageStats?.runtimeLoadScope ? { runtimeLoadScope: usageStats.runtimeLoadScope } : {}), ...(usageStats?.runtimeLoadTruncated ? { runtimeLoadTruncated: true } : {}), ...(resourceHistory && resourceHistory.length > 0 ? { resourceHistory } : {}), ...(effectiveLivenessKind ? { livenessKind: effectiveLivenessKind } : {}), ...(effectivePidSource ? { pidSource: effectivePidSource } : {}), ...(liveRuntimeMember?.processCommand ? { processCommand: liveRuntimeMember.processCommand } : {}), ...(liveRuntimeMember?.tmuxPaneId ? { paneId: liveRuntimeMember.tmuxPaneId } : {}), ...(liveRuntimeMember?.panePid ? { panePid: liveRuntimeMember.panePid } : {}), ...(liveRuntimeMember?.paneCurrentCommand ? { paneCurrentCommand: liveRuntimeMember.paneCurrentCommand } : {}), ...(liveRuntimeMember?.metricsPid ? { runtimePid: liveRuntimeMember.metricsPid } : {}), ...(liveRuntimeMember?.runtimeSessionId ? { runtimeSessionId: liveRuntimeMember.runtimeSessionId } : {}), ...(liveRuntimeMember?.runtimeLastSeenAt ? { runtimeLastSeenAt: liveRuntimeMember.runtimeLastSeenAt } : {}), ...(historicalBootstrapConfirmed ? { historicalBootstrapConfirmed: true } : {}), ...(effectiveRuntimeDiagnostic ? { runtimeDiagnostic: effectiveRuntimeDiagnostic } : {}), ...(effectiveRuntimeDiagnosticSeverity ? { runtimeDiagnosticSeverity: effectiveRuntimeDiagnosticSeverity } : {}), ...(liveRuntimeMember?.diagnostics ? { diagnostics: liveRuntimeMember.diagnostics } : {}), updatedAt, }; } try { this.pruneAgentRuntimeResourceHistory(teamName, activeResourceHistoryKeys); } catch (error) { logger.debug( `[${teamName}] Failed to prune runtime telemetry history; continuing with snapshot: ${ error instanceof Error ? error.message : String(error) }` ); } const persistedLaunchIdentity = persistedTeamMeta?.launchIdentity; const snapshotProviderId = run?.request.providerId ?? persistedLaunchIdentity?.providerId ?? persistedTeamMeta?.providerId; const snapshotProviderBackendId = run ? run.request.providerBackendId : persistedLaunchIdentity ? (persistedLaunchIdentity.providerBackendId ?? persistedTeamMeta?.providerBackendId) : persistedTeamMeta?.providerBackendId; const snapshot: TeamAgentRuntimeSnapshot = { teamName, updatedAt, runId: run?.runId ?? runId, providerBackendId: migrateProviderBackendId(snapshotProviderId, snapshotProviderBackendId), fastMode: run?.request.fastMode ?? persistedLaunchIdentity?.selectedFastMode ?? persistedTeamMeta?.fastMode, members: snapshotMembers, }; if ( this.getRuntimeSnapshotCacheGeneration(teamName) === generationAtStart && this.getTrackedRunId(teamName) === runId ) { this.agentRuntimeSnapshotCache.set(teamName, { expiresAtMs: Date.now() + this.getAgentRuntimeSnapshotCacheTtlMs(teamName, runId), snapshot, }); } return snapshot; } private getDirectTmuxRestartPaneId( persistedRuntimeMembers: readonly PersistedRuntimeMemberLike[], memberName: string ): string | null { for (const persistedRuntimeMember of persistedRuntimeMembers) { const backendType = persistedRuntimeMember.backendType?.trim().toLowerCase(); const paneId = typeof persistedRuntimeMember.tmuxPaneId === 'string' ? persistedRuntimeMember.tmuxPaneId.trim() : ''; const runtimeMemberName = typeof persistedRuntimeMember.name === 'string' ? persistedRuntimeMember.name : ''; if ( backendType === 'tmux' && paneId && matchesMemberNameOrBase(runtimeMemberName, memberName) ) { return paneId; } } return null; } private resolveDirectRestartRuntimeCwd(params: { configuredMember: NonNullable< ReturnType >; persistedRuntimeMembers: readonly PersistedRuntimeMemberLike[]; config: TeamConfig; run: ProvisioningRun; }): string { const configuredCwd = params.configuredMember.cwd?.trim(); if (configuredCwd) { return path.resolve(configuredCwd); } for (const runtimeMember of params.persistedRuntimeMembers) { const cwd = typeof runtimeMember.cwd === 'string' ? runtimeMember.cwd.trim() : ''; if (cwd) { return path.resolve(cwd); } } const projectPath = params.config.projectPath?.trim(); if (projectPath) { return path.resolve(projectPath); } const runCwd = this.getRunTrackedCwd(params.run); if (runCwd) { return path.resolve(runCwd); } throw new Error('Cannot restart teammate because its runtime cwd is unavailable'); } private async updateDirectTmuxRestartMemberConfig(input: { teamName: string; memberName: string; member: TeamCreateRequest['members'][number] & { agentType?: string }; agentId: string; color: string; prompt: string; paneId: string; cwd: string; providerId: TeamProviderId; joinedAt: number; bootstrapExpectedAfter: string; backendType?: 'tmux' | 'process'; runtimePid?: number; bootstrapRuntimeEventsPath?: string; bootstrapProofToken?: string; bootstrapRunId?: string; bootstrapContextHash?: string; bootstrapBriefingHash?: string; }): Promise { const configPath = path.join(getTeamsBasePath(), input.teamName, 'config.json'); const raw = await tryReadRegularFileUtf8(configPath, { timeoutMs: TEAM_JSON_READ_TIMEOUT_MS, maxBytes: TEAM_CONFIG_MAX_BYTES, }); if (!raw) { throw new Error(`Team "${input.teamName}" configuration is no longer available`); } const parsed = JSON.parse(raw) as TeamConfig & { members?: Record[] }; const members = Array.isArray(parsed.members) ? parsed.members : []; const existingIndex = members.findIndex((member) => { const candidateName = typeof member?.name === 'string' ? member.name.trim() : ''; return ( candidateName.length > 0 && matchesExactTeamMemberName(candidateName, input.memberName) ); }); const existing: Record = existingIndex >= 0 ? (members[existingIndex] ?? {}) : {}; const nextMember = { ...existing, agentId: input.agentId, name: input.member.name, ...(input.member.role ? { role: input.member.role } : {}), ...(input.member.workflow ? { workflow: input.member.workflow } : {}), ...(input.member.agentType ? { agentType: input.member.agentType } : {}), provider: input.providerId, providerId: input.providerId, ...(input.member.model ? { model: input.member.model } : {}), ...(input.member.effort ? { effort: input.member.effort } : {}), prompt: input.prompt, color: input.color, joinedAt: input.joinedAt, bootstrapExpectedAfter: input.bootstrapExpectedAfter, ...(input.bootstrapProofToken ? { bootstrapProofToken: input.bootstrapProofToken } : {}), ...(input.bootstrapRunId ? { bootstrapRunId: input.bootstrapRunId } : {}), ...(input.bootstrapRuntimeEventsPath ? { bootstrapRuntimeEventsPath: input.bootstrapRuntimeEventsPath } : {}), ...(input.bootstrapContextHash ? { bootstrapProofMode: 'native_app_managed_context', bootstrapContextHash: input.bootstrapContextHash, } : {}), ...(input.bootstrapBriefingHash ? { bootstrapBriefingHash: input.bootstrapBriefingHash } : {}), tmuxPaneId: input.paneId, ...(typeof input.runtimePid === 'number' ? { runtimePid: input.runtimePid } : {}), cwd: input.cwd, subscriptions: Array.isArray(existing.subscriptions) ? existing.subscriptions : [], backendType: input.backendType ?? 'tmux', }; if (existingIndex >= 0) { members[existingIndex] = nextMember; } else { members.push(nextMember); } parsed.members = members; await atomicWriteAsync(configPath, `${JSON.stringify(parsed, null, 2)}\n`); TeamConfigReader.invalidateTeam(input.teamName); } private enqueueDirectRestartPrompt(input: { teamName: string; memberName: string; leadName: string; leadSessionId: string | null; prompt: string; operation?: DirectProcessMemberLaunchReason; }): void { const timestamp = nowIso(); const operation = input.operation ?? 'manual_restart'; const isRestart = operation === 'manual_restart'; this.persistInboxMessage(input.teamName, input.memberName, { from: input.leadName, to: input.memberName, text: input.prompt, timestamp, read: false, source: 'system_notification', leadSessionId: input.leadSessionId ?? undefined, messageId: `direct-${operation}-${input.memberName}-${randomUUID()}`, summary: isRestart ? `Restart bootstrap instructions for ${input.memberName}` : `Bootstrap instructions for ${input.memberName}`, }); } private persistOpenCodeMemberRestartSystemMessage(input: { teamName: string; leadName: string; leadSessionId: string | null; displayName: string; member: TeamCreateRequest['members'][number]; reason: 'manual_restart' | 'member_updated'; }): void { const timestamp = nowIso(); const prompt = buildMemberSpawnPrompt( input.member, input.displayName, input.teamName, input.leadName, { restart: true } ); const reasonSummary = input.reason === 'member_updated' ? 'after member settings update' : 'by user request'; this.persistSentMessage(input.teamName, { from: input.leadName, to: input.member.name, text: prompt, timestamp, read: true, source: 'system_notification', leadSessionId: input.leadSessionId ?? undefined, messageId: `member-restart:${input.teamName}:${input.member.name}:${randomUUID()}`, summary: `Restarting ${input.member.name} ${reasonSummary}`, }); } private async launchDirectTmuxMemberRestart(input: { run: ProvisioningRun; teamName: string; displayName: string; leadName: string; memberName: string; config: TeamConfig; configuredMember: NonNullable< ReturnType >; persistedRuntimeMembers: readonly PersistedRuntimeMemberLike[]; paneId: string; }): Promise { const paneInfo = (await listTmuxPaneRuntimeInfoForCurrentPlatform([input.paneId])).get( input.paneId ); if (!paneInfo) { throw new Error( `Cannot restart teammate "${input.memberName}" because tmux pane ${input.paneId} is not available` ); } if (!isInteractiveShellCommand(paneInfo.currentCommand)) { throw new Error( `Cannot restart teammate "${input.memberName}" because tmux pane ${input.paneId} is busy (${paneInfo.currentCommand ?? 'unknown command'})` ); } const claudePath = await ClaudeBinaryResolver.resolve(); if (!claudePath) { throw buildMissingCliError(); } const cwd = this.resolveDirectRestartRuntimeCwd({ configuredMember: input.configuredMember, persistedRuntimeMembers: input.persistedRuntimeMembers, config: input.config, run: input.run, }); await ensureCwdExists(cwd); const operation: DirectProcessMemberLaunchReason = 'manual_restart'; const preliminaryMemberSpec = this.buildPrimaryOwnedMemberSpecForRuntime({ configuredMember: input.configuredMember, run: input.run, }); const providerId = resolveTeamProviderId(preliminaryMemberSpec.providerId); const providerBackendId = migrateProviderBackendId( providerId, preliminaryMemberSpec.providerBackendId ); const provisioningEnv = await this.buildProvisioningEnv(providerId, providerBackendId, { teamRuntimeAuth: { teamName: input.teamName, authMaterialId: `${input.run.runId}-direct-${input.configuredMember.name}-${randomUUID()}`, allowAnthropicApiKeyHelper: true, }, }); if (provisioningEnv.warning) { throw new Error(provisioningEnv.warning); } const [materializedMemberSpec] = await this.materializeEffectiveTeamMemberSpecs({ claudePath, cwd, members: [preliminaryMemberSpec], defaults: { providerId: resolveTeamProviderId(input.run.request.providerId), model: input.run.request.model, effort: input.run.request.effort, }, primaryProviderId: providerId, primaryEnv: provisioningEnv, teamRuntimeAuth: { teamName: input.teamName, authMaterialId: `${input.run.runId}-direct-${operation}-${input.configuredMember.name}-defaults-${randomUUID()}`, allowAnthropicApiKeyHelper: true, }, }); const memberSpec = materializedMemberSpec ?? preliminaryMemberSpec; const launchIdentity = await this.resolveDirectMemberLaunchIdentity({ claudePath, cwd, providerId, ...(providerBackendId ? { providerBackendId } : {}), provisioningEnv, memberSpec, run: input.run, }); const memberMcpPolicy = normalizeTeamMemberMcpPolicy(memberSpec.mcpPolicy); const mcpConfigPath = await this.mcpConfigBuilder.writeConfigFile(cwd, { mcpPolicy: memberMcpPolicy, controlApiBaseUrl: provisioningEnv.env.CLAUDE_TEAM_CONTROL_URL, }); const memberMcpConfigPaths = input.run.memberMcpConfigPaths ?? []; input.run.memberMcpConfigPaths = memberMcpConfigPaths; memberMcpConfigPaths.push(mcpConfigPath); const memberMcpSettingSources = buildTeamMemberMcpSettingSources(memberMcpPolicy); const strictMemberMcpConfig = requiresStrictTeamMemberMcpConfig(memberMcpPolicy); const agentId = `${input.configuredMember.name}@${input.teamName}`; const color = input.config.members ?.find((member) => matchesExactTeamMemberName(member.name, input.memberName)) ?.color?.trim() || getMemberColorByName(input.configuredMember.name); const parentSessionId = input.run.detectedSessionId?.trim() || input.config.leadSessionId?.trim() || input.run.runId; const prompt = buildMemberSpawnPrompt( memberSpec, input.displayName, input.teamName, input.leadName, { restart: true } ); const bootstrapExpectedAfter = nowIso(); const runtimeArgsPlan = await this.buildTeamRuntimeLaunchArgsPlan({ teamName: input.teamName, providerId, launchIdentity, envResolution: provisioningEnv, extraArgs: [], includeAnthropicHelper: providerId === 'anthropic', contextLabel: `Direct teammate restart (${input.configuredMember.name})`, }); applyAppManagedRuntimeSettingsPathEnv( provisioningEnv.env, runtimeArgsPlan.appManagedSettingsPath ); const runtimeArgs = mergeJsonSettingsArgs([ '--agent-id', agentId, '--agent-name', input.configuredMember.name, '--team-name', input.teamName, '--agent-color', color, '--parent-session-id', parentSessionId, ...(input.configuredMember.agentType ? ['--agent-type', input.configuredMember.agentType] : []), '--setting-sources', memberMcpSettingSources, '--mcp-config', mcpConfigPath, ...(strictMemberMcpConfig ? ['--strict-mcp-config'] : []), '--disallowedTools', APP_TEAM_RUNTIME_DISALLOWED_TOOLS, ...(input.run.request.skipPermissions !== false ? ['--dangerously-skip-permissions', '--permission-mode', 'bypassPermissions'] : ['--permission-prompt-tool', 'stdio', '--permission-mode', 'default']), ...(memberSpec.model ? ['--model', memberSpec.model] : []), ...(memberSpec.effort ? ['--effort', memberSpec.effort] : []), ...runtimeArgsPlan.fastModeArgs, ...runtimeArgsPlan.runtimeTurnSettledHookArgs, ...runtimeArgsPlan.providerArgs, ...runtimeArgsPlan.settingsArgs, ]); const command = buildDirectTmuxRestartCommand({ cwd, env: provisioningEnv.env, providerId, binaryPath: claudePath, args: runtimeArgs, }); await this.updateDirectTmuxRestartMemberConfig({ teamName: input.teamName, memberName: input.memberName, member: memberSpec, agentId, color, prompt, paneId: input.paneId, cwd, providerId, joinedAt: Date.now(), bootstrapExpectedAfter, }); this.enqueueDirectRestartPrompt({ teamName: input.teamName, memberName: input.configuredMember.name, leadName: input.leadName, leadSessionId: parentSessionId, prompt, operation, }); await sendKeysToTmuxPaneForCurrentPlatform(input.paneId, command); this.appendMemberBootstrapDiagnostic( input.run, input.memberName, `restart command delivered to tmux pane ${input.paneId}` ); this.setMemberSpawnStatus(input.run, input.memberName, 'waiting'); } private async launchDirectProcessMemberRestart(input: { run: ProvisioningRun; teamName: string; displayName: string; leadName: string; memberName: string; config: TeamConfig; configuredMember: NonNullable< ReturnType >; persistedRuntimeMembers: readonly PersistedRuntimeMemberLike[]; operation?: DirectProcessMemberLaunchReason; }): Promise { const operation = input.operation ?? 'manual_restart'; const claudePath = input.run.spawnContext?.claudePath ?? (await ClaudeBinaryResolver.resolve()); if (!claudePath) { throw buildMissingCliError(); } const cwd = this.resolveDirectRestartRuntimeCwd({ configuredMember: input.configuredMember, persistedRuntimeMembers: input.persistedRuntimeMembers, config: input.config, run: input.run, }); await ensureCwdExists(cwd); const preliminaryMemberSpec = this.buildPrimaryOwnedMemberSpecForRuntime({ configuredMember: input.configuredMember, run: input.run, }); const providerId = resolveTeamProviderId(preliminaryMemberSpec.providerId); const providerBackendId = migrateProviderBackendId( providerId, preliminaryMemberSpec.providerBackendId ); const provisioningEnv = await this.buildProvisioningEnv(providerId, providerBackendId, { teamRuntimeAuth: { teamName: input.teamName, authMaterialId: `${input.run.runId}-process-${operation}-${input.configuredMember.name}-${randomUUID()}`, allowAnthropicApiKeyHelper: true, }, }); if (provisioningEnv.warning) { throw new Error(provisioningEnv.warning); } const [materializedMemberSpec] = await this.materializeEffectiveTeamMemberSpecs({ claudePath, cwd, members: [preliminaryMemberSpec], defaults: { providerId: resolveTeamProviderId(input.run.request.providerId), model: input.run.request.model, effort: input.run.request.effort, }, primaryProviderId: providerId, primaryEnv: provisioningEnv, teamRuntimeAuth: { teamName: input.teamName, authMaterialId: `${input.run.runId}-process-${operation}-${input.configuredMember.name}-defaults-${randomUUID()}`, allowAnthropicApiKeyHelper: true, }, }); const memberSpec = materializedMemberSpec ?? preliminaryMemberSpec; const launchIdentity = await this.resolveDirectMemberLaunchIdentity({ claudePath, cwd, providerId, ...(providerBackendId ? { providerBackendId } : {}), provisioningEnv, memberSpec, run: input.run, }); const memberMcpPolicy = normalizeTeamMemberMcpPolicy(memberSpec.mcpPolicy); const mcpConfigPath = await this.mcpConfigBuilder.writeConfigFile(cwd, { mcpPolicy: memberMcpPolicy, controlApiBaseUrl: provisioningEnv.env.CLAUDE_TEAM_CONTROL_URL, }); const memberMcpConfigPaths = input.run.memberMcpConfigPaths ?? []; input.run.memberMcpConfigPaths = memberMcpConfigPaths; memberMcpConfigPaths.push(mcpConfigPath); const memberMcpSettingSources = buildTeamMemberMcpSettingSources(memberMcpPolicy); const strictMemberMcpConfig = requiresStrictTeamMemberMcpConfig(memberMcpPolicy); const agentId = `${input.configuredMember.name}@${input.teamName}`; const color = input.config.members ?.find((member) => matchesExactTeamMemberName(member.name, input.memberName)) ?.color?.trim() || getMemberColorByName(input.configuredMember.name); const parentSessionId = input.run.detectedSessionId?.trim() || input.config.leadSessionId?.trim() || input.run.runId; const prompt = buildMemberSpawnPrompt( memberSpec, input.displayName, input.teamName, input.leadName, operation === 'manual_restart' ? { restart: true } : undefined ); const bootstrapExpectedAfter = nowIso(); const bootstrapProofToken = randomUUID(); const runtimePaths = this.getDirectProcessRestartRuntimePaths( input.teamName, input.configuredMember.name ); await fs.promises.mkdir(runtimePaths.dir, { recursive: true }); await fs.promises.writeFile(runtimePaths.eventsPath, '', { encoding: 'utf8', mode: 0o600 }); const nativeBootstrapSpec = ( await buildNativeAppManagedBootstrapSpecs({ teamName: input.teamName, cwd, members: [memberSpec], }) ).get(input.configuredMember.name) ?? null; const nativeBootstrapEnv = await this.materializeDirectProcessNativeBootstrapContext({ teamName: input.teamName, memberName: input.configuredMember.name, agentId, providerId, runId: input.run.runId, bootstrapProofToken, spec: nativeBootstrapSpec, }); const runtimeArgsPlan = await this.buildTeamRuntimeLaunchArgsPlan({ teamName: input.teamName, providerId, launchIdentity, envResolution: provisioningEnv, extraArgs: [], includeAnthropicHelper: providerId === 'anthropic', contextLabel: `Direct process teammate ${operation} (${input.configuredMember.name})`, }); applyAppManagedRuntimeSettingsPathEnv( provisioningEnv.env, runtimeArgsPlan.appManagedSettingsPath ); const runtimeArgs = mergeJsonSettingsArgs([ '--teammate-runtime', 'headless', '--agent-id', agentId, '--agent-name', input.configuredMember.name, '--team-name', input.teamName, '--agent-color', color, '--parent-session-id', parentSessionId, ...(input.configuredMember.agentType ? ['--agent-type', input.configuredMember.agentType] : []), '--setting-sources', memberMcpSettingSources, '--mcp-config', mcpConfigPath, ...(strictMemberMcpConfig ? ['--strict-mcp-config'] : []), '--disallowedTools', APP_TEAM_RUNTIME_DISALLOWED_TOOLS, ...(input.run.request.skipPermissions !== false ? ['--dangerously-skip-permissions', '--permission-mode', 'bypassPermissions'] : ['--permission-prompt-tool', 'stdio', '--permission-mode', 'default']), ...(memberSpec.model ? ['--model', memberSpec.model] : []), ...(memberSpec.effort ? ['--effort', memberSpec.effort] : []), ...runtimeArgsPlan.fastModeArgs, ...runtimeArgsPlan.runtimeTurnSettledHookArgs, ...runtimeArgsPlan.providerArgs, ...runtimeArgsPlan.settingsArgs, ]); const stdoutLog = fs.createWriteStream(runtimePaths.stdoutPath, { flags: 'a', mode: 0o600 }); const stderrLog = fs.createWriteStream(runtimePaths.stderrPath, { flags: 'a', mode: 0o600 }); const child = spawnCli(claudePath, runtimeArgs, { cwd, detached: true, env: { ...provisioningEnv.env, ...nativeBootstrapEnv, [TEAMMATE_RUNTIME_ENV]: 'headless', [TEAMMATE_RUNTIME_EVENTS_ENV]: runtimePaths.eventsPath, [TEAMMATE_BOOTSTRAP_PROOF_TOKEN_ENV]: bootstrapProofToken, }, stdio: ['pipe', 'pipe', 'pipe'], }); if (!child.pid) { stdoutLog.destroy(); stderrLog.destroy(); throw new Error(`Failed to spawn teammate process for ${agentId}: missing pid`); } const runtimePid = child.pid; const processPaneId = `process:${runtimePid}`; const runtimeEventSource = `TeamProvisioningService.direct_process_${operation}`; child.stdout?.pipe(stdoutLog); child.stderr?.pipe(stderrLog); child.stdin?.on('error', (error) => { logger.debug( `[${input.teamName}] Direct process ${operation} stdin failed for ${agentId}: ${error.message}` ); }); child.once('close', (code, signal) => { void this.appendDirectProcessRuntimeEvent({ type: 'exited', eventsPath: runtimePaths.eventsPath, pid: runtimePid, teamName: input.teamName, agentName: input.configuredMember.name, agentId, runId: parentSessionId, bootstrapRunId: input.run.runId, source: runtimeEventSource, detail: code !== null ? `process exited with code ${code}` : signal ? `process exited from signal ${signal}` : 'process exited', }); stdoutLog.end(); stderrLog.end(); }); child.once('error', (error) => { void this.appendDirectProcessRuntimeEvent({ type: 'failed', eventsPath: runtimePaths.eventsPath, pid: runtimePid, teamName: input.teamName, agentName: input.configuredMember.name, agentId, runId: parentSessionId, bootstrapRunId: input.run.runId, source: runtimeEventSource, detail: `process error: ${error.message}`, }); }); (child.stdin as { unref?: () => void } | null)?.unref?.(); (child.stdout as { unref?: () => void } | null)?.unref?.(); (child.stderr as { unref?: () => void } | null)?.unref?.(); child.unref(); try { await this.appendDirectProcessRuntimeEvent({ type: 'process_spawned', eventsPath: runtimePaths.eventsPath, pid: runtimePid, teamName: input.teamName, agentName: input.configuredMember.name, agentId, runId: parentSessionId, bootstrapRunId: input.run.runId, source: runtimeEventSource, detail: 'process spawned', }); await this.appendDirectProcessRuntimeEvent({ type: 'stdout_attached', eventsPath: runtimePaths.eventsPath, pid: runtimePid, teamName: input.teamName, agentName: input.configuredMember.name, agentId, runId: parentSessionId, bootstrapRunId: input.run.runId, source: runtimeEventSource, detail: 'stdout and stderr attached', }); await this.updateDirectTmuxRestartMemberConfig({ teamName: input.teamName, memberName: input.memberName, member: memberSpec, agentId, color, prompt, paneId: processPaneId, cwd, providerId, joinedAt: Date.now(), bootstrapExpectedAfter, backendType: 'process', runtimePid, bootstrapRuntimeEventsPath: runtimePaths.eventsPath, bootstrapProofToken, bootstrapRunId: input.run.runId, ...(nativeBootstrapSpec ? { bootstrapContextHash: nativeBootstrapSpec.contextHash, bootstrapBriefingHash: nativeBootstrapSpec.briefingHash, } : {}), }); this.enqueueDirectRestartPrompt({ teamName: input.teamName, memberName: input.configuredMember.name, leadName: input.leadName, leadSessionId: parentSessionId, prompt, operation, }); await this.appendDirectProcessRuntimeEvent({ type: 'mailbox_bootstrap_written', eventsPath: runtimePaths.eventsPath, pid: runtimePid, teamName: input.teamName, agentName: input.configuredMember.name, agentId, runId: parentSessionId, bootstrapRunId: input.run.runId, source: runtimeEventSource, }); this.upsertRunAllEffectiveMember(input.run, memberSpec); this.appendMemberBootstrapDiagnostic( input.run, input.memberName, operation === 'manual_restart' ? `restart process spawned with pid ${runtimePid}` : `runtime process spawned with pid ${runtimePid}` ); this.setMemberSpawnStatus(input.run, input.memberName, 'waiting'); } catch (error) { try { killProcessByPid(runtimePid); } catch (killError) { logger.warn( `[${input.teamName}] Failed to stop orphaned direct process ${agentId} pid=${runtimePid}: ${ killError instanceof Error ? killError.message : String(killError) }` ); } stdoutLog.end(); stderrLog.end(); throw error; } } private getDirectProcessRestartRuntimePaths( teamName: string, memberName: string ): { dir: string; eventsPath: string; stdoutPath: string; stderrPath: string } { const dir = getTeamRuntimeEventsDir(teamName); const filePrefix = sanitizeProcessRuntimeEventFilePrefix(memberName); return { dir, eventsPath: path.join(dir, `${filePrefix}.runtime.jsonl`), stdoutPath: path.join(dir, `${filePrefix}.stdout.log`), stderrPath: path.join(dir, `${filePrefix}.stderr.log`), }; } private async materializeDirectProcessNativeBootstrapContext(input: { teamName: string; memberName: string; agentId: string; providerId: TeamProviderId; runId: string; bootstrapProofToken: string; spec: NativeAppManagedBootstrapSpec | null; }): Promise> { if (!input.spec || (input.providerId !== 'anthropic' && input.providerId !== 'codex')) { return {}; } const context = { ...input.spec, kind: 'native_app_managed_bootstrap', teamName: input.teamName, memberName: input.memberName, agentId: input.agentId, runId: input.runId, provider: input.providerId, bootstrapProofToken: input.bootstrapProofToken, }; const dir = path.join(getTeamRuntimeEventsDir(input.teamName), 'native-bootstrap'); await fs.promises.mkdir(dir, { recursive: true }); const finalPath = path.join( dir, `${sanitizeProcessRuntimeEventFilePrefix(input.memberName)}-${randomUUID()}.native-bootstrap.json` ); const tempPath = `${finalPath}.tmp`; await fs.promises.writeFile(tempPath, JSON.stringify(context), { encoding: 'utf8', mode: 0o600, }); await fs.promises.rename(tempPath, finalPath); return { [NATIVE_APP_MANAGED_BOOTSTRAP_CONTEXT_ENV]: finalPath }; } private async appendDirectProcessRuntimeEvent(input: { type: string; eventsPath: string; pid: number; teamName: string; agentName: string; agentId: string; runId: string; bootstrapRunId: string; source: string; detail?: string; }): Promise { await fs.promises.mkdir(path.dirname(input.eventsPath), { recursive: true }); await fs.promises.appendFile( input.eventsPath, `${JSON.stringify({ version: 1, type: input.type, timestamp: nowIso(), pid: input.pid, teamName: input.teamName, agentName: input.agentName, agentId: input.agentId, runId: input.runId, bootstrapRunId: input.bootstrapRunId, source: input.source, ...(input.detail ? { detail: input.detail } : {}), })}\n`, { encoding: 'utf8', mode: 0o600 } ); } private getMemberLifecycleOperationKey(teamName: string, memberName: string): string { return `${teamName.trim().toLowerCase()}\u0000${memberName.trim().toLowerCase()}`; } private getActiveMemberLifecycleOperation( teamName: string, memberName: string ): MemberLifecycleOperation | null { return ( this.memberLifecycleOperations.get( this.getMemberLifecycleOperationKey(teamName, memberName) ) ?? null ); } private isMemberLifecycleOperationActive(teamName: string, memberName: string): boolean { return this.getActiveMemberLifecycleOperation(teamName, memberName) !== null; } private createMemberLifecycleOperationInProgressError(memberName: string): Error { return new Error(`Lifecycle operation for teammate "${memberName}" is already in progress`); } private isMemberLifecycleOperationInProgressError(error: unknown): boolean { return ( error instanceof Error && /^Lifecycle operation for teammate ".+" is already in progress$/.test(error.message) ); } private async runMemberLifecycleOperation( teamName: string, memberName: string, kind: MemberLifecycleOperationKind, operation: () => Promise ): Promise { const key = this.getMemberLifecycleOperationKey(teamName, memberName); if (this.memberLifecycleOperations.has(key)) { throw this.createMemberLifecycleOperationInProgressError(memberName); } const token = Symbol(`${kind}:${teamName}:${memberName}`); this.memberLifecycleOperations.set(key, { kind, token, startedAtMs: Date.now(), }); this.invalidateRuntimeSnapshotCaches(teamName); try { return await operation(); } finally { if (this.memberLifecycleOperations.get(key)?.token === token) { this.memberLifecycleOperations.delete(key); } this.invalidateRuntimeSnapshotCaches(teamName); } } private getOpenCodeReattachLifecycleKind( reason?: 'member_added' | 'member_updated' | 'manual_restart' ): MemberLifecycleOperationKind { if (reason === 'member_added') return 'opencode_member_added'; if (reason === 'member_updated') return 'opencode_member_updated'; return 'manual_restart'; } private getLiveRosterAttachLifecycleKind( reason?: LiveRosterAttachReason ): MemberLifecycleOperationKind { if (reason === 'member_restored') return 'primary_member_restored'; if (reason === 'member_updated') return 'primary_member_updated'; return 'primary_member_added'; } async attachLiveRosterMember( teamName: string, memberName: string, options?: { reason?: LiveRosterAttachReason } ): Promise { return this.runMemberLifecycleOperation( teamName, memberName, this.getLiveRosterAttachLifecycleKind(options?.reason), () => this.attachLiveRosterMemberUnlocked(teamName, memberName, options) ); } private async stopPrimaryOwnedRosterRuntime(input: { teamName: string; memberName: string; persistedRuntimeMembers: readonly PersistedRuntimeMemberLike[]; liveRuntimeByMember: Map; actionLabel: string; }): Promise { const pidsToStop = new Set(); const tmuxPaneIdsToStop = new Set(); let hasAliveRuntimeWithoutStopHandle = false; for (const runtimeMember of input.persistedRuntimeMembers) { const backendType = runtimeMember.backendType?.trim().toLowerCase(); if (backendType === 'in-process') { throw new Error( `Member "${input.memberName}" uses an in-process runtime and cannot be detached here` ); } if ( backendType === 'process' && typeof runtimeMember.runtimePid === 'number' && Number.isFinite(runtimeMember.runtimePid) && runtimeMember.runtimePid > 0 ) { pidsToStop.add(runtimeMember.runtimePid); } const paneId = typeof runtimeMember.tmuxPaneId === 'string' ? runtimeMember.tmuxPaneId.trim() : ''; if (backendType === 'tmux' && paneId) { tmuxPaneIdsToStop.add(paneId); } } for (const [candidateName, metadata] of input.liveRuntimeByMember.entries()) { if (!matchesObservedMemberNameForExpected(candidateName, input.memberName)) { continue; } if (metadata.backendType === 'in-process') { throw new Error( `Member "${input.memberName}" uses an in-process runtime and cannot be detached here` ); } let hasStopHandle = false; if (metadata.backendType === 'tmux') { const paneId = metadata.tmuxPaneId?.trim(); if (paneId) { tmuxPaneIdsToStop.add(paneId); hasStopHandle = true; } } if (typeof metadata.pid === 'number' && Number.isFinite(metadata.pid) && metadata.pid > 0) { pidsToStop.add(metadata.pid); hasStopHandle = true; } if ( typeof metadata.metricsPid === 'number' && Number.isFinite(metadata.metricsPid) && metadata.metricsPid > 0 ) { pidsToStop.add(metadata.metricsPid); hasStopHandle = true; } if (metadata.alive && !hasStopHandle) { hasAliveRuntimeWithoutStopHandle = true; } } if (hasAliveRuntimeWithoutStopHandle) { throw new Error( `${input.actionLabel} cannot stop the existing runtime because it does not expose a pid or tmux pane.` ); } for (const paneId of tmuxPaneIdsToStop) { try { killTmuxPaneForCurrentPlatformSync(paneId); } catch (error) { logger.debug( `[${input.teamName}] Failed to stop teammate pane ${input.memberName} ${paneId} for live roster lifecycle: ${ error instanceof Error ? error.message : String(error) }` ); } } for (const pid of pidsToStop) { try { killProcessByPid(pid); } catch (error) { logger.debug( `[${input.teamName}] Failed to stop teammate process ${input.memberName} pid=${pid} for live roster lifecycle: ${ error instanceof Error ? error.message : String(error) }` ); } } if (pidsToStop.size > 0) { const lingeringPids = await waitForPidsToExit([...pidsToStop], { timeoutMs: 1_500, pollMs: 100, }); if (lingeringPids.length > 0) { throw new Error( `${input.actionLabel} is still waiting for process exit (${lingeringPids.join(', ')}).` ); } } if (tmuxPaneIdsToStop.size > 0) { const lingeringPaneIds = await waitForTmuxPanesToExit([...tmuxPaneIdsToStop], { timeoutMs: 1_500, pollMs: 100, }); if (lingeringPaneIds.length > 0) { throw new Error( `${input.actionLabel} is still waiting for tmux pane exit (${lingeringPaneIds.join(', ')}).` ); } } } private async attachLiveRosterMemberUnlocked( teamName: string, memberName: string, options?: { reason?: LiveRosterAttachReason } ): Promise { const run = this.getMutableAliveRunOrThrow(teamName); const config = await this.readConfigForStrictDecision(teamName); if (!config) { throw new Error(`Team "${teamName}" configuration is no longer available`); } const metaMembers = await this.membersMetaStore.getMembers(teamName).catch(() => []); const configuredMember = this.resolveEffectiveConfiguredMember( config.members ?? [], metaMembers, memberName ); if (!configuredMember) { throw new Error(`Member "${memberName}" is not configured in team "${teamName}"`); } if (configuredMember.removedAt) { throw new Error(`Member "${memberName}" has been removed`); } if (isLeadMember({ name: configuredMember.name, agentType: configuredMember.agentType })) { throw new Error('Lead attach is not supported from member controls'); } const leadProviderId = resolveTeamProviderId(run.request.providerId); const desiredProviderId = normalizeOptionalTeamProviderId(configuredMember.providerId) ?? leadProviderId; if (desiredProviderId === 'opencode') { await this.reattachOpenCodeOwnedMemberLaneUnlocked(teamName, memberName, { reason: options?.reason === 'member_updated' ? 'member_updated' : 'member_added', }); return; } if (leadProviderId === 'opencode') { throw new Error( 'OpenCode-led mixed teams are not supported in this phase. Stop the team and relaunch with a non-OpenCode lead.' ); } const currentStatus = run.memberSpawnStatuses.get(memberName); const currentUpdatedAtMs = parseOptionalIsoMs(currentStatus?.updatedAt); const currentStatusAgeMs = currentUpdatedAtMs > 0 ? Date.now() - currentUpdatedAtMs : Number.POSITIVE_INFINITY; const currentSpawnLooksFresh = currentStatus?.status === 'spawning' && currentStatusAgeMs < MEMBER_BOOTSTRAP_STALL_MS; if (currentSpawnLooksFresh || currentStatus?.launchState === 'runtime_pending_permission') { throw new Error(`Launch for teammate "${memberName}" is already in progress`); } const replaceExistingRuntime = options?.reason === 'member_updated'; const liveRuntimeByMember = await this.getLiveTeamAgentRuntimeMetadata(teamName).catch( () => new Map() ); const liveRuntimeMember = liveRuntimeByMember.get(memberName) ?? [...liveRuntimeByMember.entries()].find(([candidateName]) => matchesObservedMemberNameForExpected(candidateName, memberName) )?.[1]; if ( !replaceExistingRuntime && liveRuntimeMember?.alive && liveRuntimeMember.livenessKind === 'runtime_process' ) { this.upsertRunAllEffectiveMember( run, this.buildPrimaryOwnedMemberSpecForRuntime({ configuredMember, run, }) ); this.setMemberSpawnStatus(run, memberName, 'online', undefined, 'process'); return; } if ( !replaceExistingRuntime && liveRuntimeMember?.alive && (liveRuntimeMember.livenessKind === 'runtime_process_candidate' || liveRuntimeMember.livenessKind === 'permission_blocked') && currentStatus?.launchState === 'runtime_pending_bootstrap' ) { throw new Error(`Launch for teammate "${memberName}" is already in progress`); } const persistedRuntimeMembers = this.readPersistedRuntimeMembers(teamName).filter((member) => { const candidateName = typeof member.name === 'string' ? member.name.trim() : ''; return candidateName.length > 0 && matchesMemberNameOrBase(candidateName, memberName); }); const backendTypes = new Set( persistedRuntimeMembers .map((member) => member.backendType?.trim().toLowerCase()) .filter((value): value is string => Boolean(value)) ); if (backendTypes.has('in-process')) { throw new Error( `Member "${memberName}" uses an in-process runtime and cannot be attached here` ); } if (replaceExistingRuntime) { await this.stopPrimaryOwnedRosterRuntime({ teamName, memberName, persistedRuntimeMembers, liveRuntimeByMember, actionLabel: `Update for teammate "${memberName}"`, }); this.setMemberSpawnStatus(run, memberName, 'offline'); } this.invalidateRuntimeSnapshotCaches(teamName); this.resetRuntimeToolActivity(run, memberName); this.clearMemberSpawnToolTracking(run, memberName); run.pendingMemberRestarts.delete(memberName); this.setMemberSpawnStatus(run, memberName, 'spawning'); if (currentStatus?.launchState === 'runtime_pending_bootstrap') { this.appendMemberBootstrapDiagnostic( run, memberName, 'stale runtime_pending_bootstrap without live runtime process; retrying launch' ); } this.appendMemberBootstrapDiagnostic( run, memberName, `live roster ${options?.reason ?? 'member_added'} requested app-managed runtime process` ); try { await this.launchDirectProcessMemberRestart({ run, teamName, displayName: config.name?.trim() || teamName, leadName: this.resolveLeadMemberName(config.members ?? [], metaMembers), memberName, config, configuredMember, persistedRuntimeMembers, operation: options?.reason ?? 'member_added', }); } catch (error) { this.setMemberSpawnStatus( run, memberName, 'error', error instanceof Error ? error.message : String(error) ); if (run.isLaunch) { await this.persistLaunchStateSnapshot( run, run.provisioningComplete ? 'finished' : 'active' ); } throw error; } } async detachLiveRosterMember(teamName: string, memberName: string): Promise { return this.runMemberLifecycleOperation(teamName, memberName, 'primary_member_removed', () => this.detachLiveRosterMemberUnlocked(teamName, memberName) ); } private async detachLiveRosterMemberUnlocked( teamName: string, memberName: string ): Promise { const run = this.getMutableAliveRunOrThrow(teamName); const leadProviderId = resolveTeamProviderId(run.request.providerId); const config = await this.readConfigForStrictDecision(teamName); const metaMembers = await this.membersMetaStore.getMembers(teamName).catch(() => []); const configuredMember = this.resolveEffectiveConfiguredMember( config?.members ?? [], metaMembers, memberName ); const desiredProviderId = normalizeOptionalTeamProviderId(configuredMember?.providerId) ?? leadProviderId; if (desiredProviderId === 'opencode') { await this.detachOpenCodeOwnedMemberLaneUnlocked(teamName, memberName); return; } if (leadProviderId === 'opencode') { throw new Error( 'OpenCode-led mixed teams are not supported in this phase. Stop the team and relaunch with a non-OpenCode lead.' ); } const persistedRuntimeMembers = this.readPersistedRuntimeMembers(teamName).filter((member) => { const candidateName = typeof member.name === 'string' ? member.name.trim() : ''; return candidateName.length > 0 && matchesMemberNameOrBase(candidateName, memberName); }); const liveRuntimeByMember = await this.getLiveTeamAgentRuntimeMetadata(teamName).catch( () => new Map() ); await this.stopPrimaryOwnedRosterRuntime({ teamName, memberName, persistedRuntimeMembers, liveRuntimeByMember, actionLabel: `Detach for teammate "${memberName}"`, }); this.removeRunAllEffectiveMember(run, memberName); this.invalidateRuntimeSnapshotCaches(teamName); this.resetRuntimeToolActivity(run, memberName); this.clearMemberSpawnToolTracking(run, memberName); run.pendingMemberRestarts.delete(memberName); this.setMemberSpawnStatus(run, memberName, 'offline'); if (run.isLaunch) { await this.persistLaunchStateSnapshot(run, run.provisioningComplete ? 'finished' : 'active'); } } async restartMember(teamName: string, memberName: string): Promise { return this.runMemberLifecycleOperation(teamName, memberName, 'manual_restart', () => this.restartMemberUnlocked(teamName, memberName) ); } private async restartMemberUnlocked(teamName: string, memberName: string): Promise { const runId = this.getAliveRunId(teamName); if (!runId) { if (await this.restartPureOpenCodePrimaryMemberWithoutTrackedRun(teamName, memberName)) { return; } throw new Error(`Team "${teamName}" is not currently running`); } const run = this.runs.get(runId); if (!run || run.processKilled || run.cancelRequested) { if (await this.restartPureOpenCodePrimaryMemberWithoutTrackedRun(teamName, memberName)) { return; } throw new Error(`Team "${teamName}" is not currently running`); } const readCurrentConfiguredMember = async (): Promise<{ config: TeamConfig | null; configuredMembers: TeamConfig['members']; metaMembers: Awaited>; configuredMember: ReturnType; }> => { const config = await this.readConfigForStrictDecision(teamName); const configuredMembers = config?.members ?? []; let metaMembers: Awaited> = []; try { metaMembers = await this.membersMetaStore.getMembers(teamName); } catch { metaMembers = []; } return { config, configuredMembers, metaMembers, configuredMember: this.resolveEffectiveConfiguredMember( configuredMembers, metaMembers, memberName ), }; }; let currentConfiguredMemberState = await readCurrentConfiguredMember(); let config = currentConfiguredMemberState.config; let configuredMember = currentConfiguredMemberState.configuredMember; if (!config) { throw new Error(`Team "${teamName}" configuration is no longer available`); } if (!configuredMember) { throw new Error(`Member "${memberName}" is not configured in team "${teamName}"`); } if (configuredMember.removedAt) { throw new Error(`Member "${memberName}" has been removed`); } if (isLeadMember({ name: memberName, agentType: configuredMember.agentType })) { throw new Error('Lead restart is not supported from member controls'); } const desiredProviderId = normalizeOptionalTeamProviderId(configuredMember.providerId); const mixedSecondaryLanes = run.mixedSecondaryLanes ?? []; const liveSecondaryLaneMemberName = mixedSecondaryLanes .find((lane) => lane.member.name.trim() === memberName) ?.member.name?.trim() ?? null; const leadProviderId = resolveTeamProviderId(run.request.providerId); const desiredSecondaryLane = desiredProviderId === 'opencode' && leadProviderId !== 'opencode'; if (liveSecondaryLaneMemberName === memberName || desiredSecondaryLane) { await this.reattachOpenCodeOwnedMemberLaneUnlocked(teamName, memberName, { reason: 'manual_restart', }); return; } if (run.pendingMemberRestarts.has(memberName)) { throw new Error(`Restart for teammate "${memberName}" is already in progress`); } const persistedRuntimeMembers = this.readPersistedRuntimeMembers(teamName).filter((member) => { const candidateName = typeof member.name === 'string' ? member.name.trim() : ''; return candidateName.length > 0 && matchesMemberNameOrBase(candidateName, memberName); }); const directTmuxRestartCandidatePaneId = this.getDirectTmuxRestartPaneId( persistedRuntimeMembers, memberName ); const backendTypes = new Set( persistedRuntimeMembers .map((member) => member.backendType?.trim().toLowerCase()) .filter((value): value is string => Boolean(value)) ); if (backendTypes.has('in-process')) { throw new Error( `Member "${memberName}" uses an in-process runtime and cannot be restarted here` ); } this.invalidateRuntimeSnapshotCaches(teamName); const liveRuntimeByMember = await this.getLiveTeamAgentRuntimeMetadata(teamName); const livePids = new Set(); let hasAliveRuntimeWithoutPid = false; for (const [candidateName, metadata] of liveRuntimeByMember.entries()) { if (!matchesMemberNameOrBase(candidateName, memberName)) { continue; } if (metadata.pid) { livePids.add(metadata.pid); continue; } if (metadata.alive && metadata.backendType !== 'in-process') { hasAliveRuntimeWithoutPid = true; } } if (hasAliveRuntimeWithoutPid) { throw new Error( `Member "${memberName}" is running, but its backend does not expose a restartable pid yet` ); } let directTmuxRestartPaneId: string | null = null; if (directTmuxRestartCandidatePaneId) { try { const paneInfo = ( await listTmuxPaneRuntimeInfoForCurrentPlatform([directTmuxRestartCandidatePaneId]) ).get(directTmuxRestartCandidatePaneId); if (paneInfo && isInteractiveShellCommand(paneInfo.currentCommand)) { directTmuxRestartPaneId = directTmuxRestartCandidatePaneId; } } catch (error) { logger.debug( `[${teamName}] Direct tmux restart probe failed for ${memberName}: ${ error instanceof Error ? error.message : String(error) }` ); } } const tmuxPaneIdsToVerify: string[] = []; if (!directTmuxRestartPaneId) { for (const persistedRuntimeMember of persistedRuntimeMembers) { const paneId = typeof persistedRuntimeMember.tmuxPaneId === 'string' ? persistedRuntimeMember.tmuxPaneId.trim() : ''; const backendType = persistedRuntimeMember.backendType?.trim().toLowerCase(); if (!paneId || backendType !== 'tmux') { continue; } tmuxPaneIdsToVerify.push(paneId); try { killTmuxPaneForCurrentPlatformSync(paneId); logger.info( `[${teamName}] Killed teammate pane ${memberName} (${paneId}) for manual restart` ); } catch (error) { logger.debug( `[${teamName}] Failed to kill teammate pane ${memberName} (${paneId}) for manual restart: ${ error instanceof Error ? error.message : String(error) }` ); } } } for (const pid of livePids) { try { killProcessByPid(pid); } catch (error) { logger.debug( `[${teamName}] Failed to kill teammate process ${memberName} pid=${pid} for manual restart: ${ error instanceof Error ? error.message : String(error) }` ); } } if (livePids.size > 0) { const lingeringPids = await waitForPidsToExit(Array.from(livePids), { timeoutMs: 1_500, pollMs: 100, }); if (lingeringPids.length > 0) { throw new Error( `Restart for teammate "${memberName}" is still waiting for the previous process to exit (${lingeringPids.join(', ')}).` ); } } if (tmuxPaneIdsToVerify.length > 0) { let lingeringPaneIds: string[]; try { lingeringPaneIds = await waitForTmuxPanesToExit(tmuxPaneIdsToVerify, { timeoutMs: 1_500, pollMs: 100, }); } catch (error) { throw new Error( `Restart for teammate "${memberName}" could not verify that the previous tmux pane exited: ${ error instanceof Error ? error.message : String(error) }` ); } if (lingeringPaneIds.length > 0) { throw new Error( `Restart for teammate "${memberName}" is still waiting for the previous tmux pane to exit (${lingeringPaneIds.join(', ')}).` ); } } this.setMemberSpawnStatus(run, memberName, 'offline'); const latestRunId = this.getAliveRunId(teamName); const currentRun = this.runs.get(runId); if ( latestRunId !== runId || !currentRun || currentRun !== run || currentRun.processKilled || currentRun.cancelRequested ) { throw new Error(`Team "${teamName}" is not currently running`); } currentConfiguredMemberState = await readCurrentConfiguredMember(); config = currentConfiguredMemberState.config; configuredMember = currentConfiguredMemberState.configuredMember; if (!config) { throw new Error(`Team "${teamName}" configuration disappeared while restart was in progress`); } if (!configuredMember) { throw new Error( `Member "${memberName}" is no longer configured in team "${teamName}" after restart preparation` ); } if (configuredMember.removedAt) { throw new Error(`Member "${memberName}" was removed while restart was in progress`); } if (isLeadMember({ name: memberName, agentType: configuredMember.agentType })) { throw new Error('Lead restart is not supported from member controls'); } run.pendingMemberRestarts.set(memberName, { requestedAt: nowIso(), desired: { name: configuredMember.name, role: configuredMember.role, workflow: configuredMember.workflow, isolation: configuredMember.isolation === 'worktree' ? ('worktree' as const) : undefined, providerId: configuredMember.providerId, model: configuredMember.model, effort: configuredMember.effort, }, }); this.invalidateRuntimeSnapshotCaches(teamName); this.resetRuntimeToolActivity(run, memberName); this.clearMemberSpawnToolTracking(run, memberName); this.setMemberSpawnStatus(run, memberName, 'spawning'); this.appendMemberBootstrapDiagnostic(run, memberName, 'manual restart requested from UI'); const leadName = this.resolveLeadMemberName( currentConfiguredMemberState.configuredMembers, currentConfiguredMemberState.metaMembers ); if (directTmuxRestartPaneId) { try { await this.launchDirectTmuxMemberRestart({ run, teamName, displayName: config?.name?.trim() || teamName, leadName, memberName, config, configuredMember, persistedRuntimeMembers, paneId: directTmuxRestartPaneId, }); return; } catch (error) { run.pendingMemberRestarts.delete(memberName); this.setMemberSpawnStatus( run, memberName, 'error', error instanceof Error ? error.message : String(error) ); if (run.isLaunch) { await this.persistLaunchStateSnapshot( run, run.provisioningComplete ? 'finished' : 'active' ); } throw error; } } const shouldDirectProcessRestart = backendTypes.has('process') || livePids.size > 0; if (shouldDirectProcessRestart) { try { await this.launchDirectProcessMemberRestart({ run, teamName, displayName: config?.name?.trim() || teamName, leadName, memberName, config, configuredMember, persistedRuntimeMembers, }); return; } catch (error) { run.pendingMemberRestarts.delete(memberName); this.setMemberSpawnStatus( run, memberName, 'error', error instanceof Error ? error.message : String(error) ); if (run.isLaunch) { await this.persistLaunchStateSnapshot( run, run.provisioningComplete ? 'finished' : 'active' ); } throw error; } } let restartMcpLaunchConfig: RuntimeBootstrapMemberMcpLaunchConfig | null = null; try { restartMcpLaunchConfig = await this.buildTrackedMemberMcpLaunchConfig({ cwd: configuredMember.cwd?.trim() || config.projectPath?.trim() || run.request.cwd, mcpPolicy: configuredMember.mcpPolicy, run, }); const restartMessage = buildRestartMemberSpawnMessage( teamName, config?.name?.trim() || teamName, leadName, { name: configuredMember.name, role: configuredMember.role, workflow: configuredMember.workflow, isolation: configuredMember.isolation === 'worktree' ? ('worktree' as const) : undefined, providerId: configuredMember.providerId, model: configuredMember.model, effort: configuredMember.effort, }, restartMcpLaunchConfig ); await this.sendMessageToRun(run, restartMessage); } catch (error) { await this.removeTrackedMemberMcpLaunchConfig(run, restartMcpLaunchConfig).catch(() => {}); run.pendingMemberRestarts.delete(memberName); this.setMemberSpawnStatus( run, memberName, 'error', error instanceof Error ? error.message : String(error) ); if (run.isLaunch) { await this.persistLaunchStateSnapshot( run, run.provisioningComplete ? 'finished' : 'active' ); } throw error; } } private async restartPureOpenCodePrimaryMemberWithoutTrackedRun( teamName: string, memberName: string ): Promise { const runtimeRun = this.runtimeAdapterRunByTeam.get(teamName); if (runtimeRun?.providerId !== 'opencode') { return false; } const adapter = this.getOpenCodeRuntimeAdapter(); if (!adapter) { throw new Error('OpenCode runtime adapter is not available for member restart.'); } const config = await this.readConfigForStrictDecision(teamName); if (!config) { return false; } const [teamMeta, metaMembers] = await Promise.all([ this.teamMetaStore.getMeta(teamName).catch(() => null), this.membersMetaStore.getMembers(teamName).catch(() => []), ]); const configuredMember = this.resolveEffectiveConfiguredMember( config.members ?? [], metaMembers, memberName ); if (!configuredMember) { throw new Error(`Member "${memberName}" is not configured in team "${teamName}"`); } if (configuredMember.removedAt) { throw new Error(`Member "${memberName}" has been removed`); } if (isLeadMember({ name: configuredMember.name, agentType: configuredMember.agentType })) { throw new Error('Lead restart is not supported from member controls'); } const leadMember = config.members?.find((member) => isLeadMember(member)); const leadProviderId = normalizeOptionalTeamProviderId(teamMeta?.providerId) ?? normalizeOptionalTeamProviderId(leadMember?.providerId); if (leadProviderId !== 'opencode') { return false; } const configuredNames = new Set(); for (const member of config.members ?? []) { const name = member.name?.trim(); if (name) { configuredNames.add(name); } } for (const member of metaMembers) { const name = member.name?.trim(); if (name) { configuredNames.add(name); } } const activeMembers = [...configuredNames] .map((name) => this.resolveEffectiveConfiguredMember(config.members ?? [], metaMembers, name)) .filter( ( member ): member is NonNullable< ReturnType > => { if (!member || member.removedAt) { return false; } return !isLeadMember({ name: member.name, agentType: member.agentType }); } ) .sort((left, right) => left.name.localeCompare(right.name)); const targetMember = activeMembers.find((member) => matchesExactTeamMemberName(member.name, configuredMember.name) ); if (!targetMember) { throw new Error(`Member "${memberName}" is not configured in team "${teamName}"`); } const nonOpenCodeMember = activeMembers.find((member) => { const providerId = normalizeOptionalTeamProviderId(member.providerId) ?? leadProviderId; return providerId !== 'opencode'; }); if (nonOpenCodeMember) { return false; } const projectPath = targetMember.cwd?.trim() || config.projectPath?.trim() || teamMeta?.cwd?.trim() || runtimeRun.cwd?.trim() || this.readPersistedTeamProjectPath(teamName); if (!projectPath) { throw new Error(`Team "${teamName}" project path is not available for OpenCode restart`); } const effectiveMembers = await this.resolveOpenCodeMemberWorkspacesForRuntime({ teamName, baseCwd: projectPath, leadProviderId: 'opencode', members: activeMembers.map((member) => this.buildConfiguredProvisioningMember(member)), }); const targetRuntimeMember = effectiveMembers.find((member) => matchesExactTeamMemberName(member.name, targetMember.name) ); if (!targetRuntimeMember) { throw new Error(`Member "${memberName}" could not be resolved for OpenCode restart`); } this.invalidateRuntimeSnapshotCaches(teamName); this.persistOpenCodeMemberRestartSystemMessage({ teamName, leadName: leadMember?.name?.trim() || 'team-lead', leadSessionId: runtimeRun.runId, displayName: config.description?.trim() || config.name, member: targetRuntimeMember, reason: 'manual_restart', }); await this.runOpenCodeTeamRuntimeAdapterLaunch({ request: { teamName, cwd: projectPath, prompt: teamMeta?.prompt?.trim() || '', providerId: 'opencode', providerBackendId: migrateProviderBackendId('opencode', teamMeta?.providerBackendId), model: targetRuntimeMember.model?.trim() || teamMeta?.model, effort: targetRuntimeMember.effort ?? (isTeamEffortLevel(teamMeta?.effort) ? teamMeta.effort : undefined), fastMode: teamMeta?.fastMode, limitContext: teamMeta?.limitContext, skipPermissions: teamMeta?.skipPermissions, worktree: teamMeta?.worktree, extraCliArgs: teamMeta?.extraCliArgs, }, members: effectiveMembers, prompt: [ `Restarting OpenCode teammate "${targetRuntimeMember.name}" by user request.`, 'This is an app-managed OpenCode-only runtime refresh. Re-establish the team sessions and continue from persisted team context.', ].join('\n'), sourceWarning: 'OpenCode-only member restart refreshes the primary OpenCode runtime lane because pure OpenCode teams do not keep a native lead run.', onProgress: () => undefined, }); this.invalidateRuntimeSnapshotCaches(teamName); return true; } async retryFailedOpenCodeSecondaryLanes( teamName: string ): Promise { const existing = this.failedOpenCodeSecondaryRetryInFlightByTeam.get(teamName); if (existing) { return existing; } const retry = this.retryFailedOpenCodeSecondaryLanesNow(teamName).finally(() => { this.failedOpenCodeSecondaryRetryInFlightByTeam.delete(teamName); }); this.failedOpenCodeSecondaryRetryInFlightByTeam.set(teamName, retry); return retry; } private async retryFailedOpenCodeSecondaryLanesNow( teamName: string ): Promise { const run = this.getMutableAliveRunOrThrow(teamName); if (this.getProvisioningRunId(teamName)) { throw new Error('Team launch is still in progress'); } const result: RetryFailedOpenCodeSecondaryLanesResult = { attempted: [], confirmed: [], pending: [], failed: [], skipped: [], }; const candidates = await this.collectFailedOpenCodeSecondaryRetryCandidates(run); for (const candidate of candidates) { if (!this.isCurrentTrackedRun(run) || run.processKilled || run.cancelRequested) { result.skipped.push({ memberName: candidate.memberName, reason: 'Team stopped during retry', }); continue; } try { await this.runMemberLifecycleOperation( teamName, candidate.memberName, 'opencode_retry', () => this.reattachOpenCodeOwnedMemberLaneUnlocked(teamName, candidate.memberName, { reason: 'manual_restart', }) ); result.attempted.push(candidate.memberName); const outcome = await this.readOpenCodeSecondaryRetryOutcome( run, candidate.memberName, candidate.laneId ); if (outcome.launchState === 'confirmed_alive') { result.confirmed.push(candidate.memberName); } else if (outcome.launchState === 'failed_to_start') { result.failed.push({ memberName: candidate.memberName, error: outcome.reason ?? 'OpenCode retry failed', }); } else if (outcome.launchState === 'skipped_for_launch') { result.skipped.push({ memberName: candidate.memberName, reason: outcome.reason ?? 'Teammate is skipped for this launch', }); } else { result.pending.push(candidate.memberName); } } catch (error) { if (this.isMemberLifecycleOperationInProgressError(error)) { result.skipped.push({ memberName: candidate.memberName, reason: 'Lifecycle operation already in progress', }); } else { result.failed.push({ memberName: candidate.memberName, error: error instanceof Error ? error.message : String(error), }); } } } await this.notifyLeadAboutConfirmedOpenCodeRetries(run, result); return result; } private async collectFailedOpenCodeSecondaryRetryCandidates( run: ProvisioningRun ): Promise { const teamName = run.teamName; const leadProviderId = resolveTeamProviderId(run.request.providerId); if (leadProviderId === 'opencode') { throw new Error( 'Retrying OpenCode secondary lanes is only supported for mixed teams with a non-OpenCode lead.' ); } if (!this.getOpenCodeRuntimeAdapter()) { throw new Error('OpenCode runtime adapter is not available for secondary lane retry.'); } const config = await this.readConfigForStrictDecision(teamName); if (!config) { throw new Error(`Team "${teamName}" configuration is no longer available`); } const metaMembers = await this.membersMetaStore.getMembers(teamName).catch(() => []); const persistedSnapshot = await this.launchStateStore.read(teamName).catch(() => null); const names = new Set(); for (const member of config.members ?? []) { const name = member.name?.trim(); if (name) { names.add(name); } } for (const member of metaMembers) { const name = member.name?.trim(); if (name) { names.add(name); } } for (const lane of run.mixedSecondaryLanes ?? []) { const name = lane.member.name?.trim(); if (name) { names.add(name); } } for (const name of persistedSnapshot?.expectedMembers ?? []) { if (name.trim()) { names.add(name.trim()); } } for (const name of Object.keys(persistedSnapshot?.members ?? {})) { if (name.trim()) { names.add(name.trim()); } } const candidates: OpenCodeSecondaryRetryCandidate[] = []; for (const memberName of [...names].sort((left, right) => left.localeCompare(right))) { const configuredMember = this.resolveEffectiveConfiguredMember( config.members ?? [], metaMembers, memberName ); if (!configuredMember || configuredMember.removedAt) { continue; } if (isLeadMember({ name: memberName, agentType: configuredMember.agentType })) { continue; } if (normalizeOptionalTeamProviderId(configuredMember.providerId) !== 'opencode') { continue; } const laneIdentity = buildPlannedMemberLaneIdentity({ leadProviderId, member: { name: memberName, providerId: 'opencode', }, }); if ( laneIdentity.laneKind !== 'secondary' || laneIdentity.laneOwnerProviderId !== 'opencode' ) { continue; } const existingLane = (run.mixedSecondaryLanes ?? []).find( (lane) => lane.laneId === laneIdentity.laneId || matchesTeamMemberIdentity(lane.member.name, memberName) ); const liveEntry = run.memberSpawnStatuses.get(memberName); const persistedMember = persistedSnapshot?.members[memberName] ?? Object.values(persistedSnapshot?.members ?? {}).find( (member) => member.laneId === laneIdentity.laneId ); if ( this.isRetryableFailedOpenCodeSecondaryLane({ liveEntry, persistedMember, existingLane, }) ) { candidates.push({ memberName, laneId: laneIdentity.laneId }); } } return candidates; } private isRetryableFailedOpenCodeSecondaryLane(input: { liveEntry?: MemberSpawnStatusEntry; persistedMember?: PersistedTeamLaunchMemberState; existingLane?: MixedSecondaryRuntimeLaneState; }): boolean { const { liveEntry, persistedMember, existingLane } = input; if (existingLane?.state === 'queued' || existingLane?.state === 'launching') { return false; } if ( liveEntry?.launchState === 'skipped_for_launch' || liveEntry?.skippedForLaunch === true || persistedMember?.launchState === 'skipped_for_launch' || persistedMember?.skippedForLaunch === true ) { return false; } if ( liveEntry?.launchState === 'runtime_pending_permission' || liveEntry?.launchState === 'runtime_pending_bootstrap' || persistedMember?.launchState === 'runtime_pending_permission' || persistedMember?.launchState === 'runtime_pending_bootstrap' || (liveEntry?.pendingPermissionRequestIds?.length ?? 0) > 0 || (persistedMember?.pendingPermissionRequestIds?.length ?? 0) > 0 ) { return false; } if (liveEntry?.launchState === 'starting' || liveEntry?.status === 'spawning') { return false; } if ( liveEntry?.launchState === 'confirmed_alive' || liveEntry?.bootstrapConfirmed === true || persistedMember?.launchState === 'confirmed_alive' || persistedMember?.bootstrapConfirmed === true ) { return false; } return ( liveEntry?.launchState === 'failed_to_start' || liveEntry?.status === 'error' || persistedMember?.launchState === 'failed_to_start' || persistedMember?.hardFailure === true ); } private async readOpenCodeSecondaryRetryOutcome( run: ProvisioningRun, memberName: string, laneId: string ): Promise { const lane = (run.mixedSecondaryLanes ?? []).find( (candidate) => candidate.laneId === laneId || matchesTeamMemberIdentity(candidate.member.name, memberName) ); const memberEvidence = lane?.result?.members[memberName] ?? Object.values(lane?.result?.members ?? {}).find((member) => matchesTeamMemberIdentity(member.memberName, memberName) ); const persistedSnapshot = await this.launchStateStore.read(run.teamName).catch(() => null); const persistedMember = persistedSnapshot?.members[memberName] ?? Object.values(persistedSnapshot?.members ?? {}).find((member) => member.laneId === laneId); const liveEntry = run.memberSpawnStatuses.get(memberName); if ( memberEvidence?.launchState === 'confirmed_alive' || memberEvidence?.bootstrapConfirmed === true || liveEntry?.launchState === 'confirmed_alive' || liveEntry?.bootstrapConfirmed === true || persistedMember?.launchState === 'confirmed_alive' || persistedMember?.bootstrapConfirmed === true ) { return { launchState: 'confirmed_alive' }; } if ( liveEntry?.launchState === 'skipped_for_launch' || liveEntry?.skippedForLaunch === true || persistedMember?.launchState === 'skipped_for_launch' || persistedMember?.skippedForLaunch === true ) { return { launchState: 'skipped_for_launch', reason: liveEntry?.skipReason ?? persistedMember?.skipReason, }; } if ( memberEvidence?.launchState === 'failed_to_start' || memberEvidence?.hardFailure === true || liveEntry?.launchState === 'failed_to_start' || liveEntry?.status === 'error' || persistedMember?.launchState === 'failed_to_start' || persistedMember?.hardFailure === true ) { return { launchState: 'failed_to_start', reason: this.selectOpenCodeSecondaryRetryFailureReason({ memberEvidence, liveEntry, persistedMember, }), }; } return { launchState: memberEvidence?.launchState ?? liveEntry?.launchState ?? persistedMember?.launchState ?? 'runtime_pending_bootstrap', }; } private selectOpenCodeSecondaryRetryFailureReason(input: { memberEvidence?: TeamRuntimeMemberLaunchEvidence; liveEntry?: MemberSpawnStatusEntry; persistedMember?: PersistedTeamLaunchMemberState; }): string | undefined { const diagnostics = [ input.memberEvidence?.hardFailureReason, input.memberEvidence?.runtimeDiagnostic, ...(input.memberEvidence?.diagnostics ?? []), input.liveEntry?.hardFailureReason, input.liveEntry?.runtimeDiagnostic, input.liveEntry?.error, input.persistedMember?.hardFailureReason, input.persistedMember?.runtimeDiagnostic, ]; return diagnostics .find( (diagnostic): diagnostic is string => typeof diagnostic === 'string' && diagnostic.trim().length > 0 ) ?.trim(); } private async notifyLeadAboutConfirmedOpenCodeRetries( run: ProvisioningRun, result: RetryFailedOpenCodeSecondaryLanesResult ): Promise { if (result.confirmed.length === 0) { return; } const confirmedNames = result.confirmed.map((name) => `@${name}`).join(', '); const message = [ `Системное замечание: повторный запуск OpenCode-тиммейтов подтверждён: ${confirmedNames}.`, `Их можно снова считать доступными.`, ].join(' '); await this.sendMessageToRun(run, message).catch((error: unknown) => logger.warn( `[${run.teamName}] failed to send OpenCode retry recovery notice to lead: ${ error instanceof Error ? error.message : String(error) }` ) ); } async skipMemberForLaunch(teamName: string, memberName: string): Promise { const normalizedMemberName = memberName.trim(); if (!normalizedMemberName) { throw new Error('Member name is required'); } const config = await this.readConfigForStrictDecision(teamName); if (!config) { throw new Error(`Team "${teamName}" configuration is no longer available`); } let metaMembers: Awaited> = []; try { metaMembers = await this.membersMetaStore.getMembers(teamName); } catch { metaMembers = []; } const configuredMember = this.resolveEffectiveConfiguredMember( config.members ?? [], metaMembers, normalizedMemberName ); if (!configuredMember) { throw new Error(`Member "${normalizedMemberName}" is not configured in team "${teamName}"`); } if (configuredMember.removedAt) { throw new Error(`Member "${normalizedMemberName}" has been removed`); } if (isLeadMember({ name: normalizedMemberName, agentType: configuredMember.agentType })) { throw new Error('Lead cannot be skipped for a launch'); } const runId = this.getTrackedRunId(teamName); const run = runId ? this.runs.get(runId) : undefined; const persistedSnapshot = await this.launchStateStore.read(teamName).catch(() => null); const runEntry = run?.memberSpawnStatuses.get(normalizedMemberName); const persistedMember = persistedSnapshot?.members[normalizedMemberName]; const alreadySkipped = runEntry?.launchState === 'skipped_for_launch' || runEntry?.skippedForLaunch === true || persistedMember?.launchState === 'skipped_for_launch' || persistedMember?.skippedForLaunch === true; if (alreadySkipped) { return; } const failedThisLaunch = runEntry?.launchState === 'failed_to_start' || runEntry?.status === 'error' || persistedMember?.launchState === 'failed_to_start' || persistedMember?.hardFailure === true; if (!failedThisLaunch) { throw new Error(`Member "${normalizedMemberName}" has not failed this launch`); } if (run?.pendingMemberRestarts.has(normalizedMemberName)) { throw new Error(`Restart for teammate "${normalizedMemberName}" is already in progress`); } const previousFailureReason = runEntry?.hardFailureReason ?? runEntry?.error ?? persistedMember?.hardFailureReason ?? persistedMember?.runtimeDiagnostic; const reason = previousFailureReason?.trim() ? `Skipped by user after launch failure: ${previousFailureReason.trim()}` : 'Skipped by user for this launch'; if (run && !run.processKilled && !run.cancelRequested) { this.invalidateRuntimeSnapshotCaches(teamName); this.resetRuntimeToolActivity(run, normalizedMemberName); this.clearMemberSpawnToolTracking(run, normalizedMemberName); this.setMemberSpawnStatus(run, normalizedMemberName, 'skipped', reason); if (run.isLaunch) { await this.persistLaunchStateSnapshot( run, run.provisioningComplete ? 'finished' : 'active' ); } try { await this.sendMessageToRun( run, `Teammate "${normalizedMemberName}" was skipped for this launch after a startup failure. Continue without waiting for this teammate unless the user retries it.` ); } catch (error) { logger.debug( `[${teamName}] Failed to notify lead about skipped teammate "${normalizedMemberName}": ${ error instanceof Error ? error.message : String(error) }` ); } return; } if (!persistedSnapshot || !persistedMember) { throw new Error(`No launch state is available for member "${normalizedMemberName}"`); } const updatedAt = nowIso(); const nextMembers = { ...persistedSnapshot.members, [normalizedMemberName]: { ...persistedMember, launchState: 'skipped_for_launch' as const, skippedForLaunch: true, skipReason: reason, skippedAt: updatedAt, agentToolAccepted: false, runtimeAlive: false, bootstrapConfirmed: false, hardFailure: false, hardFailureReason: undefined, pendingPermissionRequestIds: undefined, livenessKind: undefined, runtimeDiagnostic: undefined, runtimeDiagnosticSeverity: undefined, lastEvaluatedAt: updatedAt, diagnostics: [`skipped for this launch: ${reason}`], }, }; const nextSnapshot = createPersistedLaunchSnapshot({ teamName: persistedSnapshot.teamName, expectedMembers: persistedSnapshot.expectedMembers, bootstrapExpectedMembers: persistedSnapshot.bootstrapExpectedMembers, leadSessionId: persistedSnapshot.leadSessionId, launchPhase: persistedSnapshot.launchPhase, members: nextMembers, updatedAt, }); await this.writeLaunchStateSnapshot(teamName, nextSnapshot); } private getMutableAliveRunOrThrow(teamName: string): ProvisioningRun { const runId = this.getAliveRunId(teamName); if (!runId) { throw new Error(`Team "${teamName}" is not currently running`); } const run = this.runs.get(runId); if (!run || run.processKilled || run.cancelRequested) { throw new Error(`Team "${teamName}" is not currently running`); } return run; } async reattachOpenCodeOwnedMemberLane( teamName: string, memberName: string, options?: { reason?: 'member_added' | 'member_updated' | 'manual_restart' } ): Promise { return this.runMemberLifecycleOperation( teamName, memberName, this.getOpenCodeReattachLifecycleKind(options?.reason), () => this.reattachOpenCodeOwnedMemberLaneUnlocked(teamName, memberName, options) ); } private async reattachOpenCodeOwnedMemberLaneUnlocked( teamName: string, memberName: string, options?: { reason?: 'member_added' | 'member_updated' | 'manual_restart' } ): Promise { const run = this.getMutableAliveRunOrThrow(teamName); const leadProviderId = resolveTeamProviderId(run.request.providerId); if (leadProviderId === 'opencode') { throw new Error( 'OpenCode-led mixed teams are not supported in this phase. Stop the team and relaunch with a non-OpenCode lead.' ); } if (!this.getOpenCodeRuntimeAdapter()) { throw new Error('OpenCode runtime adapter is not available for controlled lane reattach.'); } const config = await this.readConfigForStrictDecision(teamName); if (!config) { throw new Error(`Team "${teamName}" configuration is no longer available`); } let metaMembers: Awaited> = []; try { metaMembers = await this.membersMetaStore.getMembers(teamName); } catch { metaMembers = []; } const configuredMember = this.resolveEffectiveConfiguredMember( config.members ?? [], metaMembers, memberName ); if (!configuredMember) { throw new Error(`Member "${memberName}" is not configured in team "${teamName}"`); } if (configuredMember.removedAt) { throw new Error(`Member "${memberName}" has been removed`); } if (isLeadMember({ name: configuredMember.name, agentType: configuredMember.agentType })) { throw new Error('Lead lane reattach is not supported'); } const desiredProviderId = normalizeOptionalTeamProviderId(configuredMember.providerId); if (desiredProviderId !== 'opencode') { throw new Error( `Controlled reattach is only supported for OpenCode-owned members. "${memberName}" remains on the primary runtime owner.` ); } const [memberSpec] = await this.resolveOpenCodeMemberWorkspacesForRuntime({ teamName, baseCwd: run.request.cwd, leadProviderId, members: [this.buildConfiguredProvisioningMember(configuredMember)], }); if (!memberSpec) { throw new Error(`Member "${memberName}" could not be resolved for OpenCode lane reattach.`); } const nextLane = this.createMixedSecondaryLaneStateForMember(run, memberSpec); const existingLaneIndex = run.mixedSecondaryLanes.findIndex( (lane) => lane.laneId === nextLane.laneId || lane.member.name.trim() === memberName ); const existingLane = existingLaneIndex >= 0 ? run.mixedSecondaryLanes[existingLaneIndex] : null; if (run.pendingMemberRestarts.has(memberName)) { throw new Error(`Restart for teammate "${memberName}" is already in progress`); } if (existingLane?.state === 'queued' || existingLane?.state === 'launching') { throw new Error(`Restart for teammate "${memberName}" is already in progress`); } const hasRuntimeEvidence = await this.hasOpenCodeMemberRuntimeEvidenceForControlledRelaunch({ teamName, memberName: memberSpec.name, laneId: nextLane.laneId, existingLane, }); if (existingLane) { await this.stopSingleMixedSecondaryRuntimeLane(run, existingLane, 'relaunch'); } const laneState = existingLane ?? nextLane; laneState.laneId = nextLane.laneId; laneState.member = memberSpec; laneState.runId = null; laneState.state = 'queued'; laneState.result = null; laneState.warnings = []; laneState.diagnostics = [ ...(options?.reason ? [`controlled_reattach:${options.reason}`] : []), ...(!hasRuntimeEvidence ? ['fresh_relaunch:no_runtime_evidence'] : []), ]; if (existingLaneIndex >= 0) { run.mixedSecondaryLanes[existingLaneIndex] = laneState; } else { run.mixedSecondaryLanes.push(laneState); } this.upsertRunAllEffectiveMember(run, memberSpec); this.invalidateRuntimeSnapshotCaches(teamName); this.resetRuntimeToolActivity(run, memberName); this.clearMemberSpawnToolTracking(run, memberName); run.pendingMemberRestarts.delete(memberName); if (options?.reason === 'manual_restart' || options?.reason === 'member_updated') { this.persistOpenCodeMemberRestartSystemMessage({ teamName, leadName: this.getRunLeadName(run), leadSessionId: run.detectedSessionId?.trim() || config.leadSessionId?.trim() || run.runId, displayName: config.description?.trim() || config.name, member: this.buildConfiguredProvisioningMember(configuredMember), reason: options.reason, }); } await this.launchSingleMixedSecondaryLane(run, laneState); } private async hasOpenCodeMemberRuntimeEvidenceForControlledRelaunch(params: { teamName: string; memberName: string; laneId: string; existingLane: MixedSecondaryRuntimeLaneState | null; }): Promise { const laneResultMember = params.existingLane?.result?.members[params.memberName] ?? Object.values(params.existingLane?.result?.members ?? {}).find( (member) => member.memberName?.trim() === params.memberName ); if (hasOpenCodeRuntimeHandle(laneResultMember)) { return true; } const persistedSnapshot = await this.launchStateStore.read(params.teamName).catch(() => null); const persistedMember = persistedSnapshot?.members[params.memberName] ?? Object.values(persistedSnapshot?.members ?? {}).find( (member) => member.laneId === params.laneId ); if ( hasOpenCodeRuntimeHandle(persistedMember) || hasOpenCodeRuntimeLivenessMarker(persistedMember) ) { return true; } const liveRuntimeByMember = await this.getLiveTeamAgentRuntimeMetadata(params.teamName).catch( () => new Map() ); const liveRuntimeMember = liveRuntimeByMember.get(params.memberName) ?? [...liveRuntimeByMember.entries()].find(([candidateName]) => matchesObservedMemberNameForExpected(candidateName, params.memberName) )?.[1]; return hasOpenCodeRuntimeEntryHandle(liveRuntimeMember); } async detachOpenCodeOwnedMemberLane(teamName: string, memberName: string): Promise { return this.runMemberLifecycleOperation(teamName, memberName, 'opencode_member_removed', () => this.detachOpenCodeOwnedMemberLaneUnlocked(teamName, memberName) ); } private async detachOpenCodeOwnedMemberLaneUnlocked( teamName: string, memberName: string ): Promise { const run = this.getMutableAliveRunOrThrow(teamName); const laneIndex = run.mixedSecondaryLanes.findIndex((lane) => matchesTeamMemberIdentity(lane.member.name, memberName) ); if (laneIndex < 0) { this.removeRunAllEffectiveMember(run, memberName); this.invalidateRuntimeSnapshotCaches(teamName); await this.persistLaunchStateSnapshot(run, this.getMixedSecondaryLaunchPhase(run)); return; } const [lane] = run.mixedSecondaryLanes.splice(laneIndex, 1); await this.stopSingleMixedSecondaryRuntimeLane(run, lane, 'cleanup'); this.removeRunAllEffectiveMember(run, memberName); this.invalidateRuntimeSnapshotCaches(teamName); this.resetRuntimeToolActivity(run, memberName); this.clearMemberSpawnToolTracking(run, memberName); run.pendingMemberRestarts.delete(memberName); await this.persistLaunchStateSnapshot(run, this.getMixedSecondaryLaunchPhase(run)); } private getMemberLaunchGraceKey(run: ProvisioningRun, memberName: string): string { return `member-launch-grace:${run.runId}:${memberName}`; } private syncMemberLaunchGraceCheck( run: ProvisioningRun, memberName: string, entry: MemberSpawnStatusEntry ): void { const key = this.getMemberLaunchGraceKey(run, memberName); const existing = this.pendingTimeouts.get(key); if (entry.launchState === 'failed_to_start' || entry.launchState === 'confirmed_alive') { if (existing) { clearTimeout(existing); this.pendingTimeouts.delete(key); } return; } if (!entry.firstSpawnAcceptedAt) { if (existing) { clearTimeout(existing); this.pendingTimeouts.delete(key); } return; } const remainingMs = Date.parse(entry.firstSpawnAcceptedAt) + MEMBER_LAUNCH_GRACE_MS - Date.now(); if (remainingMs <= 0) { if (existing) { clearTimeout(existing); this.pendingTimeouts.delete(key); } void this.reevaluateMemberLaunchStatus(run, memberName); return; } if (existing) { return; } const timer = setTimeout(() => { this.pendingTimeouts.delete(key); void this.reevaluateMemberLaunchStatus(run, memberName); }, remainingMs); timer.unref?.(); this.pendingTimeouts.set(key, timer); } private async reevaluateMemberLaunchStatus( run: ProvisioningRun, memberName: string ): Promise { const current = run.memberSpawnStatuses.get(memberName); if (!current) return; if ( current.launchState === 'failed_to_start' || current.launchState === 'confirmed_alive' || !current.firstSpawnAcceptedAt ) { return; } await this.refreshMemberSpawnStatusesFromLeadInbox(run); await this.maybeAuditMemberSpawnStatuses(run, { force: true }); const refreshed = run.memberSpawnStatuses.get(memberName); if (!refreshed) return; if ( refreshed.launchState === 'failed_to_start' || refreshed.launchState === 'confirmed_alive' ) { return; } const refreshedFirstSpawnAcceptedAt = refreshed.firstSpawnAcceptedAt; if (!refreshedFirstSpawnAcceptedAt) { return; } const restartPending = run.pendingMemberRestarts.has(memberName); const runtimeByMember = await this.getLiveTeamAgentRuntimeMetadata(run.teamName); const metadata = runtimeByMember.get(memberName) ?? [...runtimeByMember.entries()].find(([candidateName]) => matchesObservedMemberNameForExpected(candidateName, memberName) )?.[1]; const acceptedAtMs = Date.parse(refreshedFirstSpawnAcceptedAt); const elapsedMs = Number.isFinite(acceptedAtMs) ? Date.now() - acceptedAtMs : Infinity; const runtimeDiagnostic = metadata?.runtimeDiagnostic; if (metadata?.livenessKind === 'runtime_process') { if (this.isOpenCodeSecondaryLaneMemberInRun(run, memberName)) { const bootstrapStalled = elapsedMs >= MEMBER_BOOTSTRAP_STALL_MS; const stalledDiagnostic = bootstrapStalled ? await this.buildOpenCodeSecondaryBootstrapStallDiagnostic(run, memberName, refreshed) : null; const runtimeProcessStallDiagnostic = stalledDiagnostic === 'OpenCode bootstrap did not complete runtime_bootstrap_checkin after 5 min.' ? 'Runtime process is alive, but no bootstrap check-in after 5 min.' : stalledDiagnostic; this.setOpenCodeRuntimePendingBootstrapStatus(run, memberName, refreshed, { bootstrapStalled, runtimeDiagnostic: bootstrapStalled ? (runtimeProcessStallDiagnostic ?? 'Runtime process is alive, but no bootstrap check-in after 5 min.') : (runtimeDiagnostic ?? 'OpenCode runtime process is alive, waiting for bootstrap check-in.'), runtimeDiagnosticSeverity: bootstrapStalled ? 'warning' : (metadata.runtimeDiagnosticSeverity ?? 'info'), }); if (bootstrapStalled) { await this.maybeSendOpenCodeSecondaryBootstrapCheckinRetryPrompt({ run, memberName, current: refreshed, runtimeDiagnostic: runtimeProcessStallDiagnostic ?? 'Runtime process is alive, but no bootstrap check-in after 5 min.', runtimeSessionId: metadata.runtimeSessionId, }); } if (elapsedMs < MEMBER_BOOTSTRAP_STALL_MS) { this.scheduleOpenCodeBootstrapStallReevaluation( run, memberName, refreshedFirstSpawnAcceptedAt ); } return; } this.setMemberSpawnStatus(run, memberName, 'online', undefined, 'process'); return; } if (metadata?.livenessKind === 'permission_blocked') { const next = { ...refreshed, livenessKind: metadata.livenessKind, runtimeDiagnostic: runtimeDiagnostic ?? 'waiting for permission approval', runtimeDiagnosticSeverity: metadata.runtimeDiagnosticSeverity ?? 'warning', livenessLastCheckedAt: nowIso(), launchState: 'runtime_pending_permission' as const, }; run.memberSpawnStatuses.set(memberName, next); this.emitMemberSpawnChange(run, memberName); return; } if ( metadata?.livenessKind === 'runtime_process_candidate' && elapsedMs < MEMBER_BOOTSTRAP_STALL_MS ) { const next = { ...refreshed, livenessKind: metadata.livenessKind, runtimeDiagnostic: runtimeDiagnostic ?? 'Runtime process candidate detected, but bootstrap is unconfirmed.', runtimeDiagnosticSeverity: metadata.runtimeDiagnosticSeverity ?? 'warning', livenessLastCheckedAt: nowIso(), }; run.memberSpawnStatuses.set(memberName, next); this.emitMemberSpawnChange(run, memberName); const stallDelayMs = Math.max( 1_000, Date.parse(refreshedFirstSpawnAcceptedAt) + MEMBER_BOOTSTRAP_STALL_MS - Date.now() ); const stallKey = `${this.getMemberLaunchGraceKey(run, memberName)}:bootstrap-stall`; if (!this.pendingTimeouts.has(stallKey)) { const timer = setTimeout(() => { this.pendingTimeouts.delete(stallKey); void this.reevaluateMemberLaunchStatus(run, memberName); }, stallDelayMs); timer.unref?.(); this.pendingTimeouts.set(stallKey, timer); } return; } if ( this.isOpenCodeSecondaryLaneMemberInRun(run, memberName) && refreshed.launchState === 'runtime_pending_bootstrap' && refreshed.bootstrapConfirmed !== true && refreshed.hardFailure !== true && elapsedMs >= MEMBER_BOOTSTRAP_STALL_MS ) { const enriched = { ...refreshed, ...(metadata?.livenessKind ? { livenessKind: metadata.livenessKind } : {}), ...(runtimeDiagnostic ? { runtimeDiagnostic } : {}), ...(metadata?.runtimeDiagnosticSeverity ? { runtimeDiagnosticSeverity: metadata.runtimeDiagnosticSeverity } : {}), }; const diagnostic = await this.buildOpenCodeSecondaryBootstrapStallDiagnostic( run, memberName, enriched ); this.setOpenCodeSecondaryBootstrapStalledStatus(run, memberName, enriched, diagnostic); await this.maybeSendOpenCodeSecondaryBootstrapCheckinRetryPrompt({ run, memberName, current: enriched, runtimeDiagnostic: diagnostic, runtimeSessionId: metadata?.runtimeSessionId, }); return; } const strictReason = restartPending ? buildRestartGraceTimeoutReason(memberName) : (runtimeDiagnostic ?? (metadata?.livenessKind === 'shell_only' ? 'Tmux pane is alive, but no teammate runtime process was found.' : 'Teammate did not join within the launch grace window.')); if (restartPending) { run.pendingMemberRestarts.delete(memberName); } const livenessObservedAt = nowIso(); const nextRuntimeLostStatus: MemberSpawnStatusEntry = { ...refreshed, runtimeAlive: false, livenessSource: undefined, bootstrapConfirmed: false, ...(metadata?.livenessKind ? { livenessKind: metadata.livenessKind } : {}), ...(runtimeDiagnostic ? { runtimeDiagnostic } : {}), ...(metadata?.runtimeDiagnosticSeverity ? { runtimeDiagnosticSeverity: metadata.runtimeDiagnosticSeverity } : {}), livenessLastCheckedAt: livenessObservedAt, }; this.syncMemberTaskActivityForRuntimeTransition( run, memberName, refreshed, nextRuntimeLostStatus, livenessObservedAt ); run.memberSpawnStatuses.set(memberName, nextRuntimeLostStatus); this.setMemberSpawnStatus(run, memberName, 'error', strictReason); } private setOpenCodeRuntimePendingBootstrapStatus( run: ProvisioningRun, memberName: string, current: MemberSpawnStatusEntry, options: { bootstrapStalled: boolean; runtimeDiagnostic: string; runtimeDiagnosticSeverity: TeamAgentRuntimeDiagnosticSeverity; } ): void { const observedAt = nowIso(); const wasBootstrapStalled = current.bootstrapStalled === true; const next: MemberSpawnStatusEntry = { ...current, status: 'waiting', launchState: 'runtime_pending_bootstrap', agentToolAccepted: true, runtimeAlive: true, bootstrapConfirmed: false, hardFailure: false, error: undefined, hardFailureReason: undefined, livenessSource: undefined, livenessKind: 'runtime_process', runtimeDiagnostic: options.runtimeDiagnostic, runtimeDiagnosticSeverity: options.runtimeDiagnosticSeverity, bootstrapStalled: options.bootstrapStalled ? true : undefined, livenessLastCheckedAt: observedAt, firstSpawnAcceptedAt: current.firstSpawnAcceptedAt ?? observedAt, updatedAt: observedAt, }; this.syncMemberTaskActivityForRuntimeTransition(run, memberName, current, next, observedAt); run.memberSpawnStatuses.set(memberName, next); const launchDiagnostics = boundLaunchDiagnostics(buildLaunchDiagnosticsFromRun(run)); if (launchDiagnostics) { run.progress = { ...run.progress, updatedAt: observedAt, launchDiagnostics, }; run.onProgress(run.progress); } if (options.bootstrapStalled && !wasBootstrapStalled) { this.appendMemberBootstrapDiagnostic(run, memberName, 'opencode_bootstrap_stalled'); } else if ( !options.bootstrapStalled && (current.status !== 'waiting' || current.livenessKind !== 'runtime_process') ) { this.appendMemberBootstrapDiagnostic( run, memberName, 'runtime process is alive, teammate check-in not yet received' ); } if (!this.isCurrentTrackedRun(run)) return; this.emitMemberSpawnChange(run, memberName); if (run.isLaunch) { void this.persistLaunchStateSnapshot(run, run.provisioningComplete ? 'finished' : 'active'); } } private async buildOpenCodeSecondaryBootstrapStallDiagnostic( run: ProvisioningRun, memberName: string, current: MemberSpawnStatusEntry ): Promise { const lane = (run.mixedSecondaryLanes ?? []).find( (candidate) => candidate.providerId === 'opencode' && matchesTeamMemberIdentity(candidate.member.name, memberName) ); if ( !isExplicitLegacyOpenCodeBootstrap( resolveOpenCodeSecondaryLaneMemberEvidence(lane, memberName) ) ) { return OPENCODE_APP_MANAGED_BOOTSTRAP_STALLED_DIAGNOSTIC; } const selectedDiagnostic = selectOpenCodeSecondaryBootstrapStallDiagnostic([ current.runtimeDiagnostic, ...(lane?.diagnostics ?? []), ...(lane?.result?.diagnostics ?? []), ...(lane?.result?.members[memberName]?.diagnostics ?? []), ...Object.values(lane?.result?.members ?? {}) .filter((member) => matchesTeamMemberIdentity(member.memberName ?? '', memberName)) .flatMap((member) => member.diagnostics ?? []), ]); if (selectedDiagnostic) { return selectedDiagnostic; } const acceptedAtMs = current.firstSpawnAcceptedAt != null ? Date.parse(current.firstSpawnAcceptedAt) : NaN; const transcriptOutcome = await this.findBootstrapTranscriptOutcome( run.teamName, memberName, Number.isFinite(acceptedAtMs) ? acceptedAtMs : null ); if (transcriptOutcome?.kind === 'success' && transcriptOutcome.source === 'member_briefing') { return 'OpenCode member_briefing completed, but runtime_bootstrap_checkin did not complete after 5 min.'; } return 'OpenCode bootstrap did not complete runtime_bootstrap_checkin after 5 min.'; } private setOpenCodeSecondaryBootstrapStalledStatus( run: ProvisioningRun, memberName: string, current: MemberSpawnStatusEntry, runtimeDiagnostic: string ): void { const observedAt = nowIso(); const wasBootstrapStalled = current.bootstrapStalled === true; const runtimeProcessAlive = current.runtimeAlive === true && current.livenessKind === 'runtime_process'; const next: MemberSpawnStatusEntry = { ...current, status: 'waiting', launchState: 'runtime_pending_bootstrap', agentToolAccepted: true, runtimeAlive: runtimeProcessAlive, bootstrapConfirmed: false, hardFailure: false, error: undefined, hardFailureReason: undefined, livenessSource: undefined, livenessKind: current.livenessKind ?? (runtimeProcessAlive ? 'runtime_process' : 'registered_only'), runtimeDiagnostic, runtimeDiagnosticSeverity: 'warning', bootstrapStalled: true, livenessLastCheckedAt: observedAt, firstSpawnAcceptedAt: current.firstSpawnAcceptedAt ?? observedAt, updatedAt: observedAt, }; this.syncMemberTaskActivityForRuntimeTransition(run, memberName, current, next, observedAt); run.memberSpawnStatuses.set(memberName, next); const launchDiagnostics = boundLaunchDiagnostics(buildLaunchDiagnosticsFromRun(run)); if (launchDiagnostics) { run.progress = { ...run.progress, updatedAt: observedAt, launchDiagnostics, }; run.onProgress(run.progress); } if (!wasBootstrapStalled) { this.appendMemberBootstrapDiagnostic(run, memberName, runtimeDiagnostic); } if (!this.isCurrentTrackedRun(run)) return; this.emitMemberSpawnChange(run, memberName); if (run.isLaunch) { void this.persistLaunchStateSnapshot(run, run.provisioningComplete ? 'finished' : 'active'); } } private async maybeSendOpenCodeSecondaryBootstrapCheckinRetryPrompt(input: { run: ProvisioningRun; memberName: string; current: MemberSpawnStatusEntry; runtimeDiagnostic: string; runtimeSessionId?: string; }): Promise { const { run, memberName, current, runtimeDiagnostic } = input; if ( !this.isCurrentTrackedRun(run) || run.processKilled || run.cancelRequested || current.launchState !== 'runtime_pending_bootstrap' || current.bootstrapConfirmed === true || current.hardFailure === true || current.skippedForLaunch === true || (current.pendingPermissionRequestIds?.length ?? 0) > 0 ) { return; } const lane = (run.mixedSecondaryLanes ?? []).find( (candidate) => candidate.providerId === 'opencode' && matchesTeamMemberIdentity(candidate.member.name, memberName) ); const laneRunId = lane?.runId?.trim(); const runtimeSessionId = input.runtimeSessionId?.trim() || lane?.result?.members[memberName]?.sessionId?.trim() || Object.values(lane?.result?.members ?? {}) .find((member) => matchesTeamMemberIdentity(member.memberName ?? '', memberName)) ?.sessionId?.trim() || ''; if (!lane || !laneRunId || !isMaterializedOpenCodeSessionId(runtimeSessionId)) { return; } if ( !isExplicitLegacyOpenCodeBootstrap( resolveOpenCodeSecondaryLaneMemberEvidence(lane, memberName) ) ) { return; } const diagnostics = [ runtimeDiagnostic, current.runtimeDiagnostic, ...(lane.diagnostics ?? []), ...(lane.result?.diagnostics ?? []), ...(lane.result?.members[memberName]?.diagnostics ?? []), ].filter((value): value is string => typeof value === 'string' && value.trim().length > 0); if (hasRealOpenCodeFailureDiagnostic(diagnostics.join('\n').toLowerCase())) { return; } const marker = getOpenCodeBootstrapCheckinRetryMarker(laneRunId, runtimeSessionId); if ( run.provisioningOutputParts.some((line) => line.includes(marker)) || diagnostics.some((line) => line.includes(marker)) ) { return; } const adapter = this.getOpenCodeRuntimeMessageAdapter(); if (!adapter) { return; } lane.diagnostics = [...new Set([...(lane.diagnostics ?? []), marker])]; this.appendMemberBootstrapDiagnostic(run, memberName, marker); try { const result = await adapter.sendMessageToMember({ runId: laneRunId, teamName: run.teamName, laneId: lane.laneId, memberName, cwd: lane.member.cwd?.trim() || run.request.cwd, text: '', messageId: `bootstrap-checkin-retry-${run.runId}-${memberName}-${runtimeSessionId}`, bootstrapCheckinRetry: { runtimeSessionId, reason: runtimeDiagnostic, }, }); if (!result.ok) { this.appendMemberBootstrapDiagnostic( run, memberName, `opencode_bootstrap_checkin_retry_prompt_failed: ${ result.diagnostics.join('; ') || 'OpenCode bridge did not accept retry prompt' }` ); } } catch (error) { this.appendMemberBootstrapDiagnostic( run, memberName, `opencode_bootstrap_checkin_retry_prompt_failed: ${getErrorMessage(error)}` ); } } private scheduleOpenCodeBootstrapStallReevaluation( run: ProvisioningRun, memberName: string, firstSpawnAcceptedAt: string ): void { const acceptedAtMs = Date.parse(firstSpawnAcceptedAt); if (!Number.isFinite(acceptedAtMs)) { return; } const stallDelayMs = Math.max(1_000, acceptedAtMs + MEMBER_BOOTSTRAP_STALL_MS - Date.now()); const stallKey = `${this.getMemberLaunchGraceKey(run, memberName)}:bootstrap-stall`; if (this.pendingTimeouts.has(stallKey)) { return; } const timer = setTimeout(() => { this.pendingTimeouts.delete(stallKey); void this.reevaluateMemberLaunchStatus(run, memberName); }, stallDelayMs); timer.unref?.(); this.pendingTimeouts.set(stallKey, timer); } private isOpenCodeBootstrapStallWindowElapsed(firstSpawnAcceptedAt: string | undefined): boolean { if (!firstSpawnAcceptedAt) { return false; } const acceptedAtMs = Date.parse(firstSpawnAcceptedAt); return Number.isFinite(acceptedAtMs) && Date.now() - acceptedAtMs >= MEMBER_BOOTSTRAP_STALL_MS; } private shouldSkipMemberSpawnAudit(run: ProvisioningRun): boolean { if (!run.expectedMembers || run.expectedMembers.length === 0) { return true; } return run.expectedMembers.every((memberName) => { const entry = run.memberSpawnStatuses.get(memberName); return ( entry?.launchState === 'failed_to_start' || entry?.launchState === 'confirmed_alive' || entry?.launchState === 'skipped_for_launch' ); }); } private async maybeAuditMemberSpawnStatuses( run: ProvisioningRun, options?: { force?: boolean } ): Promise { if (!run.expectedMembers || run.expectedMembers.length === 0) { return; } await this.reconcileBootstrapTranscriptFailures(run); await this.reconcileBootstrapTranscriptSuccesses(run); if (this.shouldSkipMemberSpawnAudit(run)) { return; } const now = Date.now(); if ( !options?.force && run.lastMemberSpawnAuditAt > 0 && now - run.lastMemberSpawnAuditAt < MEMBER_SPAWN_AUDIT_MIN_INTERVAL_MS ) { return; } run.lastMemberSpawnAuditAt = now; await this.auditMemberSpawnStatuses(run); await this.reconcileBootstrapTranscriptSuccesses(run); } private async reconcileBootstrapTranscriptFailures(run: ProvisioningRun): Promise { for (const memberName of run.expectedMembers ?? []) { const current = run.memberSpawnStatuses.get(memberName); if ( !current || current.launchState === 'failed_to_start' || current.launchState === 'confirmed_alive' || current.hardFailure === true || current.agentToolAccepted !== true ) { continue; } const acceptedAtMs = current.firstSpawnAcceptedAt != null ? Date.parse(current.firstSpawnAcceptedAt) : NaN; const transcriptFailureReason = await this.findBootstrapTranscriptFailureReason( run.teamName, memberName, Number.isFinite(acceptedAtMs) ? acceptedAtMs : null ); if (!transcriptFailureReason) { continue; } this.setMemberSpawnStatus(run, memberName, 'error', transcriptFailureReason); } } private async reconcileBootstrapTranscriptSuccesses(run: ProvisioningRun): Promise { for (const memberName of run.expectedMembers ?? []) { const current = run.memberSpawnStatuses.get(memberName); if (this.isOpenCodeSecondaryLaneMemberInRun(run, memberName)) { continue; } const failureReason = current?.hardFailureReason ?? current?.error; const canClearFailedBootstrap = current?.launchState === 'failed_to_start' && current.agentToolAccepted === true && isBootstrapProofClearableLaunchFailureReason(failureReason); if ( !current || (current.launchState === 'failed_to_start' && !canClearFailedBootstrap) || current.launchState === 'confirmed_alive' || current.bootstrapConfirmed === true || (current.agentToolAccepted !== true && !canClearFailedBootstrap) ) { continue; } const acceptedAtMs = current.firstSpawnAcceptedAt != null ? Date.parse(current.firstSpawnAcceptedAt) : NaN; const runtimeProofObservedAt = await this.findBootstrapRuntimeProofObservedAt( run.teamName, memberName, current ); if (runtimeProofObservedAt) { this.confirmMemberSpawnStatusFromTranscript( run, memberName, runtimeProofObservedAt, 'runtime-proof' ); continue; } const transcriptOutcome = await this.findBootstrapTranscriptOutcome( run.teamName, memberName, Number.isFinite(acceptedAtMs) ? acceptedAtMs : null ); if (transcriptOutcome?.kind !== 'success') { continue; } this.confirmMemberSpawnStatusFromTranscript(run, memberName, transcriptOutcome.observedAt); } } private isOpenCodeSecondaryLaneMemberInRun(run: ProvisioningRun, memberName: string): boolean { const lanes = Array.isArray(run.mixedSecondaryLanes) ? run.mixedSecondaryLanes : []; return lanes.some((lane) => lane.providerId === 'opencode' && lane.member.name === memberName); } private static readonly CONTEXT_EMIT_THROTTLE_MS = 2000; private static readonly LEAD_TEXT_EMIT_THROTTLE_MS = 2000; private emitLeadContextUsage(run: ProvisioningRun): void { if (!run.leadContextUsage || !run.provisioningComplete) return; if (!this.isCurrentTrackedRun(run)) return; const now = Date.now(); if ( now - run.leadContextUsage.lastEmittedAt < TeamProvisioningService.CONTEXT_EMIT_THROTTLE_MS ) { return; } run.leadContextUsage.lastEmittedAt = now; const payload = this.buildLeadContextUsagePayload(run); this.teamChangeEmitter?.({ type: 'lead-context', teamName: run.teamName, runId: run.runId, detail: JSON.stringify(payload), }); } async warmup(): Promise { try { const cwd = process.cwd(); if (this.getFreshCachedProbeResult(cwd, 'anthropic')) return; const result = await this.getCachedOrProbeResult(cwd, 'anthropic'); if (!result) return; logger.info('CLI warmup completed'); } catch (error) { logger.warn(`CLI warmup failed: ${error instanceof Error ? error.message : String(error)}`); } } async prepareForProvisioning( cwd?: string, opts?: { forceFresh?: boolean; providerId?: TeamProviderId; providerIds?: TeamProviderId[]; modelIds?: string[]; modelChecks?: TeamProvisioningModelCheckRequest[]; limitContext?: boolean; modelVerificationMode?: TeamProvisioningModelVerificationMode; } ): Promise { const inFlightKey = this.createPrepareForProvisioningInFlightKey(cwd, opts); const inFlight = this.prepareForProvisioningInFlight.get(inFlightKey); if (inFlight) { return this.clonePrepareForProvisioningResult(await inFlight); } const request = this.prepareForProvisioningOnce(cwd, opts).finally(() => { if (this.prepareForProvisioningInFlight.get(inFlightKey) === request) { this.prepareForProvisioningInFlight.delete(inFlightKey); } }); this.prepareForProvisioningInFlight.set(inFlightKey, request); return this.clonePrepareForProvisioningResult(await request); } private createPrepareForProvisioningInFlightKey( cwd?: string, opts?: { forceFresh?: boolean; providerId?: TeamProviderId; providerIds?: TeamProviderId[]; modelIds?: string[]; modelChecks?: TeamProvisioningModelCheckRequest[]; limitContext?: boolean; modelVerificationMode?: TeamProvisioningModelVerificationMode; } ): string { const providerIds = Array.from( new Set( [opts?.providerId, ...(opts?.providerIds ?? [])] .map((providerId) => resolveTeamProviderId(providerId)) .filter((providerId): providerId is TeamProviderId => Boolean(providerId)) ) ); const modelIds = Array.from( new Set((opts?.modelIds ?? []).map((modelId) => modelId.trim()).filter(Boolean)) ); const modelChecks = normalizeProvisioningModelCheckRequests(opts?.modelChecks) .map((check) => ({ providerId: check.providerId, model: check.model, effort: check.effort ?? null, })) .sort( (left, right) => left.providerId.localeCompare(right.providerId) || left.model.localeCompare(right.model) || (left.effort ?? '').localeCompare(right.effort ?? '') ); return JSON.stringify({ cwd: cwd?.trim() || process.cwd(), forceFresh: opts?.forceFresh === true, providerIds, modelIds, modelChecks, limitContext: opts?.limitContext === true, modelVerificationMode: opts?.modelVerificationMode ?? null, }); } private clonePrepareForProvisioningResult( result: TeamProvisioningPrepareResult ): TeamProvisioningPrepareResult { return { ...result, details: result.details ? [...result.details] : undefined, warnings: result.warnings ? [...result.warnings] : undefined, issues: result.issues?.map((issue) => ({ ...issue })), supportDiagnostics: result.supportDiagnostics?.map((diagnostic) => ({ ...diagnostic })), }; } private async prepareForProvisioningOnce( cwd?: string, opts?: { forceFresh?: boolean; providerId?: TeamProviderId; providerIds?: TeamProviderId[]; modelIds?: string[]; modelChecks?: TeamProvisioningModelCheckRequest[]; limitContext?: boolean; modelVerificationMode?: TeamProvisioningModelVerificationMode; } ): Promise { const targetCwdForValidation = cwd?.trim() || process.cwd(); await this.validatePrepareCwd(targetCwdForValidation); const providerIds = Array.from( new Set( [opts?.providerId, ...(opts?.providerIds ?? [])] .map((providerId) => resolveTeamProviderId(providerId)) .filter((providerId): providerId is TeamProviderId => Boolean(providerId)) ) ); if (providerIds.length === 0) { providerIds.push('anthropic'); } // Allow callers (e.g. scheduler warm-up) to bypass the 36h probe cache if (opts?.forceFresh) { for (const providerId of providerIds) { this.clearProbeCache(targetCwdForValidation, providerId); } } const targetCwd = cwd?.trim() || process.cwd(); if (!path.isAbsolute(targetCwd)) { throw new Error('cwd must be an absolute path'); } const warnings: string[] = []; const details: string[] = []; const blockingMessages: string[] = []; const issues: TeamProvisioningPrepareIssue[] = []; const supportDiagnostics: TeamProvisioningSupportDiagnostic[] = []; const selectedModelIds = Array.from( new Set((opts?.modelIds ?? []).map((modelId) => modelId.trim()).filter(Boolean)) ); const selectedModelChecks = normalizeProvisioningModelCheckRequests(opts?.modelChecks); const useStructuredModelChecks = selectedModelChecks.length > 0; for (const providerId of providerIds) { const providerModelChecks = selectedModelChecks .filter((check) => check.providerId === providerId) .map((check) => ({ modelId: check.model, ...(check.effort ? { effort: check.effort } : {}), })); const providerSelectedModelIds = useStructuredModelChecks ? Array.from(new Set(providerModelChecks.map((check) => check.modelId))) : selectedModelIds; if (providerId === 'opencode') { const adapter = this.getOpenCodeRuntimeAdapter(); if (!adapter) { blockingMessages.push( 'OpenCode team launch is not enabled yet. Production launch requires the gated OpenCode runtime adapter.' ); continue; } if (providerSelectedModelIds.length === 0) { const prepare = await adapter.prepare({ runId: `prepare-${randomUUID()}`, teamName: '__prepare_opencode__', cwd: targetCwd, providerId: 'opencode', model: undefined, runtimeOnly: true, skipPermissions: true, expectedMembers: [], previousLaunchState: null, }); const prepareReason = prepare.ok ? undefined : prepare.reason; details.push( ...prepare.diagnostics.map((diagnostic) => normalizeOpenCodePrepareDiagnostic(diagnostic, prepareReason) ) ); warnings.push( ...prepare.warnings.map((warning) => normalizeOpenCodePrepareDiagnostic(warning, prepareReason) ) ); pushUniqueSupportDiagnostics(supportDiagnostics, prepare.supportDiagnostics); if (!prepare.ok) { const providerDiagnostic = selectOpenCodePrepareProviderDiagnostic(prepare); blockingMessages.push( providerDiagnostic ? normalizeOpenCodePrepareDiagnostic(providerDiagnostic, prepare.reason) : normalizeOpenCodePrepareDiagnostic(`OpenCode: ${prepare.reason}`, prepare.reason) ); } continue; } const openCodeModelPrepare = await this.prepareSelectedOpenCodeModels({ adapter, cwd: targetCwd, modelIds: providerSelectedModelIds, verificationMode: opts?.modelVerificationMode ?? 'deep', }); details.push(...openCodeModelPrepare.details); warnings.push(...openCodeModelPrepare.warnings); blockingMessages.push(...openCodeModelPrepare.blockingMessages); issues.push(...openCodeModelPrepare.issues); pushUniqueSupportDiagnostics(supportDiagnostics, openCodeModelPrepare.supportDiagnostics); continue; } const cached = this.getFreshCachedProbeResult(targetCwdForValidation, providerId); const probeResult = cached ?? (await this.getCachedOrProbeResult(targetCwd, providerId)); if (!probeResult?.claudePath) { throw buildMissingCliError(); } const providerLabel = getTeamProviderLabel(providerId); const { authSource } = probeResult; if (authSource === 'anthropic_api_key' || authSource === 'anthropic_api_key_helper') { logger.info(`Auth: using explicit ANTHROPIC_API_KEY for ${providerLabel}`); } else if (authSource === 'anthropic_auth_token') { logger.info( `Auth: using ANTHROPIC_AUTH_TOKEN mapped to ANTHROPIC_API_KEY for ${providerLabel}` ); } const appendSelectedModelVerification = async (): Promise => { if (providerSelectedModelIds.length === 0) { return; } const modelVerification = await this.verifySelectedProviderModels({ claudePath: probeResult.claudePath, cwd: targetCwd, providerId, modelIds: providerSelectedModelIds, modelChecks: providerModelChecks, limitContext: opts?.limitContext === true, }); details.push(...modelVerification.details); warnings.push(...modelVerification.warnings); blockingMessages.push(...modelVerification.blockingMessages); issues.push(...(modelVerification.issues ?? [])); }; const appendOneShotDiagnostic = async (): Promise => { let envResolution: ProvisioningEnvResolution | null = null; const ensureEnvResolution = async (): Promise => { if (!envResolution) { envResolution = await this.buildProvisioningEnv(providerId); } return envResolution; }; let shouldRequireRuntimePingForAnthropicDirectCredential = isAnthropicDirectCredentialAuthSource(authSource); if ( resolveTeamProviderId(providerId) === 'anthropic' && !shouldRequireRuntimePingForAnthropicDirectCredential ) { const resolvedEnv = await ensureEnvResolution(); shouldRequireRuntimePingForAnthropicDirectCredential = isAnthropicDirectCredentialAuthSource(resolvedEnv.authSource); if (resolvedEnv.authSource === 'configured_api_key_missing' && resolvedEnv.warning) { blockingMessages.push( providerIds.length > 1 ? `${providerLabel}: ${resolvedEnv.warning}` : resolvedEnv.warning ); return; } } if ( opts?.modelVerificationMode !== 'deep' && !shouldRequireRuntimePingForAnthropicDirectCredential ) { return; } const resolvedEnv = await ensureEnvResolution(); if (resolvedEnv.warning) { const prefixedWarning = providerIds.length > 1 ? `${providerLabel}: ${resolvedEnv.warning}` : resolvedEnv.warning; if (resolvedEnv.authSource === 'configured_api_key_missing') { blockingMessages.push(prefixedWarning); return; } warnings.push(prefixedWarning); return; } const diagnostic = await this.runProviderOneShotDiagnostic( probeResult.claudePath, targetCwd, resolvedEnv.env, providerId, resolvedEnv.providerArgs ); if (diagnostic.warning) { const prefixedWarning = providerIds.length > 1 ? `${providerLabel}: ${diagnostic.warning}` : diagnostic.warning; if ( shouldRequireRuntimePingForAnthropicDirectCredential && this.isAuthFailureWarning(diagnostic.warning, 'probe') ) { blockingMessages.push(prefixedWarning); return; } warnings.push(prefixedWarning); } }; if (!probeResult.warning) { const blockingCountBeforeModelChecks = blockingMessages.length; await appendSelectedModelVerification(); if (blockingMessages.length === blockingCountBeforeModelChecks) { await appendOneShotDiagnostic(); } continue; } { const prefixedWarning = providerIds.length > 1 ? `${providerLabel}: ${probeResult.warning}` : probeResult.warning; const isAuthFailure = this.isAuthFailureWarning(probeResult.warning, 'probe'); const isBlockingPreflightWarning = authSource === 'configured_api_key_missing' || (isAnthropicDirectCredentialAuthSource(authSource) && isAuthFailure) || ((authSource === 'none' || authSource === 'codex_runtime' || authSource === 'gemini_runtime') && isAuthFailure) || isBinaryProbeWarning(probeResult.warning); if (authSource === 'configured_api_key_missing') { blockingMessages.push(prefixedWarning); } else if ( (authSource === 'none' || authSource === 'codex_runtime' || authSource === 'gemini_runtime') && isAuthFailure ) { blockingMessages.push(prefixedWarning); } else if (isAnthropicDirectCredentialAuthSource(authSource) && isAuthFailure) { blockingMessages.push(prefixedWarning); } else if (isBinaryProbeWarning(probeResult.warning)) { blockingMessages.push(prefixedWarning); } else { // Preflight warnings (including timeouts) should not block provisioning. warnings.push(prefixedWarning); const blockingCountBeforeModelChecks = blockingMessages.length; if (!isBlockingPreflightWarning && providerSelectedModelIds.length > 0) { await appendSelectedModelVerification(); } if ( !isBlockingPreflightWarning && blockingMessages.length === blockingCountBeforeModelChecks ) { await appendOneShotDiagnostic(); } } } } if (blockingMessages.length > 0) { const failureWarnings = Array.from(new Set([...warnings, ...blockingMessages])); return { ready: false, details: details.length > 0 ? details : undefined, message: blockingMessages.length === 1 ? blockingMessages[0] : 'Some provider runtimes are not ready', warnings: failureWarnings.length > 0 ? failureWarnings : undefined, issues: issues.length > 0 ? issues : undefined, supportDiagnostics: supportDiagnostics.length > 0 ? supportDiagnostics.map((diagnostic) => ({ ...diagnostic })) : undefined, }; } return { ready: true, details: details.length > 0 ? details : undefined, message: providerIds.length > 1 ? warnings.length > 0 ? `Validated ${providerIds.length}/${providerIds.length} provider runtimes (see notes)` : `Validated ${providerIds.length}/${providerIds.length} provider runtimes` : warnings.length > 0 ? 'CLI is ready to launch (see notes)' : 'CLI is warmed up and ready to launch', warnings: warnings.length > 0 ? warnings : undefined, issues: issues.length > 0 ? issues : undefined, supportDiagnostics: supportDiagnostics.length > 0 ? supportDiagnostics.map((diagnostic) => ({ ...diagnostic })) : undefined, }; } private async prepareSelectedOpenCodeModels({ adapter, cwd, modelIds, verificationMode, }: { adapter: TeamLaunchRuntimeAdapter; cwd: string; modelIds: string[]; verificationMode: TeamProvisioningModelVerificationMode; }): Promise<{ details: string[]; warnings: string[]; blockingMessages: string[]; issues: TeamProvisioningPrepareIssue[]; supportDiagnostics: TeamProvisioningSupportDiagnostic[]; }> { const details: string[] = []; const warnings: string[] = []; const blockingMessages: string[] = []; const issues: TeamProvisioningPrepareIssue[] = []; const supportDiagnostics: TeamProvisioningSupportDiagnostic[] = []; const startedAt = Date.now(); if (modelIds.length === 0) { return { details, warnings, blockingMessages, issues, supportDiagnostics }; } if (verificationMode === 'compatibility') { const sharedCompatibilityPrepare = await this.prepareSelectedOpenCodeModelsCompatibilityBatch( { adapter, cwd, modelIds, } ); if (sharedCompatibilityPrepare) { return sharedCompatibilityPrepare; } } const results = new Array<{ modelId: string; prepare: TeamRuntimePrepareResult } | undefined>( modelIds.length ); let providerBusyDeferred: { modelId: string; reason: string; code: string; } | null = null; const prepareModel = async (modelId: string): Promise => { const startedAt = Date.now(); try { const prepare = await adapter.prepare({ runId: `prepare-${randomUUID()}`, teamName: '__prepare_opencode__', cwd, providerId: 'opencode', model: modelId, runtimeOnly: verificationMode === 'compatibility', skipPermissions: true, expectedMembers: [], previousLaunchState: null, }); appendPreflightDebugLog('opencode_model_prepare_result', { cwd, modelId, verificationMode, durationMs: Date.now() - startedAt, ok: prepare.ok, reason: prepare.ok ? null : prepare.reason, diagnostics: prepare.diagnostics, warnings: prepare.warnings, supportDiagnostics: prepare.supportDiagnostics?.map((diagnostic) => ({ id: diagnostic.id, kind: diagnostic.kind, title: diagnostic.title, })), }); return prepare; } catch (error) { const message = getErrorMessage(error).trim() || 'OpenCode model verification failed'; appendPreflightDebugLog('opencode_model_prepare_result', { cwd, modelId, verificationMode, durationMs: Date.now() - startedAt, ok: false, reason: 'unknown_error', diagnostics: [message], warnings: [], }); return { ok: false, providerId: 'opencode', reason: 'unknown_error', retryable: false, diagnostics: [message], warnings: [], }; } }; // Facts: // - Deep OpenCode preflight maps to a real foreground execution probe. // - The host reports "session status busy" while another probe/member turn is active. // - Once busy is observed, probing more selected models only repeats the same host state. for (let index = 0; index < modelIds.length; index += 1) { const modelId = modelIds[index]; const prepare = await prepareModel(modelId); results[index] = { modelId, prepare }; if (verificationMode === 'compatibility' || prepare.ok) { continue; } const primaryReason = normalizeOpenCodePrepareDiagnostic( selectOpenCodeModelPreparePrimaryReason(prepare), prepare.reason ); if (isOpenCodeModelPrepareBusyDeferred(prepare, primaryReason)) { providerBusyDeferred = { modelId, reason: primaryReason, code: prepare.reason, }; appendPreflightDebugLog('opencode_model_prepare_batch_busy_deferred', { cwd, modelId, verificationMode, skippedModelIds: modelIds.slice(index + 1), reason: primaryReason, }); break; } } for (const result of results) { if (!result) { if (providerBusyDeferred) { continue; } blockingMessages.push( 'OpenCode preflight could not collect model verification results for all selected models.' ); continue; } const { modelId, prepare } = result; pushUniqueSupportDiagnostics(supportDiagnostics, prepare.supportDiagnostics); const prepareReason = prepare.ok ? undefined : prepare.reason; warnings.push( ...prepare.warnings.map((warning) => normalizeOpenCodePrepareDiagnostic(warning, prepareReason) ) ); if (prepare.ok) { details.push( verificationMode === 'compatibility' ? `Selected model ${modelId} is compatible. Deep verification pending.` : `Selected model ${modelId} verified for launch.` ); continue; } const primaryReason = normalizeOpenCodePrepareDiagnostic( selectOpenCodeModelPreparePrimaryReason(prepare), prepare.reason ); if (isOpenCodeModelPrepareBusyDeferred(prepare, primaryReason)) { providerBusyDeferred ??= { modelId, reason: primaryReason, code: prepare.reason, }; continue; } if (this.isProviderScopedOpenCodePrepareFailure(prepare, primaryReason)) { pushUniqueLine(details, primaryReason); pushUniqueLine(blockingMessages, primaryReason); if ( !issues.some( (issue) => issue.providerId === 'opencode' && issue.scope === 'provider' && issue.severity === 'blocking' && issue.code === prepare.reason && issue.message === primaryReason ) ) { issues.push({ providerId: 'opencode', scope: 'provider', severity: 'blocking', code: prepare.reason, message: primaryReason, }); } continue; } const unavailableLine = `Selected model ${modelId} is unavailable. ${primaryReason}`; const verificationWarningLine = `Selected model ${modelId} could not be verified. ${primaryReason}`; const issueSeverity = prepare.retryable && verificationMode !== 'compatibility' ? 'warning' : 'blocking'; issues.push({ providerId: 'opencode', modelId, scope: 'model', severity: issueSeverity, code: prepare.reason, message: primaryReason, }); if (prepare.retryable) { warnings.push(verificationWarningLine); if (verificationMode === 'compatibility') { blockingMessages.push(verificationWarningLine); } } else { if (verificationMode === 'compatibility') { details.push(unavailableLine); } blockingMessages.push(unavailableLine); } } if (providerBusyDeferred) { const providerBusyLine = buildOpenCodeProviderVerificationDeferredLine( providerBusyDeferred.reason ); pushUniqueLine(warnings, providerBusyLine); issues.push({ providerId: 'opencode', scope: 'provider', severity: 'warning', code: providerBusyDeferred.code, message: providerBusyLine, }); } appendPreflightDebugLog('opencode_model_prepare_batch_complete', { cwd, modelIds, verificationMode, durationMs: Date.now() - startedAt, details, warnings, blockingMessages, }); return { details, warnings, blockingMessages, issues, supportDiagnostics }; } private isProviderScopedOpenCodePrepareFailure( prepare: Extract, primaryReason: string ): boolean { if (OPENCODE_PROVIDER_SCOPED_PREPARE_FAILURE_REASONS.has(prepare.reason)) { return true; } return ( prepare.reason === 'unknown_error' && [primaryReason, ...prepare.diagnostics].some(looksLikeOpenCodeProviderPrepareDiagnostic) ); } private async prepareSelectedOpenCodeModelsCompatibilityBatch({ adapter, cwd, modelIds, }: { adapter: TeamLaunchRuntimeAdapter; cwd: string; modelIds: string[]; }): Promise<{ details: string[]; warnings: string[]; blockingMessages: string[]; issues: TeamProvisioningPrepareIssue[]; supportDiagnostics: TeamProvisioningSupportDiagnostic[]; } | null> { const details: string[] = []; const warnings: string[] = []; const blockingMessages: string[] = []; const issues: TeamProvisioningPrepareIssue[] = []; const supportDiagnostics: TeamProvisioningSupportDiagnostic[] = []; const startedAt = Date.now(); appendPreflightDebugLog('opencode_compatibility_batch_start', { cwd, modelIds, }); let sharedPrepare: TeamRuntimePrepareResult; try { sharedPrepare = await adapter.prepare({ runId: `prepare-${randomUUID()}`, teamName: '__prepare_opencode__', cwd, providerId: 'opencode', model: undefined, runtimeOnly: true, skipPermissions: true, expectedMembers: [], previousLaunchState: null, }); } catch (error) { const message = getErrorMessage(error).trim() || 'OpenCode model verification failed'; sharedPrepare = { ok: false, providerId: 'opencode', reason: 'unknown_error', retryable: false, diagnostics: [message], warnings: [], }; } const sharedPrepareReason = sharedPrepare.ok ? undefined : sharedPrepare.reason; warnings.push( ...sharedPrepare.warnings.map((warning) => normalizeOpenCodePrepareDiagnostic(warning, sharedPrepareReason) ) ); appendPreflightDebugLog('opencode_compatibility_batch_shared_prepare', { cwd, modelIds, durationMs: Date.now() - startedAt, ok: sharedPrepare.ok, reason: sharedPrepare.ok ? null : sharedPrepare.reason, diagnostics: sharedPrepare.diagnostics, supportDiagnostics: sharedPrepare.supportDiagnostics?.map((diagnostic) => ({ id: diagnostic.id, kind: diagnostic.kind, title: diagnostic.title, })), }); if (!sharedPrepare.ok) { pushUniqueSupportDiagnostics(supportDiagnostics, sharedPrepare.supportDiagnostics); const providerDiagnostic = selectOpenCodePrepareProviderDiagnostic(sharedPrepare); const primaryReason = normalizeOpenCodePrepareDiagnostic( providerDiagnostic ?? sharedPrepare.diagnostics.find((entry) => entry.trim().length > 0) ?? sharedPrepare.reason, sharedPrepare.reason ); if (primaryReason.trim().length > 0) { details.push(primaryReason); blockingMessages.push(primaryReason); } else { blockingMessages.push(`OpenCode: ${sharedPrepare.reason}`); } issues.push({ providerId: 'opencode', scope: 'provider', severity: 'blocking', code: sharedPrepare.reason, message: primaryReason.trim() || `OpenCode: ${sharedPrepare.reason}`, }); return { details, warnings, blockingMessages, issues, supportDiagnostics }; } const latestReadiness = 'getLastOpenCodeTeamLaunchReadiness' in adapter && typeof adapter.getLastOpenCodeTeamLaunchReadiness === 'function' ? adapter.getLastOpenCodeTeamLaunchReadiness(cwd) : null; const availableModels: string[] = Array.from( new Set( (Array.isArray(latestReadiness?.availableModels) ? latestReadiness.availableModels : []) .filter((modelId: unknown): modelId is string => typeof modelId === 'string') .map((modelId: string) => modelId.trim()) .filter((modelId: string) => modelId.length > 0) ) ); appendPreflightDebugLog('opencode_compatibility_batch_catalog', { cwd, modelIds, availableModelCount: availableModels.length, availableModelsSample: availableModels.slice(0, 20), fellBackToPerModelPrepare: availableModels.length === 0, }); if (availableModels.length === 0) { return null; } for (const modelId of modelIds) { const resolvedModel = this.resolveOpenCodeCompatibilityModel(modelId, availableModels); if (resolvedModel.ok) { details.push(`Selected model ${modelId} is compatible. Deep verification pending.`); continue; } const unavailableLine = `Selected model ${modelId} is unavailable. ${resolvedModel.reason}`; details.push(unavailableLine); blockingMessages.push(unavailableLine); issues.push({ providerId: 'opencode', modelId, scope: 'model', severity: 'blocking', code: 'model_unavailable', message: resolvedModel.reason, }); } appendPreflightDebugLog('opencode_compatibility_batch_complete', { cwd, modelIds, durationMs: Date.now() - startedAt, blockingMessages, details, }); return { details, warnings, blockingMessages, issues, supportDiagnostics }; } private resolveOpenCodeCompatibilityModel( requestedModelId: string, availableModels: readonly string[] ): { ok: true; resolvedModelId: string } | { ok: false; reason: string } { const trimmedModelId = requestedModelId.trim(); if (!trimmedModelId) { return { ok: false, reason: 'Selected model id is empty.', }; } if (availableModels.includes(trimmedModelId)) { return { ok: true, resolvedModelId: trimmedModelId, }; } const equivalentOpenRouterMatches = this.findEquivalentOpenRouterModelIds( trimmedModelId, availableModels ); if (equivalentOpenRouterMatches.length === 1) { return { ok: true, resolvedModelId: equivalentOpenRouterMatches[0], }; } if (equivalentOpenRouterMatches.length > 1) { return { ok: false, reason: `Selected model ${trimmedModelId} matched multiple live provider models: ` + equivalentOpenRouterMatches.join(', '), }; } if (trimmedModelId.includes('/')) { const requestedProviderId = this.extractOpenCodeCatalogProviderId(trimmedModelId); const availableProviderIds = this.getOpenCodeCatalogProviderIds(availableModels); if ( requestedProviderId === 'openrouter' && !availableProviderIds.includes(requestedProviderId) ) { const availableProviderList = availableProviderIds.length > 0 ? availableProviderIds.join(', ') : 'none'; return { ok: false, reason: `OpenCode provider "openrouter" for selected model "${trimmedModelId}" ` + 'is not available in the current runtime catalog for this project/profile. ' + `Live catalog providers: ${availableProviderList}. ` + 'Connect OpenRouter in OpenCode provider management or choose one of the listed OpenCode models.', }; } return { ok: false, reason: `Selected model ${trimmedModelId} was not found in the live provider catalog.`, }; } const matchingProviderScopedModels = availableModels.filter( (candidate) => candidate.split('/').at(-1) === trimmedModelId ); if (matchingProviderScopedModels.length === 1) { return { ok: true, resolvedModelId: matchingProviderScopedModels[0], }; } if (matchingProviderScopedModels.length > 1) { return { ok: false, reason: `Selected model ${trimmedModelId} matched multiple live provider models: ` + matchingProviderScopedModels.join(', '), }; } return { ok: false, reason: `Selected model ${trimmedModelId} was not found in the live provider catalog.`, }; } private extractOpenCodeCatalogProviderId(modelId: string): string | null { const separatorIndex = modelId.indexOf('/'); if (separatorIndex <= 0) { return null; } return modelId.slice(0, separatorIndex).trim().toLowerCase() || null; } private getOpenCodeCatalogProviderIds(availableModels: readonly string[]): string[] { return Array.from( new Set( availableModels .map((modelId) => this.extractOpenCodeCatalogProviderId(modelId.trim())) .filter((providerId): providerId is string => Boolean(providerId)) ) ).sort((left, right) => left.localeCompare(right)); } private findEquivalentOpenRouterModelIds( requestedModelId: string, availableModels: readonly string[] ): string[] { const equivalentIds = new Set(); if (requestedModelId.startsWith('openrouter/')) { equivalentIds.add(requestedModelId.slice('openrouter/'.length)); } else if (requestedModelId.includes('/')) { equivalentIds.add(`openrouter/${requestedModelId}`); } if (equivalentIds.size === 0) { return []; } return Array.from( new Set(availableModels.filter((candidate) => equivalentIds.has(candidate.trim()))) ); } private resolveProviderCompatibilityModel(params: { providerId: TeamProviderId; requestedModelId: string; runtimeFacts: RuntimeProviderLaunchFacts; limitContext: boolean; }): | { kind: 'available'; resolvedModelId: string | null } | { kind: 'compatible'; reason: string } | { kind: 'unavailable'; reason: string } { const trimmedModelId = params.requestedModelId.trim(); if (!trimmedModelId) { return { kind: 'unavailable', reason: 'Selected model id is empty.', }; } if (isDefaultProviderModelSelection(trimmedModelId)) { return { kind: 'available', resolvedModelId: params.runtimeFacts.defaultModel, }; } const availableModels = params.runtimeFacts.modelIds; let resolvedModelId: string | null = availableModels.has(trimmedModelId) ? trimmedModelId : null; if (!resolvedModelId && params.providerId === 'anthropic') { resolvedModelId = resolveAnthropicLaunchModel({ selectedModel: trimmedModelId, limitContext: params.limitContext, availableLaunchModels: availableModels, defaultLaunchModel: params.runtimeFacts.defaultModel, }) ?? null; } if (!resolvedModelId && !trimmedModelId.includes('/')) { const scopedMatches = Array.from(availableModels).filter( (candidate) => candidate.split('/').at(-1) === trimmedModelId ); if (scopedMatches.length === 1) { resolvedModelId = scopedMatches[0]; } else if (scopedMatches.length > 1) { return { kind: 'unavailable', reason: `Selected model ${trimmedModelId} matched multiple live provider models: ` + scopedMatches.join(', '), }; } } if (resolvedModelId && (availableModels.size === 0 || availableModels.has(resolvedModelId))) { return { kind: 'available', resolvedModelId, }; } const dynamicCatalog = params.runtimeFacts.runtimeCapabilities?.modelCatalog?.dynamic === true; const hasAuthoritativeCatalog = params.providerId === 'codex' ? hasAuthoritativeCodexLaunchCatalog(params.runtimeFacts) : availableModels.size > 0 || params.runtimeFacts.modelCatalog != null || params.runtimeFacts.runtimeCapabilities?.modelCatalog?.dynamic === false; if (params.providerId === 'codex' && (dynamicCatalog || !hasAuthoritativeCatalog)) { return { kind: 'available', resolvedModelId: trimmedModelId, }; } if (dynamicCatalog || !hasAuthoritativeCatalog) { return { kind: 'compatible', reason: dynamicCatalog ? 'Runtime catalog allows dynamic model launch.' : 'Runtime model catalog was unavailable.', }; } return { kind: 'unavailable', reason: `Selected model ${trimmedModelId} was not found in the live provider catalog.`, }; } private async verifySelectedProviderModels({ claudePath, cwd, providerId, modelIds, modelChecks, limitContext, }: { claudePath: string; cwd: string; providerId: TeamProviderId; modelIds: string[]; modelChecks?: ProviderSelectedModelCheck[]; limitContext: boolean; }): Promise<{ details: string[]; warnings: string[]; blockingMessages: string[]; issues?: TeamProvisioningPrepareIssue[]; }> { const details: string[] = []; const warnings: string[] = []; const blockingMessages: string[] = []; const issues: TeamProvisioningPrepareIssue[] = []; const startedAt = Date.now(); const selectedModelChecks = normalizeProviderSelectedModelChecks(modelIds, modelChecks); if (selectedModelChecks.length === 0) { return { details, warnings, blockingMessages }; } const { env, providerArgs = [] } = await this.buildProvisioningEnv(providerId); const runtimeFacts = await this.readRuntimeProviderLaunchFacts({ claudePath, cwd, providerId, env, providerArgs, limitContext, }); const recordOutcome = ( requestedModelId: string, outcome: | { kind: 'available'; resolvedModelId: string | null } | { kind: 'compatible'; reason: string } | { kind: 'unavailable'; reason: string } ): void => { if (outcome.kind === 'available') { details.push(`Selected model ${requestedModelId} is available for launch.`); return; } if (outcome.kind === 'compatible') { details.push( `Selected model ${requestedModelId} is compatible. Deep verification pending.` ); return; } blockingMessages.push(`Selected model ${requestedModelId} is unavailable. ${outcome.reason}`); issues.push({ providerId, modelId: requestedModelId, scope: 'model', severity: 'blocking', code: 'model_unavailable', message: outcome.reason, }); }; const recordAnthropicEffortOutcome = ( requestedModelId: string, effort: EffortLevel ): boolean => { const selection = resolveAnthropicSelectionFromFacts({ selectedModel: requestedModelId, limitContext, facts: runtimeFacts, }); const modelLabel = selection.displayName ?? selection.resolvedLaunchModel ?? requestedModelId; const effortSupport = resolveAnthropicEffortSupport({ selection, effort, runtimeCapabilities: runtimeFacts.runtimeCapabilities, }); if (effortSupport.kind === 'supported') { return true; } const reason = formatAnthropicEffortSupportFailure({ effort, modelLabel, kind: effortSupport.kind, supportedEfforts: effortSupport.kind === 'unverified-catalog-missing' ? undefined : effortSupport.supportedEfforts, }); blockingMessages.push(`Selected model ${requestedModelId} is unavailable. ${reason}`); issues.push({ providerId, modelId: requestedModelId, scope: 'model', severity: 'blocking', code: effortSupport.kind === 'unverified-catalog-missing' ? 'effort_unverified' : 'effort_unsupported', message: reason, }); return false; }; appendPreflightDebugLog('provider_model_catalog_check_start', { providerId, cwd, modelIds: selectedModelChecks.map((check) => check.modelId), }); const checksByModelId = new Map(); for (const check of selectedModelChecks) { const label = check.modelId.trim(); if (!label) { continue; } checksByModelId.set(label, [...(checksByModelId.get(label) ?? []), check]); } for (const [label, checks] of checksByModelId.entries()) { const outcome = this.resolveProviderCompatibilityModel({ providerId, requestedModelId: label, runtimeFacts, limitContext, }); let effortSupported = true; if (outcome.kind !== 'unavailable' && providerId === 'anthropic') { for (const check of checks) { if (check.effort && !recordAnthropicEffortOutcome(label, check.effort)) { effortSupported = false; } } } if (!effortSupported) { continue; } recordOutcome(label, outcome); } appendPreflightDebugLog('provider_model_catalog_check_complete', { providerId, cwd, modelIds: selectedModelChecks.map((check) => check.modelId), durationMs: Date.now() - startedAt, modelCount: runtimeFacts.modelIds.size, details, warnings, blockingMessages, }); return { details, warnings, blockingMessages, ...(issues.length > 0 ? { issues } : {}), }; } private async resolveProviderDefaultModel( claudePath: string, cwd: string, providerId: TeamProviderId, env: NodeJS.ProcessEnv, providerArgs: string[] = [], limitContext: boolean ): Promise { let parsed: ProviderModelListCommandResponse; try { const { stdout } = await execCli( claudePath, buildProviderCliCommandArgs(providerArgs, [ 'model', 'list', '--json', '--provider', providerId, ]), { cwd, env, timeout: PROVIDER_MODEL_LIST_TIMEOUT_MS, } ); parsed = extractJsonObjectFromCli(stdout); } catch (error) { const fallbackDefaultModel = await this.resolveProviderDefaultModelFromRuntimeStatus( claudePath, cwd, providerId, env, providerArgs, limitContext ).catch(() => null); if (fallbackDefaultModel) { return fallbackDefaultModel; } const message = error instanceof Error ? error.message : String(error); throw new Error( `Failed to load runtime default model list for ${getTeamProviderLabel(providerId)} (${providerId}): ${message}` ); } const defaultModel = parsed.providers?.[providerId]?.defaultModel; const normalizedDefaultModel = typeof defaultModel === 'string' && defaultModel.trim().length > 0 ? defaultModel.trim() : null; const modelIds = normalizeProviderModelListModels(parsed.providers?.[providerId]); if (providerId === 'anthropic') { return resolveAnthropicLaunchModel({ limitContext, availableLaunchModels: modelIds, defaultLaunchModel: normalizedDefaultModel, }); } return normalizedDefaultModel; } private async resolveProviderDefaultModelFromRuntimeStatus( claudePath: string, cwd: string, providerId: TeamProviderId, env: NodeJS.ProcessEnv, providerArgs: string[] = [], limitContext: boolean ): Promise { const { stdout } = await execCli( claudePath, buildProviderCliCommandArgs(providerArgs, [ 'runtime', 'status', '--json', '--provider', providerId, ]), { cwd, env, timeout: PROVIDER_RUNTIME_STATUS_TIMEOUT_MS, } ); const parsed = extractJsonObjectFromCli(stdout); const providerStatus = parsed.providers?.[providerId] ?? null; const modelCatalog = providerStatus?.modelCatalog?.providerId === providerId ? providerStatus.modelCatalog : null; const defaultLaunchModel = modelCatalog?.defaultLaunchModel?.trim() || null; if (providerId === 'anthropic') { return resolveAnthropicLaunchModel({ limitContext, availableLaunchModels: modelCatalog?.models.map((model) => model.launchModel) ?? [], defaultLaunchModel, }); } return defaultLaunchModel; } private async materializeEffectiveTeamMemberSpecs(params: { claudePath: string; cwd: string; members: TeamCreateRequest['members']; defaults: { providerId?: TeamProviderId; model?: string; effort?: TeamCreateRequest['effort']; }; primaryProviderId?: TeamProviderId; primaryEnv?: ProvisioningEnvResolution; teamRuntimeAuth?: TeamRuntimeAuthContext; limitContext?: boolean; providerArgsResolver?: (input: { providerId: TeamProviderId; providerArgs: string[]; phase: 'default-model-resolution'; }) => string[]; }): Promise { const envByProvider = new Map>(); const defaultModelByProvider = new Map>(); const normalizedPrimaryProviderId = resolveTeamProviderId(params.primaryProviderId); const getProvisioningEnv = (providerId: TeamProviderId): Promise => { if (normalizedPrimaryProviderId === providerId && params.primaryEnv != null) { return Promise.resolve(params.primaryEnv); } const cached = envByProvider.get(providerId); if (cached) { return cached; } const created = this.buildProvisioningEnv(providerId, undefined, { teamRuntimeAuth: params.teamRuntimeAuth, }); envByProvider.set(providerId, created); return created; }; const getResolvedDefaultModel = (providerId: TeamProviderId): Promise => { const cached = defaultModelByProvider.get(providerId); if (cached) { return cached; } const providerLabel = getTeamProviderLabel(providerId); const created = (async () => { const envResolution = await getProvisioningEnv(providerId); if (envResolution.warning) { throw new Error(envResolution.warning); } const resolvedDefaultModel = await this.resolveProviderDefaultModel( params.claudePath, params.cwd, providerId, envResolution.env, params.providerArgsResolver?.({ providerId, providerArgs: envResolution.providerArgs ?? [], phase: 'default-model-resolution', }) ?? envResolution.providerArgs ?? [], params.limitContext === true ); const normalized = resolvedDefaultModel?.trim(); if (!normalized) { throw new Error( `Could not resolve the runtime default model for ${providerLabel} teammates. Select an explicit model and retry.` ); } return normalized; })(); defaultModelByProvider.set(providerId, created); return created; }; const effectiveMembers: TeamCreateRequest['members'] = []; for (const member of params.members) { const effectiveMember = buildEffectiveTeamMemberSpec(member, params.defaults); const providerId = normalizeTeamMemberProviderId(effectiveMember.providerId) ?? 'anthropic'; if (providerId === 'anthropic' || effectiveMember.model?.trim()) { effectiveMembers.push(effectiveMember); continue; } effectiveMembers.push({ ...effectiveMember, model: await getResolvedDefaultModel(providerId), }); } return effectiveMembers; } private getOpenCodeRuntimeLaunchCwd( fallbackCwd: string, members: TeamCreateRequest['members'] ): string { if (members.length > 1 && members.some((member) => member.isolation === 'worktree')) { throw new Error( 'OpenCode worktree isolation currently supports one isolated OpenCode member per runtime lane.' ); } const memberCwds = [ ...new Set( members.map((member) => member.cwd?.trim()).filter((cwd): cwd is string => Boolean(cwd)) ), ]; if (memberCwds.length === 0) { return fallbackCwd; } if (memberCwds.length === 1) { return memberCwds[0]; } throw new Error( 'OpenCode runtime lanes support exactly one project path in this release. Use mixed-team OpenCode side lanes for per-teammate worktree isolation.' ); } private async materializeOpenCodeRuntimeAdapterDefaults< TRequest extends TeamCreateRequest | TeamLaunchRequest, >(params: { request: TRequest; members: TeamCreateRequest['members']; }): Promise<{ request: TRequest; members: TeamCreateRequest['members']; }> { const effectiveMembers = buildEffectiveTeamMemberSpecs(params.members, { providerId: params.request.providerId, model: params.request.model, effort: params.request.effort, }); const explicitRootModel = getExplicitLaunchModelSelection(params.request.model); const memberModels = [ ...new Set( effectiveMembers .map((member) => member.model?.trim()) .filter((model): model is string => Boolean(model)) ), ]; if (!explicitRootModel && memberModels.length > 1) { throw new Error( 'OpenCode runtime adapter launch supports one selected model per lane. Select one team model or align OpenCode teammate models.' ); } const inheritedRootModel = explicitRootModel ? undefined : memberModels[0]; const rootModel = explicitRootModel ?? inheritedRootModel; const needsMemberModel = effectiveMembers.some((member) => { const providerId = normalizeTeamMemberProviderId(member.providerId) ?? 'opencode'; return providerId === 'opencode' && !member.model?.trim(); }); if (rootModel && !needsMemberModel) { return { request: { ...params.request, model: rootModel, } as TRequest, members: effectiveMembers, }; } if (rootModel) { return { request: { ...params.request, model: rootModel, } as TRequest, members: effectiveMembers.map((member) => { const providerId = normalizeTeamMemberProviderId(member.providerId) ?? 'opencode'; if (providerId !== 'opencode' || member.model?.trim()) { return member; } return { ...member, model: rootModel, }; }), }; } const claudePath = await ClaudeBinaryResolver.resolve(); if (!claudePath) { throw buildMissingCliError(); } const provisioningEnv = await this.buildProvisioningEnv( 'opencode', params.request.providerBackendId ); if (provisioningEnv.warning) { throw new Error(provisioningEnv.warning); } const resolvedDefaultModel = await this.resolveProviderDefaultModel( claudePath, params.request.cwd, 'opencode', provisioningEnv.env, provisioningEnv.providerArgs ?? [], params.request.limitContext === true ); const normalizedDefaultModel = resolvedDefaultModel?.trim(); if (!normalizedDefaultModel) { throw new Error( 'Could not resolve the runtime default model for OpenCode teammates. Select an explicit model and retry.' ); } return { request: { ...params.request, model: normalizedDefaultModel, } as TRequest, members: effectiveMembers.map((member) => { const providerId = normalizeTeamMemberProviderId(member.providerId) ?? 'opencode'; if (providerId !== 'opencode' || member.model?.trim()) { return member; } return { ...member, model: normalizedDefaultModel, }; }), }; } private async resolveOpenCodeMemberWorkspacesForRuntime(params: { teamName: string; baseCwd: string; leadProviderId?: TeamProviderId; members: TeamCreateRequest['members']; }): Promise { const isolatedOpenCodeMembers = params.members.filter((member) => { const providerId = normalizeTeamMemberProviderId(member.providerId); return providerId === 'opencode' && member.isolation === 'worktree'; }); if (isolatedOpenCodeMembers.length === 0) { return params.members; } if ( isPureOpenCodeProvisioningRequest({ providerId: params.leadProviderId, members: params.members, }) && params.members.length > 1 ) { throw new Error( 'OpenCode worktree isolation currently supports mixed-team OpenCode side lanes or one-member OpenCode runtime lanes. Multiple OpenCode members in one lane cannot use separate worktrees yet.' ); } const nextMembers: TeamCreateRequest['members'] = []; for (const member of params.members) { const providerId = normalizeTeamMemberProviderId(member.providerId); if (providerId !== 'opencode' || member.isolation !== 'worktree') { nextMembers.push(member); continue; } const existingCwd = member.cwd?.trim(); if (existingCwd) { if (!path.isAbsolute(existingCwd)) { throw new Error( `OpenCode worktree path for "${member.name}" must be absolute: ${existingCwd}` ); } const existingCwdStat = await fs.promises.stat(existingCwd).catch(() => null); if (existingCwdStat) { if (!existingCwdStat.isDirectory()) { throw new Error( `OpenCode worktree path for "${member.name}" is not a directory: ${existingCwd}` ); } nextMembers.push({ ...member, cwd: existingCwd }); continue; } } const resolution = await this.memberWorktreeManager.ensureMemberWorktree({ teamName: params.teamName, memberName: member.name, baseCwd: params.baseCwd, }); nextMembers.push({ ...member, cwd: resolution.worktreePath }); } return nextMembers; } private getFreshCachedProbeResult( cwd: string, providerId: TeamProviderId | undefined ): CachedProbeResult | null { const cacheKey = createProbeCacheKey(cwd, providerId); const cached = cachedProbeResults.get(cacheKey); if (!cached) return null; const ageMs = Date.now() - cached.cachedAtMs; if (ageMs >= PROBE_CACHE_TTL_MS) { cachedProbeResults.delete(cacheKey); return null; } return cached; } private clearProbeCache(cwd: string, providerId: TeamProviderId | undefined): void { cachedProbeResults.delete(createProbeCacheKey(cwd, providerId)); } private async validatePrepareCwd(cwd: string): Promise { if (!path.isAbsolute(cwd)) { throw new Error('cwd must be an absolute path'); } try { const stat = await fs.promises.stat(cwd); if (!stat.isDirectory()) { throw new Error('cwd must be a directory'); } } catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') { // Allow the runtime probe to degrade a missing cwd into a warning. // This keeps prepareForProvisioning side-effect free for future/missing paths. return; } throw error; } } private async getCachedOrProbeResult( cwd: string, providerId: TeamProviderId | undefined ): Promise { const cacheKey = createProbeCacheKey(cwd, providerId); const cached = this.getFreshCachedProbeResult(cwd, providerId); if (cached) { return { claudePath: cached.claudePath, authSource: cached.authSource, warning: cached.warning, }; } const existingProbe = probeInFlightByKey.get(cacheKey); if (existingProbe) { return await existingProbe; } const probePromise = (async () => { const claudePath = await ClaudeBinaryResolver.resolve(); if (!claudePath) return null; const { env, authSource, providerArgs = [], warning, } = await this.buildProvisioningEnv(providerId); if (warning) { return { claudePath, authSource, warning, }; } const probe = await this.probeClaudeRuntime(claudePath, cwd, env, providerId, providerArgs); const result = { claudePath, authSource, ...(probe.warning ? { warning: probe.warning } : {}), }; const shouldCache = !probe.warning || (!this.isAuthFailureWarning(probe.warning, 'probe') && !isTransientProbeWarning(probe.warning) && !isBinaryProbeWarning(probe.warning)); if (shouldCache) { cachedProbeResults.set(cacheKey, { cacheKey, ...result, cachedAtMs: Date.now() }); } else { // Don't pin auth failures / transient failures in cache — user may fix and retry. cachedProbeResults.delete(cacheKey); } return result; })(); probeInFlightByKey.set(cacheKey, probePromise); try { return await probePromise; } finally { probeInFlightByKey.delete(cacheKey); } } private isAuthFailureWarning(text: string, source: AuthWarningSource): boolean { const lower = text.toLowerCase(); const hasExplicitCliAuthSignal = lower.includes('not authenticated') || lower.includes('not logged in') || lower.includes('please run /login') || lower.includes('missing api key') || lower.includes('invalid api key') || lower.includes('authentication failed') || lower.includes('not configured for runtime use') || lower.includes('set gemini_api_key') || lower.includes('google adc credentials') || lower.includes('google_cloud_project') || lower.includes('codex provider is not authenticated') || lower.includes('run `claude auth login`') || lower.includes('claude auth login') || lower.includes('claude-multimodel auth login'); if (hasExplicitCliAuthSignal) { return true; } if (source === 'assistant' || source === 'stdout') { return false; } const hasAuthStatus401 = /api error:\s*401\b/i.test(text) || /\b401 unauthorized\b/i.test(lower) || (/(^|\D)401(\D|$)/.test(lower) && (lower.includes('auth') || lower.includes('api') || lower.includes('login'))); return ( hasAuthStatus401 || (lower.includes('unauthorized') && (lower.includes('api') || lower.includes('auth') || lower.includes('login'))) ); } private hasApiError(text: string): boolean { return /api error:\s*\d{3}\b/i.test(text) || /invalid_request_error/i.test(text); } private sanitizeCliSnippet(text: string): string { // Remove control characters that often show up as binary noise in CLI error payloads. // Preserve newlines/tabs for readability. // eslint-disable-next-line no-control-regex, sonarjs/no-control-regex -- intentionally stripping control chars return text.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ''); } private normalizeApiRetryErrorMessage(text: string): string { const sanitized = this.sanitizeCliSnippet(text).trim(); if (!sanitized) { return sanitized; } const jsonMatch = /^\d{3}\s+(\{[\s\S]*\})$/.exec(sanitized); const jsonCandidate = jsonMatch?.[1] ?? (sanitized.startsWith('{') ? sanitized : null); if (jsonCandidate) { try { const parsed = JSON.parse(jsonCandidate) as { error?: { message?: unknown }; message?: unknown; }; const nestedMessage = typeof parsed.error?.message === 'string' ? parsed.error.message : typeof parsed.message === 'string' ? parsed.message : null; if (nestedMessage) { return this.normalizeApiRetryErrorMessage(nestedMessage); } } catch { // Fall through to raw sanitized text. } } return sanitized .replace(/^gemini cli backend error:\s*/i, '') .replace(/^gemini api backend error:\s*/i, '') .replace(/^api error:\s*\d+\s*/i, '') .trim(); } private isQuotaRetryMessage(text: string | undefined): boolean { const lower = (text ?? '').toLowerCase(); return ( lower.includes('quota will reset after') || lower.includes('exhausted your capacity on this model') || lower.includes('resource exhausted') || lower.includes('model cooldown') || lower.includes('cooling down') || lower.includes('rate limit') || lower.includes('rate_limit') ); } private toMarkdownCodeSafe(text: string): string { return this.sanitizeCliSnippet(text).replace(/```/g, '``\\`'); } private extractApiErrorSnippet(text: string): string | null { const match = /api error:\s*\d{3}\b/i.exec(text) ?? /invalid_request_error/i.exec(text); if (match?.index === undefined) return null; const start = Math.max(0, match.index - 200); const end = Math.min(text.length, match.index + 4000); const raw = text.slice(start, end).trim(); if (!raw) return null; // Avoid breaking markdown fences if the payload contains ``` accidentally. return this.sanitizeCliSnippet(raw).replace(/```/g, '``\\`'); } private failProvisioningWithApiError(run: ProvisioningRun, source: string): void { if (run.provisioningComplete || run.processKilled || run.authRetryInProgress) return; if (run.progress.state === 'failed' || run.cancelRequested) return; const combined = [ buildCombinedLogs(run.stdoutBuffer, run.stderrBuffer), run.provisioningOutputParts.length > 0 ? run.provisioningOutputParts.join('\n') : '', ] .filter(Boolean) .join('\n') .trim(); const snippet = this.extractApiErrorSnippet(combined) ?? this.extractApiErrorSnippet(source) ?? null; const status = /api error:\s*(\d{3})\b/i.exec(combined)?.[1] ?? /api error:\s*(\d{3})\b/i.exec(source)?.[1]; const hint = run.isLaunch ? 'Launch' : 'Provisioning'; const statusLabel = status ? `API Error ${status}` : 'API Error'; if (snippet) { run.provisioningOutputParts.push( `**${hint} failed: ${statusLabel} detected**\n\n\`\`\`\n${snippet}\n\`\`\`` ); } else { run.provisioningOutputParts.push(`**${hint} failed: ${statusLabel} detected**`); } const progress = updateProgress(run, 'failed', `${hint} failed — ${statusLabel}`, { error: `Claude CLI reported ${statusLabel} during startup. The team was not started.`, cliLogsTail: extractCliLogsFromRun(run), }); run.onProgress(progress); run.processKilled = true; run.cancelRequested = true; // SIGKILL: newer Claude CLI versions handle SIGTERM gracefully and delete // team files during cleanup. SIGKILL is uncatchable — files are preserved. killTeamProcess(run.child); this.cleanupRun(run); } /** * Shows a non-fatal API error warning in the Live output section. * Unlike failProvisioningWithApiError, does NOT kill the process — lets the SDK retry. * Deduplicates: only the first warning per run is shown. */ private emitApiErrorWarning(run: ProvisioningRun, text: string): void { if (run.provisioningComplete || run.processKilled || run.authRetryInProgress) return; if (run.progress.state === 'failed' || run.cancelRequested) return; if (run.apiErrorWarningEmitted) return; run.apiErrorWarningEmitted = true; const snippet = this.extractApiErrorSnippet(text); const status = /api error:\s*(\d{3})\b/i.exec(text)?.[1] ?? null; const label = status ? `API Error ${status}` : 'API Error'; const warningText = snippet ? `**${label} — SDK is retrying**\n\n\`\`\`\n${snippet}\n\`\`\`\n\nWaiting for retry...` : `**${label} — SDK is retrying**\n\nWaiting for retry...`; run.provisioningOutputParts.push(warningText); run.progress.message = `${label} — SDK retrying...`; emitLogsProgress(run); // Prevent double-emit: the calling stderr/stdout handler will also try throttled emitLogsProgress // after this returns. Updating lastLogProgressAt ensures the throttle check rejects it. run.lastLogProgressAt = Date.now(); } /** * Starts a periodic watchdog that detects when the CLI process has produced * no stdout/stderr data for an extended period. Pushes progressive warnings * into provisioningOutputParts so they appear in the Live output section. */ private startStallWatchdog(run: ProvisioningRun): void { if (run.stallCheckHandle) return; run.stallCheckHandle = setInterval(() => { // try/catch: Node.js does NOT catch errors in setInterval callbacks — // without this, an exception would silently kill the watchdog. try { if ( run.provisioningComplete || run.processKilled || run.cancelRequested || run.authRetryInProgress ) { this.stopStallWatchdog(run); return; } const now = Date.now(); const silenceMs = now - run.lastStdoutReceivedAt; if (silenceMs < STALL_WARNING_THRESHOLD_MS) return; // Instead of pushing new warnings (which bloats Live output), // replace the existing stall warning in-place so the displayed // silence duration stays current (20s → 30s → 1m → ...). const silenceSec = Math.round(silenceMs / 1000); const warningText = this.buildStallWarningText(silenceSec, run); if (run.stallWarningIndex != null) { run.provisioningOutputParts[run.stallWarningIndex] = warningText; } else { // Save current message ONLY if it's a normal provisioning message, // not a retry message (which has higher priority and its own lifecycle). if (run.progress.messageSeverity !== 'error') { run.preStallMessage = run.progress.message; } run.stallWarningIndex = run.provisioningOutputParts.length; run.provisioningOutputParts.push(warningText); } const mins = Math.floor(silenceSec / 60); const secs = silenceSec % 60; const elapsed = mins > 0 ? (secs > 0 ? `${mins}m ${secs}s` : `${mins}m`) : `${secs}s`; // If retry messages are flowing, they are more informative than our // generic stall text — don't overwrite progress.message / severity. // Only update the Live output (assistantOutput) with the stall warning. const retryActive = run.lastRetryAt > 0 && now - run.lastRetryAt < 90_000; run.progress = { ...run.progress, updatedAt: nowIso(), ...(!retryActive && { message: this.buildStallProgressMessage(silenceSec, elapsed), messageSeverity: 'warning' as const, }), assistantOutput: buildProvisioningLiveOutput(run) ?? run.progress.assistantOutput, }; run.onProgress(run.progress); } catch (err) { logger.error( `[${run.teamName}] Stall watchdog error: ${ err instanceof Error ? err.message : String(err) }` ); } }, STALL_CHECK_INTERVAL_MS); } private stopStallWatchdog(run: ProvisioningRun): void { if (run.stallCheckHandle) { clearInterval(run.stallCheckHandle); run.stallCheckHandle = null; } } private buildStallWarningText(silenceSec: number, run: ProvisioningRun): string { const mins = Math.floor(silenceSec / 60); const secs = silenceSec % 60; const elapsed = mins > 0 ? (secs > 0 ? `${mins}m ${secs}s` : `${mins}m`) : `${secs}s`; if (silenceSec < 60) { return ( `---\n\n` + `**Waiting for CLI response** (silent for ${elapsed})\n\n` + `The process is running but not producing output yet. Model responses can delay logs, ` + `and short waits like this are normal. The SDK also retries automatically if the ` + `request briefly hits rate limiting.\n\n` + `Waiting...` ); } if (silenceSec < 120) { return ( `---\n\n` + `**Waiting for CLI response** (silent for ${elapsed})\n\n` + `The process is still waiting for a model response. Logs can sometimes show up after ` + `1-1.5 minutes, and that is still okay. The SDK retries automatically if the ` + `request hits rate limiting (error 429 / model cooldown).\n\n` + `If there is still no output after 2 minutes, that starts to look unusual.\n\n` + `You can cancel and try again later if the wait continues.` ); } const modelName = run.request.model ?? 'default'; const effortLabel = run.request.effort ? ` (effort: ${run.request.effort})` : ''; return ( `---\n\n` + `**Extended CLI wait** (silent for ${elapsed})\n\n` + `Model **${modelName}**${effortLabel} is still waiting to respond. Some delay is normal, ` + `but no logs for ${elapsed} is already unusual.\n\n` + `Possible causes:\n` + `- Rate limiting / model cooldown (429) - SDK retries automatically\n` + `- API server overload for this model\n` + `- A stalled or delayed model response\n\n` + `Consider canceling and trying with a different model.` ); } private buildStallProgressMessage(silenceSec: number, elapsed: string): string { if (silenceSec < 120) { return `Waiting for model response for ${elapsed} - logs can be delayed, this is still OK`; } return `Still waiting for model response for ${elapsed} - this is unusual`; } /** * Detects auth failure keywords in stderr/stdout during provisioning. * On first detection: kills process, waits, and respawns automatically. * On second detection (after retry): fails fast with a clear error. */ private handleAuthFailureInOutput( run: ProvisioningRun, text: string, source: AuthWarningSource ): void { if (run.provisioningComplete || run.processKilled || run.authRetryInProgress) return; if (!this.isAuthFailureWarning(text, source)) return; if (!run.authFailureRetried) { logger.warn( `[${run.teamName}] Auth failure detected in ${source} during provisioning — ` + `will kill process and retry after ${PREFLIGHT_AUTH_RETRY_DELAY_MS}ms` ); run.authRetryInProgress = true; void this.respawnAfterAuthFailure(run); } else { logger.error(`[${run.teamName}] Auth failure detected in ${source} after retry — giving up`); run.processKilled = true; killTeamProcess(run.child); const progress = updateProgress(run, 'failed', 'Authentication failed — CLI requires login', { error: 'Claude CLI is not authenticated. Run `claude auth login` (or start `claude` and run `/login`) ' + 'to authenticate, or set ANTHROPIC_API_KEY and try again.', cliLogsTail: extractCliLogsFromRun(run), }); run.onProgress(progress); this.cleanupRun(run); } } /** * Kills the current process, waits for lock release, and respawns with saved context. * Reattaches all stream listeners and resends the prompt. */ private async respawnAfterAuthFailure(run: ProvisioningRun): Promise { const ctx = run.spawnContext; const stopAllGenerationAtStart = this.stopAllTeamsGeneration; if (!ctx) { logger.error(`[${run.teamName}] Cannot respawn — no spawn context saved`); run.authRetryInProgress = false; return; } // Tear down current process without full cleanupRun (keep run alive) if (run.timeoutHandle) { clearTimeout(run.timeoutHandle); run.timeoutHandle = null; } this.stopFilesystemMonitor(run); this.stopStallWatchdog(run); if (run.child) { run.child.stdout?.removeAllListeners('data'); run.child.stderr?.removeAllListeners('data'); run.child.removeAllListeners('error'); run.child.removeAllListeners('exit'); run.child.removeAllListeners('close'); killTeamProcess(run.child); run.child = null; } // Reset buffers for fresh attempt run.stdoutBuffer = ''; run.stderrBuffer = ''; run.claudeLogLines = []; run.lastClaudeLogStream = null; run.stdoutLogLineBuf = ''; run.stderrLogLineBuf = ''; run.claudeLogsUpdatedAt = undefined; run.authFailureRetried = true; run.apiErrorWarningEmitted = false; updateProgress(run, 'spawning', 'Auth failed — retrying after short delay'); run.onProgress(run.progress); await sleep(PREFLIGHT_AUTH_RETRY_DELAY_MS); if (run.cancelRequested) { run.authRetryInProgress = false; return; } // Verify --mcp-config still exists; regenerate if deleted (e.g. by stale GC) const mcpFlagIdx = ctx.args.indexOf('--mcp-config'); const bootstrapPromptFlagIdx = ctx.args.indexOf('--team-bootstrap-user-prompt-file'); if (mcpFlagIdx !== -1 && mcpFlagIdx + 1 < ctx.args.length) { const existingConfigPath = ctx.args[mcpFlagIdx + 1]; try { await fs.promises.access(existingConfigPath, fs.constants.F_OK); } catch { logger.warn(`[${run.teamName}] MCP config ${existingConfigPath} missing, regenerating`); try { const newConfigPath = await this.mcpConfigBuilder.writeConfigFile(ctx.cwd, { controlApiBaseUrl: ctx.env.CLAUDE_TEAM_CONTROL_URL, }); ctx.args[mcpFlagIdx + 1] = newConfigPath; run.mcpConfigPath = newConfigPath; logger.info(`[${run.teamName}] Regenerated MCP config at ${newConfigPath}`); } catch (regenErr) { run.authRetryInProgress = false; const progress = updateProgress(run, 'failed', 'Failed to regenerate MCP config', { error: regenErr instanceof Error ? regenErr.message : String(regenErr), cliLogsTail: extractCliLogsFromRun(run), }); run.onProgress(progress); this.cleanupRun(run); return; } } } if (bootstrapPromptFlagIdx !== -1 && bootstrapPromptFlagIdx + 1 < ctx.args.length) { const existingPromptPath = ctx.args[bootstrapPromptFlagIdx + 1]; try { await fs.promises.access(existingPromptPath, fs.constants.F_OK); } catch { const submissionState = await readBootstrapRealTaskSubmissionState(run.teamName); if (submissionState === 'submitted') { ctx.args.splice(bootstrapPromptFlagIdx, 2); ctx.prompt = ''; run.bootstrapUserPromptPath = null; } else if (submissionState === 'unknown') { run.authRetryInProgress = false; const progress = updateProgress( run, 'failed', 'Unable to safely retry first task after auth failure', { error: 'deterministic bootstrap recorded the first real task as unknown, so retry would risk a duplicate submission', cliLogsTail: extractCliLogsFromRun(run), } ); run.onProgress(progress); this.cleanupRun(run); return; } else if (ctx.prompt.trim().length === 0) { run.authRetryInProgress = false; const progress = updateProgress( run, 'failed', 'Failed to restore deferred first task after auth retry', { error: 'deterministic bootstrap user prompt file was missing and no prompt was available to regenerate it', cliLogsTail: extractCliLogsFromRun(run), } ); run.onProgress(progress); this.cleanupRun(run); return; } else { logger.warn( `[${run.teamName}] Bootstrap user prompt file ${existingPromptPath} missing, regenerating` ); try { const newPromptPath = await writeDeterministicBootstrapUserPromptFile(ctx.prompt); ctx.args[bootstrapPromptFlagIdx + 1] = newPromptPath; run.bootstrapUserPromptPath = newPromptPath; } catch (regenErr) { run.authRetryInProgress = false; const progress = updateProgress( run, 'failed', 'Failed to regenerate deferred first task for auth retry', { error: regenErr instanceof Error ? regenErr.message : String(regenErr), cliLogsTail: extractCliLogsFromRun(run), } ); run.onProgress(progress); this.cleanupRun(run); return; } } } } // Respawn with saved context — CLI handles its own auth refresh. let child: ReturnType; try { if (mcpFlagIdx !== -1 && mcpFlagIdx + 1 < ctx.args.length) { await this.validateAgentTeamsMcpRuntime( ctx.claudePath, ctx.cwd, ctx.env, ctx.args[mcpFlagIdx + 1], { isCancelled: () => run.cancelRequested || run.processKilled || this.stopAllTeamsGeneration !== stopAllGenerationAtStart, } ); } if ( run.cancelRequested || run.processKilled || this.stopAllTeamsGeneration !== stopAllGenerationAtStart ) { throw new Error('Team launch cancelled by app shutdown'); } child = spawnCli(ctx.claudePath, ctx.args, { cwd: ctx.cwd, env: { ...ctx.env }, stdio: ['pipe', 'pipe', 'pipe'], }); } catch (error) { run.authRetryInProgress = false; const progress = updateProgress(run, 'failed', 'Failed to respawn Claude CLI', { error: error instanceof Error ? error.message : String(error), }); run.onProgress(progress); this.cleanupRun(run); return; } logger.info( `[${run.teamName}] Respawned CLI process after auth failure (pid=${child.pid ?? '?'})` ); run.child = child; run.processClosed = false; run.authRetryInProgress = false; updateProgress(run, 'spawning', 'CLI respawned — sending prompt', { pid: child.pid ?? undefined, }); run.onProgress(run.progress); // Resend prompt only for legacy direct-stdin flows. Deterministic bootstrap // owns the first real task via --team-bootstrap-user-prompt-file. if (bootstrapPromptFlagIdx === -1 && child.stdin?.writable) { const message = JSON.stringify({ type: 'user', message: { role: 'user', content: [{ type: 'text', text: ctx.prompt }], }, }); child.stdin.write(message + '\n'); } // Reattach stdout handler this.attachStdoutHandler(run); // Reattach stderr handler this.attachStderrHandler(run); run.lastDataReceivedAt = Date.now(); run.lastStdoutReceivedAt = Date.now(); this.startStallWatchdog(run); // Restart filesystem monitor for createTeam (launch skips it) if (!run.isLaunch) { updateProgress(run, 'configuring', 'Waiting for team configuration...'); run.onProgress(run.progress); this.startFilesystemMonitor(run, run.request); } else { updateProgress( run, 'configuring', run.deterministicBootstrap ? 'CLI running - deterministic launch in progress' : 'CLI running - reconnecting with teammates' ); run.onProgress(run.progress); } // Restart timeout run.timeoutHandle = setTimeout(() => { if (!run.processKilled && !run.provisioningComplete) { run.processKilled = true; run.finalizingByTimeout = true; void (async () => { const readyOnTimeout = await this.tryCompleteAfterTimeout(run); killTeamProcess(run.child); if (readyOnTimeout) return; const hint = run.isLaunch ? ' (launch)' : ''; const progress = updateProgress(run, 'failed', `Timed out waiting for CLI${hint}`, { error: `Timed out waiting for CLI${hint}.`, cliLogsTail: extractCliLogsFromRun(run), }); run.onProgress(progress); this.cleanupRun(run); })(); } }, getProvisioningRunTimeoutMs(run)); child.once('error', (error) => { const hint = run.isLaunch ? ' (launch)' : ''; const progress = updateProgress(run, 'failed', `Failed to start Claude CLI${hint}`, { error: error.message, cliLogsTail: extractCliLogsFromRun(run), }); run.onProgress(progress); this.cleanupRun(run); }); child.once('close', (code) => { void this.handleProcessExit(run, code); }); } /** Attaches the stdout stream-json parser to the current child process. */ private attachStdoutHandler(run: ProvisioningRun): void { const child = run.child; if (!child?.stdout) return; let stdoutLineBuf = ''; child.stdout.on('data', (chunk: Buffer) => { // Reset generic data timestamp (used for other purposes, not stall detection). run.lastDataReceivedAt = Date.now(); const text = chunk.toString('utf8'); this.appendCliLogs(run, 'stdout', text); run.stdoutBuffer += text; if (run.stdoutBuffer.length > STDOUT_RING_LIMIT) { run.stdoutBuffer = run.stdoutBuffer.slice(run.stdoutBuffer.length - STDOUT_RING_LIMIT); } // Parse stream-json lines (newline-delimited JSON) stdoutLineBuf += text; const lines = stdoutLineBuf.split('\n'); stdoutLineBuf = lines.pop() ?? ''; this.updateStdoutParserCarry(run, stdoutLineBuf); for (const line of lines) { const trimmed = line.trim(); this.handleStdoutParserLine(run, trimmed); } const currentTs = Date.now(); if (currentTs - run.lastLogProgressAt >= LOG_PROGRESS_THROTTLE_MS) { run.lastLogProgressAt = currentTs; emitLogsProgress(run); } }); } private updateStdoutParserCarry(run: ProvisioningRun, carry: string): void { run.stdoutParserCarry = carry; const trimmedCarry = carry.trim(); if (!trimmedCarry) { run.stdoutParserCarryIsCompleteJson = false; run.stdoutParserCarryLooksLikeClaudeJson = false; return; } try { JSON.parse(trimmedCarry); run.stdoutParserCarryIsCompleteJson = true; } catch { run.stdoutParserCarryIsCompleteJson = false; } run.stdoutParserCarryLooksLikeClaudeJson = looksLikeClaudeStdoutJsonFragment(trimmedCarry); } private flushStdoutParserCarry(run: ProvisioningRun): void { const stdoutParserCarry = typeof run.stdoutParserCarry === 'string' ? run.stdoutParserCarry : ''; const trimmed = stdoutParserCarry.trim(); if (!trimmed || !run.stdoutParserCarryIsCompleteJson) { return; } logger.warn( `[${run.teamName}] Flushing final stream-json stdout carry before process close handling`, this.buildStdoutCarryDiagnostic(run) ); this.handleStdoutParserLine(run, trimmed); this.updateStdoutParserCarry(run, ''); } private buildStdoutCarryDiagnostic(run: ProvisioningRun): Record { const stdoutParserCarry = typeof run.stdoutParserCarry === 'string' ? run.stdoutParserCarry : ''; const diagnostic: Record = { runId: run.runId, stdoutCarryLength: stdoutParserCarry.length, stdoutCarryCompleteJson: run.stdoutParserCarryIsCompleteJson === true, stdoutCarryLooksLikeClaudeJson: run.stdoutParserCarryLooksLikeClaudeJson === true, }; if (run.stdoutParserCarryIsCompleteJson === true) { try { const parsed = JSON.parse(stdoutParserCarry.trim()) as Record; diagnostic.messageType = typeof parsed.type === 'string' ? parsed.type : null; diagnostic.messageSubtype = typeof parsed.subtype === 'string' ? parsed.subtype : null; diagnostic.bootstrapEvent = typeof parsed.event === 'string' ? parsed.event : null; diagnostic.sequence = typeof parsed.seq === 'number' ? parsed.seq : null; } catch { diagnostic.messageType = null; } } return diagnostic; } private getUnconfirmedBootstrapMemberNames(run: ProvisioningRun): string[] { return run.expectedMembers.filter((expected) => { const status = run.memberSpawnStatuses.get(expected); return status?.bootstrapConfirmed !== true && status?.skippedForLaunch !== true; }); } private handleStdoutParserLine(run: ProvisioningRun, trimmed: string): void { if (!trimmed) { return; } try { const msg = JSON.parse(trimmed) as Record; this.handleParsedStdoutJsonMessage(run, msg); } catch { // Not valid JSON - check for auth failure in raw text output. this.handleAuthFailureInOutput(run, trimmed, 'stdout'); if (this.hasApiError(trimmed) && !this.isAuthFailureWarning(trimmed, 'stdout')) { // Show warning but do not kill - the SDK may be retrying internally (e.g. 429 model_cooldown). // If all retries fail, result.subtype="error" will catch it and kill then. this.emitApiErrorWarning(run, trimmed); } } } private handleParsedStdoutJsonMessage(run: ProvisioningRun, msg: Record): void { // Only reset stall timer on messages that represent actual API progress // (assistant response or result). System messages like retry attempts // (type=system, subtype=attempt) are informational - the CLI is still // waiting for the API and the user should see the stall warning. const msgType = msg.type; if (msgType === 'assistant' || msgType === 'result') { run.lastStdoutReceivedAt = Date.now(); if (run.stallWarningIndex != null) { const removedIndex = run.stallWarningIndex; run.provisioningOutputParts.splice(removedIndex, 1); this.shiftProvisioningOutputIndexesAfterRemoval(run, removedIndex); run.stallWarningIndex = null; if (run.preStallMessage != null) { run.progress.message = run.preStallMessage; run.preStallMessage = null; delete run.progress.messageSeverity; } } } this.handleStreamJsonMessage(run, msg); } /** Attaches the stderr handler with auth failure detection. */ private attachStderrHandler(run: ProvisioningRun): void { const child = run.child; if (!child?.stderr) return; child.stderr.on('data', (chunk: Buffer) => { // Reset stall watchdog FIRST — any data (even partial JSON) means the CLI is alive. run.lastDataReceivedAt = Date.now(); const text = chunk.toString('utf8'); this.appendCliLogs(run, 'stderr', text); run.stderrBuffer += text; if (run.stderrBuffer.length > STDERR_RING_LIMIT) { run.stderrBuffer = run.stderrBuffer.slice(run.stderrBuffer.length - STDERR_RING_LIMIT); } // Detect auth failure early instead of waiting for 5-minute timeout this.handleAuthFailureInOutput(run, text, 'stderr'); if (this.hasApiError(text) && !this.isAuthFailureWarning(text, 'stderr')) { // Show warning but do NOT kill — the SDK may be retrying internally (e.g. 429 model_cooldown). // If all retries fail, result.subtype="error" will catch it and kill then. this.emitApiErrorWarning(run, text); } const currentTs = Date.now(); if (currentTs - run.lastLogProgressAt >= LOG_PROGRESS_THROTTLE_MS) { run.lastLogProgressAt = currentTs; emitLogsProgress(run); } }); } async createTeam( request: TeamCreateRequest, onProgress: (progress: TeamProvisioningProgress) => void ): Promise { return this.withTeamLock(request.teamName, async () => { return this._createTeamInner(request, onProgress); }); } private async _createTeamInner( request: TeamCreateRequest, onProgress: (progress: TeamProvisioningProgress) => void ): Promise { this.cleanedStoppedTeamOpenCodeRuntimeLanes.delete(request.teamName); const existingProvisioningRunId = this.getResolvableProvisioningRunId(request.teamName); if (existingProvisioningRunId) { return { runId: existingProvisioningRunId }; } const previousLaunchSnapshot = await this.readTaskActivityRepairLaunchSnapshot( request.teamName ); this.repairStaleTaskActivityIntervalsOnce(request.teamName, previousLaunchSnapshot); const stopAllGenerationAtStart = this.stopAllTeamsGeneration; assertAppDeterministicBootstrapEnabled(); if (this.shouldRouteOpenCodeToRuntimeAdapter(request)) { return this.createOpenCodeTeamThroughRuntimeAdapter(request, onProgress); } assertOpenCodeNotLaunchedThroughLegacyProvisioning(request); // Set immediately to prevent TOCTOU (defense in depth alongside withTeamLock) const pendingKey = `pending-${randomUUID()}`; this.provisioningRunByTeam.set(request.teamName, pendingKey); try { const teamsBasePathsToProbe = getTeamsBasePathsToProbe(); for (const probe of teamsBasePathsToProbe) { const configPath = path.join(probe.basePath, request.teamName, 'config.json'); if (await this.pathExists(configPath)) { const suffix = probe.location === 'configured' ? '' : ` (found under ${probe.basePath})`; throw new Error(`Team already exists${suffix}`); } } await ensureCwdExists(request.cwd); const claudePath = await ClaudeBinaryResolver.resolve(); if (!claudePath) { throw buildMissingCliError(); } const runtimeAuthMaterialId = randomUUID(); const teamRuntimeAuth: TeamRuntimeAuthContext = { teamName: request.teamName, authMaterialId: runtimeAuthMaterialId, allowAnthropicApiKeyHelper: true, }; const provisioningEnv = await this.buildProvisioningEnv( request.providerId, request.providerBackendId, { includeCodexTeammateAuth: teamRequestIncludesCodexMember(request), teamRuntimeAuth } ); const { env: shellEnv, geminiRuntimeAuth, providerArgs = [], warning: envWarning, } = provisioningEnv; if (envWarning) { throw new Error(envWarning); } const workspaceTrustFeatureFlags = resolveWorkspaceTrustFeatureFlags(); const workspaceTrustProviders = workspaceTrustFeatureFlags.enabled ? this.collectWorkspaceTrustProviders({ leadProviderId: request.providerId, members: request.members, }) : []; const workspaceTrustEarlyWorkspaces = workspaceTrustFeatureFlags.enabled ? await this.collectWorkspaceTrustWorkspaces({ cwd: request.cwd, members: [], }) : []; const workspaceTrustEarlyPlan = workspaceTrustFeatureFlags.enabled ? await this.planWorkspaceTrustArgsOnlySafely({ providers: workspaceTrustProviders, workspaces: workspaceTrustEarlyWorkspaces, targetSurfaces: ['default_model_probe'], featureFlags: workspaceTrustFeatureFlags, }) : { launchArgPatches: [] }; const workspaceTrustProviderArgsResolver = this.createDefaultModelWorkspaceTrustProviderArgsResolver(workspaceTrustEarlyPlan); const materializedMemberSpecs = await this.materializeEffectiveTeamMemberSpecs({ claudePath, cwd: request.cwd, members: request.members, defaults: { providerId: request.providerId, model: request.model, effort: request.effort, }, primaryProviderId: request.providerId, primaryEnv: provisioningEnv, teamRuntimeAuth, limitContext: request.limitContext, providerArgsResolver: workspaceTrustProviderArgsResolver, }); const allEffectiveMemberSpecs = await this.resolveOpenCodeMemberWorkspacesForRuntime({ teamName: request.teamName, baseCwd: request.cwd, leadProviderId: request.providerId, members: materializedMemberSpecs, }); Object.assign( shellEnv, await this.buildRuntimeTurnSettledEnvironmentForMembers( request.providerId, allEffectiveMemberSpecs ) ); const lanePlan = this.planRuntimeLanesOrThrow(request.providerId, allEffectiveMemberSpecs); const primaryMemberNames = new Set(lanePlan.primaryMembers.map((member) => member.name)); const effectiveMemberSpecs = allEffectiveMemberSpecs.filter((member) => primaryMemberNames.has(member.name) ); assertDeterministicBootstrapPrimaryMemberLimit(effectiveMemberSpecs.length); const largeTeamWarning = buildLargeDeterministicBootstrapWarning(effectiveMemberSpecs.length); const resolvedProviderId = resolveTeamProviderId(request.providerId); const crossProviderMemberArgs = await this.buildCrossProviderMemberArgs( resolvedProviderId, effectiveMemberSpecs, { teamRuntimeAuth } ); const workspaceTrustFullWorkspaces = workspaceTrustFeatureFlags.enabled ? await this.collectWorkspaceTrustWorkspaces({ cwd: request.cwd, members: allEffectiveMemberSpecs, }) : []; const workspaceTrustFullPlan = workspaceTrustFeatureFlags.enabled ? await this.planWorkspaceTrustFullSafely({ providers: this.collectWorkspaceTrustProviders({ leadProviderId: request.providerId, members: allEffectiveMemberSpecs, }), workspaces: workspaceTrustFullWorkspaces, featureFlags: workspaceTrustFeatureFlags, }) : null; const workspaceTrustPatches = workspaceTrustFullPlan?.launchArgPatches ?? []; const providerArgsForLaunch = this.applyWorkspaceTrustArgPatches({ args: providerArgs, patches: workspaceTrustPatches, targetProvider: resolvedProviderId, targetSurface: 'primary_provider_args', }); const crossProviderArgsForLaunch = crossProviderMemberArgs.providerArgsByProvider.has('codex') ? this.applyWorkspaceTrustArgPatches({ args: crossProviderMemberArgs.args, patches: workspaceTrustPatches, targetProvider: 'codex', targetSurface: 'cross_provider_member_args', }) : crossProviderMemberArgs.args; const crossProviderMemberArgsForLaunch = { ...crossProviderMemberArgs, args: crossProviderArgsForLaunch, }; Object.assign(shellEnv, crossProviderMemberArgs.envPatch); if (crossProviderMemberArgs.usesAnthropicApiKeyHelper) { for (const key of ANTHROPIC_HELPER_MODE_COMPETING_AUTH_ENV_KEYS) { delete shellEnv[key]; } } const providerArgsByProvider = new Map(); for (const [providerId, args] of new Map([ [resolvedProviderId, providerArgsForLaunch], ...crossProviderMemberArgs.providerArgsByProvider, ])) { providerArgsByProvider.set( providerId, this.applyWorkspaceTrustArgPatches({ args, patches: workspaceTrustPatches, targetProvider: providerId, targetSurface: 'provider_facts_probe', }) ); } const launchIdentity = await this.resolveAndValidateLaunchIdentity({ claudePath, cwd: request.cwd, env: shellEnv, request, effectiveMembers: effectiveMemberSpecs, providerArgsByProvider, }); const runId = randomUUID(); const startedAt = nowIso(); const run: ProvisioningRun = { runId, teamName: request.teamName, startedAt, stdoutBuffer: '', stderrBuffer: '', claudeLogLines: [], lastClaudeLogStream: null, stdoutLogLineBuf: '', stderrLogLineBuf: '', stdoutParserCarry: '', stdoutParserCarryIsCompleteJson: false, stdoutParserCarryLooksLikeClaudeJson: false, claudeLogsUpdatedAt: undefined, deterministicBootstrapStartedAt: undefined, lastDeterministicBootstrapEvent: undefined, lastDeterministicBootstrapPhase: undefined, deterministicBootstrapMemberSpawnSeen: false, deterministicBootstrapMemberResultSeen: false, processKilled: false, finalizingByTimeout: false, cancelRequested: false, teamsBasePathsToProbe, child: null, timeoutHandle: null, fsMonitorHandle: null, onProgress, expectedMembers: effectiveMemberSpecs.map((member) => member.name), request, allEffectiveMembers: allEffectiveMemberSpecs, effectiveMembers: effectiveMemberSpecs, launchIdentity, mixedSecondaryLanes: this.createMixedSecondaryLaneStates(lanePlan), lastLogProgressAt: 0, lastDataReceivedAt: 0, // intentionally 0 — real reset happens after spawn (see startStallWatchdog call sites) lastStdoutReceivedAt: 0, stallCheckHandle: null, stallWarningIndex: null, preStallMessage: null, lastRetryAt: 0, apiRetryWarningIndex: null, apiErrorWarningEmitted: false, waitingTasksSince: null, provisioningComplete: false, processClosed: false, requiresFirstRealTurnSuccess: false, firstRealTurnSucceeded: false, mcpConfigPath: null, memberMcpConfigPaths: [], bootstrapSpecPath: null, bootstrapUserPromptPath: null, isLaunch: false, launchStateClearedForRun: false, deterministicBootstrap: true, workspaceTrustPlan: workspaceTrustFullPlan, workspaceTrustExecution: null, workspaceTrustDiagnostics: null, workspaceTrustRetryAttempted: false, fsPhase: 'waiting_config', leadRelayCapture: null, activeCrossTeamReplyHints: [], leadMsgSeq: 0, liveLeadTextBuffer: null, pendingToolCalls: [], activeToolCalls: new Map(), pendingDirectCrossTeamSendRefresh: false, lastLeadTextEmitMs: 0, silentUserDmForward: null, silentUserDmForwardClearHandle: null, pendingInboxRelayCandidates: [], provisioningOutputParts: [], provisioningTraceLines: [], lastProvisioningTraceKey: null, provisioningOutputIndexByMessageId: new Map(), detectedSessionId: null, leadActivityState: 'active', leadContextUsage: null, authFailureRetried: false, authRetryInProgress: false, spawnContext: null, anthropicApiKeyHelper: provisioningEnv.anthropicApiKeyHelper ?? null, pendingApprovals: new Map(), processedPermissionRequestIds: new Set(), pendingPostCompactReminder: false, postCompactReminderInFlight: false, suppressPostCompactReminderOutput: false, pendingGeminiPostLaunchHydration: false, geminiPostLaunchHydrationInFlight: false, geminiPostLaunchHydrationSent: false, suppressGeminiPostLaunchHydrationOutput: false, memberSpawnStatuses: new Map( effectiveMemberSpecs.map((member) => [member.name, createInitialMemberSpawnStatusEntry()]) ), memberSpawnToolUseIds: new Map(), pendingMemberRestarts: new Map(), memberSpawnLeadInboxCursorByMember: new Map(), lastDeterministicBootstrapSeq: 0, lastMemberSpawnAuditAt: 0, lastMemberSpawnAuditConfigReadWarningAt: 0, lastMemberSpawnAuditMissingWarningAt: new Map(), progress: { runId, teamName: request.teamName, state: 'validating', message: 'Validating team provisioning request', startedAt, updatedAt: startedAt, warnings: largeTeamWarning ? [largeTeamWarning] : undefined, cliLogsTail: undefined, }, }; this.resetTeamScopedTransientStateForNewRun(request.teamName); this.runs.set(runId, run); this.provisioningRunByTeam.set(request.teamName, runId); initializeProvisioningTrace(run); run.onProgress(run.progress); await this.prepareWorkspaceTrustForDeterministicRun({ mode: 'create', run, claudePath, shellEnv, stopAllGenerationAtStart, workspaceTrustPlan: workspaceTrustFullPlan, featureFlags: workspaceTrustFeatureFlags, provisioningEnv, }); emitProvisioningCheckpoint(run, 'Clearing persisted launch state'); await this.clearPersistedLaunchState(request.teamName, { expectedRunId: run.runId }); run.launchStateClearedForRun = true; const initialUserPrompt = request.prompt?.trim() ?? ''; const promptSize = getPromptSizeSummary(initialUserPrompt); let child: ReturnType; shellEnv.CLAUDE_ENABLE_DETERMINISTIC_TEAM_BOOTSTRAP = '1'; const teammateModeDecision = await resolveDesktopTeammateModeDecision(request.extraCliArgs); applyDesktopTeammateModeDecisionToEnv(shellEnv, teammateModeDecision); let mcpConfigPath: string; let bootstrapSpecPath: string; let bootstrapUserPromptPath: string | null = null; try { // Pre-save our meta files before native app-managed briefing generation. // member_briefing intentionally reads canonical team metadata/inboxes, so // createTeam must materialize those files before building the bootstrap spec. emitProvisioningCheckpoint(run, 'Persisting team metadata before spawn'); const teamDir = path.join(getTeamsBasePath(), request.teamName); const tasksDir = path.join(getTasksBasePath(), request.teamName); await fs.promises.mkdir(teamDir, { recursive: true }); await fs.promises.mkdir(tasksDir, { recursive: true }); await this.teamMetaStore.writeMeta(request.teamName, { displayName: request.displayName, description: request.description, color: request.color, cwd: request.cwd, prompt: request.prompt, providerId: request.providerId, providerBackendId: request.providerBackendId, model: request.model, effort: request.effort, fastMode: request.fastMode, skipPermissions: request.skipPermissions, worktree: request.worktree, extraCliArgs: request.extraCliArgs, limitContext: request.limitContext, launchIdentity, createdAt: Date.now(), }); const membersToWrite = this.buildMembersMetaWritePayload(allEffectiveMemberSpecs); await this.membersMetaStore.writeMembers(request.teamName, membersToWrite, { providerBackendId: request.providerBackendId, }); emitProvisioningCheckpoint( run, 'Building deterministic create bootstrap spec', `expectedMembers=${effectiveMemberSpecs.length}` ); const nativeBootstrapBuild = await buildNativeAppManagedBootstrapSpecsWithDiagnostics({ teamName: request.teamName, cwd: request.cwd, members: effectiveMemberSpecs, }); const memberMcpLaunchConfigs = await this.buildRuntimeBootstrapMemberMcpLaunchConfigs({ controlApiBaseUrl: provisioningEnv.env.CLAUDE_TEAM_CONTROL_URL, cwd: request.cwd, members: effectiveMemberSpecs, run, }); if (nativeBootstrapBuild.diagnostics.warning) { run.progress = { ...run.progress, warnings: mergeProvisioningWarnings( run.progress.warnings, nativeBootstrapBuild.diagnostics.warning ), }; emitProvisioningCheckpoint( run, 'Native bootstrap startup context is large', nativeBootstrapBuild.diagnostics.warning ); } const bootstrapSpec = buildDeterministicCreateBootstrapSpec( runId, request, effectiveMemberSpecs, nativeBootstrapBuild.specs, memberMcpLaunchConfigs ); emitProvisioningCheckpoint(run, 'Writing deterministic bootstrap spec file'); bootstrapSpecPath = await writeDeterministicBootstrapSpecFile(bootstrapSpec); run.bootstrapSpecPath = bootstrapSpecPath; if (initialUserPrompt) { emitProvisioningCheckpoint( run, 'Writing deferred user prompt file', `chars=${promptSize.chars} lines=${promptSize.lines}` ); bootstrapUserPromptPath = await writeDeterministicBootstrapUserPromptFile(initialUserPrompt); run.bootstrapUserPromptPath = bootstrapUserPromptPath; run.requiresFirstRealTurnSuccess = true; } emitProvisioningCheckpoint(run, 'Writing MCP config file'); mcpConfigPath = await this.mcpConfigBuilder.writeConfigFile(request.cwd, { controlApiBaseUrl: provisioningEnv.env.CLAUDE_TEAM_CONTROL_URL, }); run.mcpConfigPath = mcpConfigPath; emitProvisioningCheckpoint(run, 'Validating agent-teams MCP runtime'); await this.validateAgentTeamsMcpRuntime(claudePath, request.cwd, shellEnv, mcpConfigPath, { isCancelled: () => run.cancelRequested || run.processKilled || this.stopAllTeamsGeneration !== stopAllGenerationAtStart, }); } catch (error) { this.runs.delete(runId); this.provisioningRunByTeam.delete(request.teamName); if (provisioningEnv.anthropicApiKeyHelper) { await cleanupAnthropicTeamApiKeyHelperMaterial({ directory: provisioningEnv.anthropicApiKeyHelper.directory, }).catch(() => undefined); } await this.teamMetaStore.deleteMeta(request.teamName).catch(() => {}); const teamDir = path.join(getTeamsBasePath(), request.teamName); const tasksDir = path.join(getTasksBasePath(), request.teamName); await fs.promises.rm(teamDir, { recursive: true, force: true }).catch(() => {}); await fs.promises.rm(tasksDir, { recursive: true, force: true }).catch(() => {}); await removeDeterministicBootstrapSpecFile(run.bootstrapSpecPath).catch(() => {}); run.bootstrapSpecPath = null; await removeDeterministicBootstrapUserPromptFile(run.bootstrapUserPromptPath).catch( () => {} ); run.bootstrapUserPromptPath = null; if (run.mcpConfigPath) { await this.mcpConfigBuilder.removeConfigFile(run.mcpConfigPath).catch(() => {}); run.mcpConfigPath = null; } await this.removeRunMemberMcpConfigFiles(run).catch(() => {}); throw error; } const launchModelArg = getLaunchModelArg( resolveTeamProviderId(request.providerId), request.model, launchIdentity ); const extraCliArgs = parseCliArgs(request.extraCliArgs); const runtimeArgsPlan = await this.buildTeamRuntimeLaunchArgsPlan({ teamName: request.teamName, providerId: resolvedProviderId, launchIdentity, envResolution: { ...provisioningEnv, providerArgs: providerArgsForLaunch }, extraArgs: extraCliArgs, inheritedProviderArgs: crossProviderMemberArgsForLaunch.args, includeAnthropicHelper: resolvedProviderId === 'anthropic', contextLabel: 'Team create launch', }); const spawnArgs = mergeJsonSettingsArgs([ '--print', '--input-format', 'stream-json', '--output-format', 'stream-json', '--verbose', '--setting-sources', 'user,project,local', '--mcp-config', mcpConfigPath, '--team-bootstrap-spec', bootstrapSpecPath, ...(bootstrapUserPromptPath ? ['--team-bootstrap-user-prompt-file', bootstrapUserPromptPath] : []), '--disallowedTools', APP_TEAM_RUNTIME_DISALLOWED_TOOLS, // Explicit --permission-mode overrides user's defaultMode in ~/.claude/settings.json // (e.g. "acceptEdits") which otherwise takes precedence over CLI flags ...(request.skipPermissions !== false ? ['--dangerously-skip-permissions', '--permission-mode', 'bypassPermissions'] : ['--permission-prompt-tool', 'stdio', '--permission-mode', 'default']), ...(launchModelArg ? ['--model', launchModelArg] : []), ...(launchIdentity.resolvedEffort ? ['--effort', launchIdentity.resolvedEffort] : []), ...runtimeArgsPlan.fastModeArgs, ...runtimeArgsPlan.runtimeTurnSettledHookArgs, ...(request.worktree ? ['--worktree', request.worktree] : []), ...buildDesktopTeammateModeCliArgs(teammateModeDecision), ...runtimeArgsPlan.extraArgs, ...runtimeArgsPlan.providerArgs, ...runtimeArgsPlan.settingsArgs, ...runtimeArgsPlan.inheritedProviderArgs, ]); applyAppManagedRuntimeSettingsPathEnv(shellEnv, runtimeArgsPlan.appManagedSettingsPath); const runtimeWarning = buildRuntimeLaunchWarning(request, shellEnv, { geminiRuntimeAuth, promptSize, expectedMembersCount: effectiveMemberSpecs.length, }); logRuntimeLaunchSnapshot(logger, request.teamName, claudePath, spawnArgs, request, shellEnv, { geminiRuntimeAuth, promptSize, expectedMembersCount: effectiveMemberSpecs.length, launchIdentity, }); try { if ( run.cancelRequested || run.processKilled || this.stopAllTeamsGeneration !== stopAllGenerationAtStart ) { throw new Error('Team launch cancelled by app shutdown'); } if (request.skipPermissions === false) { emitProvisioningCheckpoint(run, 'Seeding lead bootstrap permission rules'); await this.seedLeadBootstrapPermissionRules(request.teamName, request.cwd); } emitProvisioningCheckpoint( run, 'Spawning Claude CLI process', `args=${spawnArgs.length} cwd=${request.cwd}` ); child = spawnCli(claudePath, spawnArgs, { cwd: request.cwd, env: { ...shellEnv }, stdio: ['pipe', 'pipe', 'pipe'], }); } catch (error) { // Clean up pre-saved meta files if spawn failed (instant failure, not transient) await this.teamMetaStore.deleteMeta(request.teamName).catch(() => {}); const teamDir = path.join(getTeamsBasePath(), request.teamName); const tasksDir = path.join(getTasksBasePath(), request.teamName); await fs.promises.rm(teamDir, { recursive: true, force: true }).catch(() => {}); await fs.promises.rm(tasksDir, { recursive: true, force: true }).catch(() => {}); await removeDeterministicBootstrapSpecFile(run.bootstrapSpecPath).catch(() => {}); run.bootstrapSpecPath = null; await removeDeterministicBootstrapUserPromptFile(run.bootstrapUserPromptPath).catch( () => {} ); run.bootstrapUserPromptPath = null; if (run.mcpConfigPath) { await this.mcpConfigBuilder.removeConfigFile(run.mcpConfigPath).catch(() => {}); run.mcpConfigPath = null; } await this.removeRunMemberMcpConfigFiles(run).catch(() => {}); if (provisioningEnv.anthropicApiKeyHelper) { await cleanupAnthropicTeamApiKeyHelperMaterial({ directory: provisioningEnv.anthropicApiKeyHelper.directory, }).catch(() => undefined); } this.runs.delete(runId); this.provisioningRunByTeam.delete(request.teamName); throw error; } updateProgress(run, 'spawning', 'Starting Claude CLI process', { pid: child.pid ?? undefined, warnings: mergeProvisioningWarnings(run.progress.warnings, runtimeWarning), }); run.onProgress(run.progress); run.child = child; run.processClosed = false; run.spawnContext = { claudePath, args: spawnArgs, cwd: request.cwd, env: { ...shellEnv }, prompt: initialUserPrompt, }; this.attachStdoutHandler(run); this.attachStderrHandler(run); // Reset AFTER spawn — not at run init — because async operations (buildProvisioningEnv, // writeConfigFile) between init and spawn can take seconds, causing false stall warnings. run.lastDataReceivedAt = Date.now(); run.lastStdoutReceivedAt = Date.now(); this.startStallWatchdog(run); // Filesystem-based progress monitor: actively polls team files instead // of relying on stdout (which only arrives at the end in text mode). // When config + members + tasks are all present, kill the process early // rather than waiting for it to deadlock on system-reminder shutdown. updateProgress(run, 'configuring', 'Waiting for team configuration...'); run.onProgress(run.progress); this.startFilesystemMonitor(run, request); run.timeoutHandle = setTimeout(() => { if (!run.processKilled && !run.provisioningComplete) { run.processKilled = true; run.finalizingByTimeout = true; void (async () => { const readyOnTimeout = await this.tryCompleteAfterTimeout(run); killTeamProcess(run.child); if (readyOnTimeout) { return; // cleanupRun already called inside tryCompleteAfterTimeout } const progress = updateProgress(run, 'failed', 'Timed out waiting for CLI', { error: 'Timed out waiting for CLI. Run `claude` once in terminal to complete onboarding and try again.', cliLogsTail: extractCliLogsFromRun(run), }); run.onProgress(progress); this.cleanupRun(run); })(); } }, getProvisioningRunTimeoutMs(run)); child.once('error', (error) => { const progress = updateProgress(run, 'failed', 'Failed to start Claude CLI', { error: error.message, cliLogsTail: extractCliLogsFromRun(run), }); run.onProgress(progress); this.cleanupRun(run); }); child.once('close', (code) => { void this.handleProcessExit(run, code); }); return { runId }; } catch (error) { // Ensure the per-team lock doesn't get stuck on failures. if (this.provisioningRunByTeam.get(request.teamName) === pendingKey) { this.provisioningRunByTeam.delete(request.teamName); } throw error; } } private async createOpenCodeTeamThroughRuntimeAdapter( request: TeamCreateRequest, onProgress: (progress: TeamProvisioningProgress) => void ): Promise { const teamsBasePathsToProbe = getTeamsBasePathsToProbe(); for (const probe of teamsBasePathsToProbe) { const configPath = path.join(probe.basePath, request.teamName, 'config.json'); if (await this.pathExists(configPath)) { const suffix = probe.location === 'configured' ? '' : ` (found under ${probe.basePath})`; throw new Error(`Team already exists${suffix}`); } } await ensureCwdExists(request.cwd); const materialized = await this.materializeOpenCodeRuntimeAdapterDefaults({ request, members: request.members, }); const launchRequest = materialized.request; const effectiveMembers = await this.resolveOpenCodeMemberWorkspacesForRuntime({ teamName: launchRequest.teamName, baseCwd: launchRequest.cwd, leadProviderId: launchRequest.providerId, members: materialized.members, }); const teamDir = path.join(getTeamsBasePath(), launchRequest.teamName); const tasksDir = path.join(getTasksBasePath(), launchRequest.teamName); await fs.promises.mkdir(teamDir, { recursive: true }); await fs.promises.mkdir(tasksDir, { recursive: true }); await this.teamMetaStore.writeMeta(launchRequest.teamName, { displayName: launchRequest.displayName, description: launchRequest.description, color: launchRequest.color, cwd: launchRequest.cwd, prompt: launchRequest.prompt, providerId: launchRequest.providerId, providerBackendId: launchRequest.providerBackendId, model: launchRequest.model, effort: launchRequest.effort, skipPermissions: launchRequest.skipPermissions, worktree: launchRequest.worktree, extraCliArgs: launchRequest.extraCliArgs, limitContext: launchRequest.limitContext, createdAt: Date.now(), }); const membersToWrite = this.buildMembersMetaWritePayload(effectiveMembers); await this.membersMetaStore.writeMembers(launchRequest.teamName, membersToWrite, { providerBackendId: launchRequest.providerBackendId, }); await this.writeOpenCodeTeamConfig(launchRequest, effectiveMembers); return this.runOpenCodeTeamRuntimeAdapterLaunch({ request: launchRequest, members: effectiveMembers, prompt: launchRequest.prompt?.trim() ?? '', sourceWarning: undefined, onProgress, }); } private async launchOpenCodeTeamThroughRuntimeAdapter( request: TeamLaunchRequest, onProgress: (progress: TeamProvisioningProgress) => void ): Promise { const configPath = path.join(getTeamsBasePath(), request.teamName, 'config.json'); const configRaw = await tryReadRegularFileUtf8(configPath, { timeoutMs: TEAM_JSON_READ_TIMEOUT_MS, maxBytes: TEAM_CONFIG_MAX_BYTES, }); if (!configRaw) { throw new Error(`Team "${request.teamName}" not found — config.json does not exist`); } await ensureCwdExists(request.cwd); const { members, warning } = await this.resolveLaunchExpectedMembers( request.teamName, configRaw, request.providerId ); const materialized = await this.materializeOpenCodeRuntimeAdapterDefaults({ request, members, }); const launchRequest = materialized.request; const effectiveMembers = await this.resolveOpenCodeMemberWorkspacesForRuntime({ teamName: launchRequest.teamName, baseCwd: launchRequest.cwd, leadProviderId: launchRequest.providerId, members: materialized.members, }); await this.updateConfigProjectPath(launchRequest.teamName, launchRequest.cwd); let existingTasks: TeamTask[] = []; try { existingTasks = await new TeamTaskReader().getTasks(request.teamName); } catch (error) { logger.warn( `[${request.teamName}] Failed to read tasks for OpenCode launch prompt: ${String(error)}` ); } const prompt = buildDeterministicLaunchHydrationPrompt( launchRequest, effectiveMembers, existingTasks, false ); return this.runOpenCodeTeamRuntimeAdapterLaunch({ request: launchRequest, members: effectiveMembers, prompt, sourceWarning: warning, onProgress, }); } private async runOpenCodeTeamRuntimeAdapterLaunch(input: { request: TeamCreateRequest | TeamLaunchRequest; members: TeamCreateRequest['members']; prompt: string; sourceWarning?: string; onProgress: (progress: TeamProvisioningProgress) => void; }): Promise { const adapter = this.getOpenCodeRuntimeAdapter(); if (!adapter) { throw new Error('OpenCode runtime adapter is not registered'); } const stopAllGenerationAtStart = this.stopAllTeamsGeneration; const previousRuntimeRun = this.runtimeAdapterRunByTeam.get(input.request.teamName); if (previousRuntimeRun?.providerId === 'opencode') { await this.stopOpenCodeRuntimeAdapterTeam(input.request.teamName, previousRuntimeRun.runId); } const previousPendingRunId = this.provisioningRunByTeam.get(input.request.teamName); const previousRuntimeProgress = previousPendingRunId ? this.runtimeAdapterProgressByRunId.get(previousPendingRunId) : null; if ( previousPendingRunId && previousRuntimeProgress && this.isCancellableRuntimeAdapterProgress(previousRuntimeProgress) ) { await this.cancelRuntimeAdapterProvisioning(previousPendingRunId, previousRuntimeProgress); } if (this.stopAllTeamsGeneration !== stopAllGenerationAtStart) { return this.recordCancelledOpenCodeRuntimeAdapterLaunch( input.request.teamName, input.sourceWarning, input.onProgress ); } const runId = randomUUID(); const startedAt = nowIso(); const initialProgress: TeamProvisioningProgress = { runId, teamName: input.request.teamName, state: 'validating', message: 'Validating OpenCode team launch gate', startedAt, updatedAt: startedAt, warnings: input.sourceWarning ? [input.sourceWarning] : undefined, }; this.provisioningRunByTeam.set(input.request.teamName, runId); this.setRuntimeAdapterProgress(initialProgress, input.onProgress); this.resetTeamScopedTransientStateForNewRun(input.request.teamName); const previousLaunchState = await this.launchStateStore.read(input.request.teamName); await this.clearPersistedLaunchState(input.request.teamName); await migrateLegacyOpenCodeRuntimeState({ teamsBasePath: getTeamsBasePath(), teamName: input.request.teamName, laneId: 'primary', }); await upsertOpenCodeRuntimeLaneIndexEntry({ teamsBasePath: getTeamsBasePath(), teamName: input.request.teamName, laneId: 'primary', state: 'active', }); const launchCwd = this.getOpenCodeRuntimeLaunchCwd(input.request.cwd, input.members); const launchInput: TeamRuntimeLaunchInput = { runId, laneId: 'primary', teamName: input.request.teamName, cwd: launchCwd, prompt: input.prompt, providerId: 'opencode', model: input.request.model, effort: input.request.effort, skipPermissions: input.request.skipPermissions !== false, expectedMembers: input.members.map((member) => ({ name: member.name, role: member.role, workflow: member.workflow, isolation: member.isolation === 'worktree' ? ('worktree' as const) : undefined, providerId: 'opencode', model: member.model ?? input.request.model, effort: member.effort ?? input.request.effort, cwd: member.cwd?.trim() || launchCwd, })), previousLaunchState, }; const launching = this.setRuntimeAdapterProgress( { ...initialProgress, state: 'spawning', message: 'Starting OpenCode sessions through runtime adapter', updatedAt: nowIso(), }, input.onProgress ); try { await setOpenCodeRuntimeActiveRunManifest({ teamsBasePath: getTeamsBasePath(), teamName: input.request.teamName, laneId: 'primary', runId, }); const launchResult = await adapter.launch(launchInput); if ( this.cancelledRuntimeAdapterRunIds.delete(runId) || this.provisioningRunByTeam.get(input.request.teamName) !== runId ) { await this.clearOpenCodeRuntimeAdapterPrimaryLaneIfOwned(input.request.teamName, runId); return { runId }; } const { result } = await this.persistOpenCodeRuntimeAdapterLaunchResult( launchResult, launchInput ); const requestTeamColor = 'color' in input.request ? input.request.color : undefined; const requestTeamDisplayName = 'displayName' in input.request ? input.request.displayName : undefined; this.syncOpenCodeRuntimeToolApprovals({ teamName: input.request.teamName, runId, laneId: 'primary', cwd: launchCwd, members: result.members, expectedMembers: launchInput.expectedMembers, teamColor: requestTeamColor, teamDisplayName: requestTeamDisplayName, }); const success = result.teamLaunchState === 'clean_success'; const pending = result.teamLaunchState === 'partial_pending'; const failed = result.teamLaunchState === 'partial_failure'; const finalProgress = this.setRuntimeAdapterProgress( { ...launching, state: success || pending ? 'ready' : 'failed', message: success ? 'OpenCode team launch is ready' : pending ? 'OpenCode team launch is waiting for runtime evidence or permissions' : 'OpenCode team launch failed readiness gate', messageSeverity: pending ? 'warning' : result.teamLaunchState === 'partial_failure' ? 'error' : undefined, updatedAt: nowIso(), warnings: result.warnings.length > 0 ? result.warnings : launching.warnings, error: result.teamLaunchState === 'partial_failure' ? result.diagnostics.join('\n') || 'OpenCode launch failed' : undefined, cliLogsTail: result.diagnostics.join('\n') || undefined, configReady: true, }, input.onProgress ); if (failed) { await clearOpenCodeRuntimeLaneStorage({ teamsBasePath: getTeamsBasePath(), teamName: input.request.teamName, laneId: 'primary', }).catch(() => undefined); this.runtimeAdapterRunByTeam.delete(input.request.teamName); this.deleteAliveRunId(input.request.teamName); this.invalidateRuntimeSnapshotCaches(input.request.teamName); } else { this.runtimeAdapterRunByTeam.set(input.request.teamName, { runId, providerId: 'opencode', cwd: launchCwd, members: result.members, }); this.setAliveRunId(input.request.teamName, runId); this.invalidateRuntimeSnapshotCaches(input.request.teamName); } if (this.provisioningRunByTeam.get(input.request.teamName) === runId) { this.provisioningRunByTeam.delete(input.request.teamName); } this.teamChangeEmitter?.({ type: 'process', teamName: input.request.teamName, runId, detail: finalProgress.state, }); return { runId }; } catch (error) { if ( this.cancelledRuntimeAdapterRunIds.delete(runId) || this.provisioningRunByTeam.get(input.request.teamName) !== runId ) { await this.clearOpenCodeRuntimeAdapterPrimaryLaneIfOwned(input.request.teamName, runId); return { runId }; } await clearOpenCodeRuntimeLaneStorage({ teamsBasePath: getTeamsBasePath(), teamName: input.request.teamName, laneId: 'primary', }).catch(() => undefined); const message = error instanceof Error ? error.message : String(error); this.setRuntimeAdapterProgress( { ...launching, state: 'failed', message: 'OpenCode runtime adapter launch failed', messageSeverity: 'error', updatedAt: nowIso(), error: message, cliLogsTail: message, }, input.onProgress ); if (this.provisioningRunByTeam.get(input.request.teamName) === runId) { this.provisioningRunByTeam.delete(input.request.teamName); } throw error; } } private async writeOpenCodeTeamConfig( request: TeamCreateRequest, members: TeamCreateRequest['members'] ): Promise { const configPath = path.join(getTeamsBasePath(), request.teamName, 'config.json'); const config: TeamConfig = { name: request.displayName?.trim() || request.teamName, description: request.description, color: request.color, projectPath: request.cwd, members: [ { name: 'team-lead', role: 'Team Lead', agentType: 'team-lead', providerId: normalizeOptionalTeamProviderId(request.providerId), model: request.model, effort: request.effort, cwd: request.cwd, }, ...members.map((member) => ({ name: member.name, role: member.role, workflow: member.workflow, isolation: member.isolation === 'worktree' ? ('worktree' as const) : undefined, providerId: normalizeOptionalTeamProviderId(member.providerId), model: member.model, effort: member.effort, mcpPolicy: normalizeTeamMemberMcpPolicy(member.mcpPolicy), cwd: member.cwd?.trim() || undefined, })), ], }; await atomicWriteAsync(configPath, `${JSON.stringify(config, null, 2)}\n`); TeamConfigReader.invalidateTeam(request.teamName); } private async persistOpenCodeRuntimeAdapterLaunchResult( result: TeamRuntimeLaunchResult, input: TeamRuntimeLaunchInput ): Promise<{ snapshot: PersistedTeamLaunchSnapshot; result: TeamRuntimeLaunchResult; }> { const committedResult = await this.commitOpenCodeRuntimeAdapterLaunchSessionEvidence({ teamName: input.teamName, laneId: input.laneId?.trim() || 'primary', result, }); const members: Record = {}; for (const member of input.expectedMembers) { const evidence = committedResult.members[member.name]; members[member.name] = this.toOpenCodePersistedLaunchMember( member, evidence, committedResult.runId ); } const snapshot = createPersistedLaunchSnapshot({ teamName: input.teamName, expectedMembers: input.expectedMembers.map((member) => member.name), bootstrapExpectedMembers: input.expectedMembers.map((member) => member.name), leadSessionId: result.leadSessionId, launchPhase: committedResult.launchPhase, members, }); return { snapshot: await this.writeLaunchStateSnapshot(input.teamName, snapshot), result: committedResult, }; } private async commitOpenCodeRuntimeAdapterLaunchSessionEvidence(params: { teamName: string; laneId: string; result: TeamRuntimeLaunchResult; }): Promise { let changed = false; const members: Record = { ...params.result.members }; for (const [memberName, evidence] of Object.entries(params.result.members)) { const runtimeSessionId = evidence.sessionId?.trim(); const confirmed = evidence.launchState === 'confirmed_alive' || evidence.bootstrapConfirmed === true || evidence.livenessKind === 'confirmed_bootstrap'; const appManagedCandidate = evidence.bootstrapEvidenceSource === 'app_managed_bootstrap' && evidence.bootstrapMode === 'app_managed_context' ? evidence.appManagedBootstrapCandidate : undefined; const appManagedCandidateMatches = appManagedCandidate?.source === 'app_managed_bootstrap' && appManagedCandidate.teamName === params.teamName && appManagedCandidate.memberName === memberName && appManagedCandidate.runId === params.result.runId && appManagedCandidate.laneId === params.laneId && appManagedCandidate.runtimeSessionId === runtimeSessionId; if ((!confirmed && !appManagedCandidateMatches) || !runtimeSessionId) { continue; } // For app-managed bootstrap, promotion is intentionally two-phase: // write the candidate as runtime evidence, then verify it using the same // reader path used by later reconciliation/restart flows. const source: OpenCodeBootstrapEvidenceSource = appManagedCandidateMatches ? 'app_managed_bootstrap' : (evidence.bootstrapEvidenceSource ?? 'runtime_bootstrap_checkin'); await this.commitOpenCodeRuntimeBootstrapSessionEvidence({ teamName: params.teamName, runId: params.result.runId, laneId: params.laneId, memberName, runtimeSessionId, observedAt: nowIso(), source, appManagedBootstrapCandidate: appManagedCandidateMatches ? appManagedCandidate : evidence.appManagedBootstrapCandidate, }); const verified = await this.hasCommittedOpenCodeRuntimeBootstrapSessionEvidence({ teamName: params.teamName, runId: params.result.runId, laneId: params.laneId, memberName, runtimeSessionId, source, appManagedBootstrapCandidate: appManagedCandidateMatches ? appManagedCandidate : evidence.appManagedBootstrapCandidate, }); if (appManagedCandidateMatches && verified && !confirmed) { members[memberName] = promoteCommittedOpenCodeAppManagedBootstrapEvidence(evidence); changed = true; } } if (!changed) { return params.result; } const teamLaunchState = summarizeRuntimeLaunchResultMembers(members); return { ...params.result, launchPhase: teamLaunchState === 'clean_success' ? 'finished' : params.result.launchPhase, teamLaunchState, members, diagnostics: appendDiagnosticOnce( params.result.diagnostics, 'OpenCode app-managed bootstrap evidence was committed and read back before readiness promotion.' ), }; } private toOpenCodePersistedLaunchMember( member: TeamRuntimeLaunchInput['expectedMembers'][number], evidence: TeamRuntimeMemberLaunchEvidence | undefined, runId?: string ): PersistedTeamLaunchMemberState { const now = nowIso(); const launchState = evidence?.launchState ?? 'failed_to_start'; const hardFailure = evidence?.hardFailure === true || launchState === 'failed_to_start'; return { name: member.name, providerId: 'opencode', providerBackendId: undefined, model: member.model?.trim() || evidence?.model?.trim() || undefined, effort: member.effort, cwd: member.cwd?.trim() || undefined, laneId: 'primary', laneKind: 'primary', laneOwnerProviderId: 'opencode', launchState, agentToolAccepted: evidence?.agentToolAccepted === true, runtimeAlive: evidence?.runtimeAlive === true, bootstrapConfirmed: evidence?.bootstrapConfirmed === true, hardFailure, hardFailureReason: hardFailure ? evidence?.hardFailureReason : undefined, pendingPermissionRequestIds: evidence?.pendingPermissionRequestIds?.length ? [...new Set(evidence.pendingPermissionRequestIds)] : undefined, ...(evidence?.runtimePid ? { runtimePid: evidence.runtimePid } : {}), ...(evidence?.sessionId ? { runtimeSessionId: evidence.sessionId } : {}), ...(evidence?.sessionId ? { runtimeRunId: evidence.appManagedBootstrapCandidate?.runId ?? runId } : {}), ...(evidence?.bootstrapEvidenceSource ? { bootstrapEvidenceSource: evidence.bootstrapEvidenceSource } : {}), ...(evidence?.bootstrapMode ? { bootstrapMode: evidence.bootstrapMode } : {}), ...(evidence?.appManagedBootstrapCandidate ? { appManagedBootstrapCandidate: evidence.appManagedBootstrapCandidate } : {}), ...(evidence?.livenessKind ? { livenessKind: evidence.livenessKind } : {}), ...(evidence?.pidSource ? { pidSource: evidence.pidSource } : {}), ...(evidence?.runtimeDiagnostic ? { runtimeDiagnostic: evidence.runtimeDiagnostic } : {}), ...(evidence?.runtimeDiagnosticSeverity ? { runtimeDiagnosticSeverity: evidence.runtimeDiagnosticSeverity } : evidence?.runtimeDiagnostic ? { runtimeDiagnosticSeverity: 'info' as const } : {}), ...(evidence?.runtimeAlive ? { runtimeLastSeenAt: now } : {}), firstSpawnAcceptedAt: evidence?.agentToolAccepted ? now : undefined, lastHeartbeatAt: evidence?.bootstrapConfirmed ? now : undefined, lastRuntimeAliveAt: evidence?.runtimeAlive ? now : undefined, lastEvaluatedAt: now, sources: { processAlive: evidence?.runtimeAlive === true, nativeHeartbeat: evidence?.bootstrapConfirmed === true, }, diagnostics: evidence?.diagnostics, }; } async launchTeam( request: TeamLaunchRequest, onProgress: (progress: TeamProvisioningProgress) => void ): Promise { return this.withTeamLock(request.teamName, async () => { return this._launchTeamInner(request, onProgress); }); } private async _launchTeamInner( request: TeamLaunchRequest, onProgress: (progress: TeamProvisioningProgress) => void ): Promise { const existingProvisioningRunId = this.getResolvableProvisioningRunId(request.teamName); if (existingProvisioningRunId) { return { runId: existingProvisioningRunId }; } const stopAllGenerationAtStart = this.stopAllTeamsGeneration; assertAppDeterministicBootstrapEnabled(); if (this.shouldRouteOpenCodeToRuntimeAdapter(request)) { return this.launchOpenCodeTeamThroughRuntimeAdapter(request, onProgress); } assertOpenCodeNotLaunchedThroughLegacyProvisioning(request); // Set immediately to prevent TOCTOU (defense in depth alongside withTeamLock) const pendingKey = `pending-${randomUUID()}`; this.provisioningRunByTeam.set(request.teamName, pendingKey); try { // Verify config.json exists — team must already be provisioned const configPath = path.join(getTeamsBasePath(), request.teamName, 'config.json'); const configRaw = await tryReadRegularFileUtf8(configPath, { timeoutMs: TEAM_JSON_READ_TIMEOUT_MS, maxBytes: TEAM_CONFIG_MAX_BYTES, }); if (!configRaw) { throw new Error(`Team "${request.teamName}" not found — config.json does not exist`); } let configProjectPath: string | null = null; try { const parsedConfig = JSON.parse(configRaw) as { projectPath?: unknown }; configProjectPath = typeof parsedConfig.projectPath === 'string' && parsedConfig.projectPath.trim().length > 0 ? path.resolve(parsedConfig.projectPath.trim()) : null; } catch { configProjectPath = null; } const existingAliveRunId = this.getAliveRunId(request.teamName); if (existingAliveRunId) { const existingRun = this.runs.get(existingAliveRunId); const requestedCwd = path.resolve(request.cwd); const existingRunCwd = this.getRunTrackedCwd(existingRun) ?? configProjectPath; if (existingRun?.child && !existingRun.processKilled && !existingRun.cancelRequested) { if (!existingRunCwd) { this.provisioningRunByTeam.delete(request.teamName); throw new Error( `Team "${request.teamName}" is already running, but its cwd could not be determined. ` + 'Stop it before launching again.' ); } if (existingRunCwd && existingRunCwd !== requestedCwd) { this.provisioningRunByTeam.delete(request.teamName); throw new Error( `Team "${request.teamName}" is already running in "${existingRunCwd}". ` + `Stop it before launching with cwd "${request.cwd}".` ); } this.provisioningRunByTeam.delete(request.teamName); return { runId: existingAliveRunId }; } } const launchCompatibility = await this.probeLaunchCompatibility( request.teamName, configRaw, request.providerId ); if (launchCompatibility.level === 'unsafe') { this.provisioningRunByTeam.delete(request.teamName); throw new Error(launchCompatibility.blockers[0] ?? getMixedLaunchFallbackRecoveryError()); } if (launchCompatibility.repairAction === 'materialize-members-meta') { await this.materializeLaunchCompatibilityRepair(request, launchCompatibility); } const { members: expectedMemberSpecs, source, warning, } = this.resolveLaunchExpectedMembersFromCompatibility(launchCompatibility); assertOpenCodeNotLaunchedThroughLegacyProvisioning({ providerId: request.providerId, members: expectedMemberSpecs, }); // Deterministic launch always sends --team-bootstrap-spec. The orchestrator // rejects combining that startup mode with --resume, so relaunch starts a // fresh lead runtime session and restores operational context from durable // team state instead of the previous transcript. if (request.clearContext) { logger.info( `[${request.teamName}] clearContext requested - starting fresh deterministic bootstrap session` ); } else { logger.info( `[${request.teamName}] Starting fresh deterministic bootstrap session because ` + `--team-bootstrap-spec cannot be combined with --resume` ); } // IMPORTANT: The CLI auto-suffixes teammate names when they already exist in config.json. // Normalize config.json to keep only the team-lead before spawning the CLI, so we get stable names. try { await this.normalizeTeamConfigForLaunch(request.teamName, configRaw); await this.assertConfigLeadOnlyForLaunch(request.teamName); // Update projectPath in config IMMEDIATELY so TeamDetailView shows the correct path // even if provisioning is interrupted or the user stops the team early. // If launch fails, restorePrelaunchConfig() will revert to the backup (old projectPath). await this.updateConfigProjectPath(request.teamName, request.cwd); } catch (error) { // Restore pre-launch backup so config.json is not left in normalized (lead-only) state. await this.restorePrelaunchConfig(request.teamName); throw error; } let claudePath: string | null; try { await ensureCwdExists(request.cwd); claudePath = await ClaudeBinaryResolver.resolve(); if (!claudePath) { throw buildMissingCliError(); } } catch (error) { // Restore pre-launch backup so config.json is not left in normalized (lead-only) state await this.restorePrelaunchConfig(request.teamName); throw error; } const teamsBasePathsToProbe = getTeamsBasePathsToProbe(); const runId = randomUUID(); const startedAt = nowIso(); const teamRuntimeAuth: TeamRuntimeAuthContext = { teamName: request.teamName, authMaterialId: runId, allowAnthropicApiKeyHelper: true, }; const provisioningEnv = await this.buildProvisioningEnv( request.providerId, request.providerBackendId, { includeCodexTeammateAuth: teamRequestIncludesCodexMember(request), teamRuntimeAuth } ); const { env: shellEnv, geminiRuntimeAuth, providerArgs = [], warning: envWarning, } = provisioningEnv; if (envWarning) { throw new Error(envWarning); } const workspaceTrustFeatureFlags = resolveWorkspaceTrustFeatureFlags(); const workspaceTrustProviders = workspaceTrustFeatureFlags.enabled ? this.collectWorkspaceTrustProviders({ leadProviderId: request.providerId, members: expectedMemberSpecs, }) : []; const workspaceTrustEarlyWorkspaces = workspaceTrustFeatureFlags.enabled ? await this.collectWorkspaceTrustWorkspaces({ cwd: request.cwd, members: [], }) : []; const workspaceTrustEarlyPlan = workspaceTrustFeatureFlags.enabled ? await this.planWorkspaceTrustArgsOnlySafely({ providers: workspaceTrustProviders, workspaces: workspaceTrustEarlyWorkspaces, targetSurfaces: ['default_model_probe'], featureFlags: workspaceTrustFeatureFlags, }) : { launchArgPatches: [] }; const workspaceTrustProviderArgsResolver = this.createDefaultModelWorkspaceTrustProviderArgsResolver(workspaceTrustEarlyPlan); const materializedMemberSpecs = await this.materializeEffectiveTeamMemberSpecs({ claudePath, cwd: request.cwd, members: expectedMemberSpecs, defaults: { providerId: request.providerId, model: request.model, effort: request.effort, }, primaryProviderId: request.providerId, primaryEnv: provisioningEnv, teamRuntimeAuth, limitContext: request.limitContext, providerArgsResolver: workspaceTrustProviderArgsResolver, }); const allEffectiveMemberSpecs = await this.resolveOpenCodeMemberWorkspacesForRuntime({ teamName: request.teamName, baseCwd: request.cwd, leadProviderId: request.providerId, members: materializedMemberSpecs, }); Object.assign( shellEnv, await this.buildRuntimeTurnSettledEnvironmentForMembers( request.providerId, allEffectiveMemberSpecs ) ); const lanePlan = this.planRuntimeLanesOrThrow(request.providerId, allEffectiveMemberSpecs); const primaryMemberNames = new Set(lanePlan.primaryMembers.map((member) => member.name)); const effectiveMemberSpecs = allEffectiveMemberSpecs.filter((member) => primaryMemberNames.has(member.name) ); assertDeterministicBootstrapPrimaryMemberLimit(effectiveMemberSpecs.length); const largeTeamWarning = buildLargeDeterministicBootstrapWarning(effectiveMemberSpecs.length); const initialLaunchWarnings = [warning, largeTeamWarning].filter((value): value is string => Boolean(value) ); const expectedMembers = effectiveMemberSpecs.map((member) => member.name); const resolvedProviderId = resolveTeamProviderId(request.providerId); const crossProviderMemberArgs = await this.buildCrossProviderMemberArgs( resolvedProviderId, effectiveMemberSpecs, { teamRuntimeAuth } ); const workspaceTrustFullWorkspaces = workspaceTrustFeatureFlags.enabled ? await this.collectWorkspaceTrustWorkspaces({ cwd: request.cwd, members: allEffectiveMemberSpecs, }) : []; const workspaceTrustFullPlan = workspaceTrustFeatureFlags.enabled ? await this.planWorkspaceTrustFullSafely({ providers: this.collectWorkspaceTrustProviders({ leadProviderId: request.providerId, members: allEffectiveMemberSpecs, }), workspaces: workspaceTrustFullWorkspaces, featureFlags: workspaceTrustFeatureFlags, }) : null; const workspaceTrustPatches = workspaceTrustFullPlan?.launchArgPatches ?? []; const providerArgsForLaunch = this.applyWorkspaceTrustArgPatches({ args: providerArgs, patches: workspaceTrustPatches, targetProvider: resolvedProviderId, targetSurface: 'primary_provider_args', }); const crossProviderArgsForLaunch = crossProviderMemberArgs.providerArgsByProvider.has('codex') ? this.applyWorkspaceTrustArgPatches({ args: crossProviderMemberArgs.args, patches: workspaceTrustPatches, targetProvider: 'codex', targetSurface: 'cross_provider_member_args', }) : crossProviderMemberArgs.args; const crossProviderMemberArgsForLaunch = { ...crossProviderMemberArgs, args: crossProviderArgsForLaunch, }; Object.assign(shellEnv, crossProviderMemberArgs.envPatch); if (crossProviderMemberArgs.usesAnthropicApiKeyHelper) { for (const key of ANTHROPIC_HELPER_MODE_COMPETING_AUTH_ENV_KEYS) { delete shellEnv[key]; } } const providerArgsByProvider = new Map(); for (const [providerId, args] of new Map([ [resolvedProviderId, providerArgsForLaunch], ...crossProviderMemberArgs.providerArgsByProvider, ])) { providerArgsByProvider.set( providerId, this.applyWorkspaceTrustArgPatches({ args, patches: workspaceTrustPatches, targetProvider: providerId, targetSurface: 'provider_facts_probe', }) ); } const launchIdentity = await this.resolveAndValidateLaunchIdentity({ claudePath, cwd: request.cwd, env: shellEnv, request, effectiveMembers: effectiveMemberSpecs, providerArgsByProvider, }); // Build a synthetic TeamCreateRequest for reuse by shared infrastructure const syntheticRequest: TeamCreateRequest = { teamName: request.teamName, members: allEffectiveMemberSpecs, cwd: request.cwd, providerId: request.providerId, providerBackendId: request.providerBackendId, model: request.model, effort: request.effort, fastMode: request.fastMode, skipPermissions: request.skipPermissions, }; // Enrich with color/displayName from config.json (always available for launched teams) try { const cfg = JSON.parse(configRaw) as Record; if (typeof cfg.color === 'string' && cfg.color.trim().length > 0) { syntheticRequest.color = cfg.color.trim(); } if (typeof cfg.name === 'string' && cfg.name.trim().length > 0) { syntheticRequest.displayName = cfg.name.trim(); } } catch { // config already validated above — ignore parse errors here } const run: ProvisioningRun = { runId, teamName: request.teamName, startedAt, stdoutBuffer: '', stderrBuffer: '', claudeLogLines: [], lastClaudeLogStream: null, stdoutLogLineBuf: '', stderrLogLineBuf: '', stdoutParserCarry: '', stdoutParserCarryIsCompleteJson: false, stdoutParserCarryLooksLikeClaudeJson: false, claudeLogsUpdatedAt: undefined, deterministicBootstrapStartedAt: undefined, lastDeterministicBootstrapEvent: undefined, lastDeterministicBootstrapPhase: undefined, deterministicBootstrapMemberSpawnSeen: false, deterministicBootstrapMemberResultSeen: false, processKilled: false, finalizingByTimeout: false, cancelRequested: false, teamsBasePathsToProbe, child: null, timeoutHandle: null, fsMonitorHandle: null, onProgress, expectedMembers, request: syntheticRequest, allEffectiveMembers: allEffectiveMemberSpecs, effectiveMembers: effectiveMemberSpecs, launchIdentity, mixedSecondaryLanes: this.createMixedSecondaryLaneStates(lanePlan), lastLogProgressAt: 0, lastDataReceivedAt: 0, // intentionally 0 — real reset happens after spawn (see startStallWatchdog call sites) lastStdoutReceivedAt: 0, stallCheckHandle: null, stallWarningIndex: null, preStallMessage: null, lastRetryAt: 0, apiRetryWarningIndex: null, apiErrorWarningEmitted: false, waitingTasksSince: null, provisioningComplete: false, processClosed: false, requiresFirstRealTurnSuccess: false, firstRealTurnSucceeded: false, mcpConfigPath: null, memberMcpConfigPaths: [], bootstrapSpecPath: null, bootstrapUserPromptPath: null, isLaunch: true, launchStateClearedForRun: false, deterministicBootstrap: true, workspaceTrustPlan: workspaceTrustFullPlan, workspaceTrustExecution: null, workspaceTrustDiagnostics: null, workspaceTrustRetryAttempted: false, fsPhase: 'waiting_members', leadRelayCapture: null, activeCrossTeamReplyHints: [], leadMsgSeq: 0, liveLeadTextBuffer: null, pendingToolCalls: [], activeToolCalls: new Map(), pendingDirectCrossTeamSendRefresh: false, lastLeadTextEmitMs: 0, silentUserDmForward: null, silentUserDmForwardClearHandle: null, pendingInboxRelayCandidates: [], provisioningOutputParts: [], provisioningTraceLines: [], lastProvisioningTraceKey: null, provisioningOutputIndexByMessageId: new Map(), detectedSessionId: null, leadActivityState: 'active', leadContextUsage: null, authFailureRetried: false, authRetryInProgress: false, spawnContext: null, anthropicApiKeyHelper: provisioningEnv.anthropicApiKeyHelper ?? null, pendingApprovals: new Map(), processedPermissionRequestIds: new Set(), pendingPostCompactReminder: false, postCompactReminderInFlight: false, suppressPostCompactReminderOutput: false, pendingGeminiPostLaunchHydration: false, geminiPostLaunchHydrationInFlight: false, geminiPostLaunchHydrationSent: false, suppressGeminiPostLaunchHydrationOutput: false, memberSpawnStatuses: new Map( expectedMembers.map((name) => [name, createInitialMemberSpawnStatusEntry()]) ), memberSpawnToolUseIds: new Map(), pendingMemberRestarts: new Map(), memberSpawnLeadInboxCursorByMember: new Map(), lastDeterministicBootstrapSeq: 0, lastMemberSpawnAuditAt: 0, lastMemberSpawnAuditConfigReadWarningAt: 0, lastMemberSpawnAuditMissingWarningAt: new Map(), progress: { runId, teamName: request.teamName, state: 'validating', message: source === 'members-meta' ? 'Validating team launch request (members from members.meta.json)' : source === 'inboxes' ? 'Validating team launch request (members from inboxes)' : 'Validating team launch request (fallback members from config.json)', startedAt, updatedAt: startedAt, warnings: initialLaunchWarnings.length > 0 ? initialLaunchWarnings : undefined, cliLogsTail: undefined, }, }; this.resetTeamScopedTransientStateForNewRun(request.teamName); this.runs.set(runId, run); this.provisioningRunByTeam.set(request.teamName, runId); initializeProvisioningTrace(run); run.onProgress(run.progress); await this.prepareWorkspaceTrustForDeterministicRun({ mode: 'launch', run, claudePath, shellEnv, stopAllGenerationAtStart, workspaceTrustPlan: workspaceTrustFullPlan, featureFlags: workspaceTrustFeatureFlags, provisioningEnv, }); emitProvisioningCheckpoint(run, 'Clearing persisted launch state'); await this.clearPersistedLaunchState(request.teamName, { expectedRunId: run.runId }); run.launchStateClearedForRun = true; emitProvisioningCheckpoint(run, 'Publishing mixed secondary lane status'); for (const lane of run.mixedSecondaryLanes ?? []) { await this.publishMixedSecondaryLaneStatusChange(run, lane); } // Read existing tasks to include in teammate prompts for work resumption emitProvisioningCheckpoint(run, 'Reading existing tasks for launch prompt'); const taskReader = new TeamTaskReader(); let existingTasks: TeamTask[] = []; try { existingTasks = await taskReader.getTasks(request.teamName); } catch (error) { logger.warn( `[${request.teamName}] Failed to read tasks for launch prompt: ${String(error)}` ); } const prompt = buildDeterministicLaunchHydrationPrompt( request, effectiveMemberSpecs, existingTasks, false ); const promptSize = getPromptSizeSummary(prompt); let child: ReturnType; shellEnv.CLAUDE_ENABLE_DETERMINISTIC_TEAM_BOOTSTRAP = '1'; const teammateModeDecision = await resolveDesktopTeammateModeDecision(request.extraCliArgs); applyDesktopTeammateModeDecisionToEnv(shellEnv, teammateModeDecision); let mcpConfigPath: string; let bootstrapSpecPath: string; let bootstrapUserPromptPath: string | null = null; try { emitProvisioningCheckpoint( run, 'Building deterministic launch bootstrap spec', `expectedMembers=${effectiveMemberSpecs.length}` ); const nativeBootstrapBuild = await buildNativeAppManagedBootstrapSpecsWithDiagnostics({ teamName: request.teamName, cwd: request.cwd, members: effectiveMemberSpecs, }); const memberMcpLaunchConfigs = await this.buildRuntimeBootstrapMemberMcpLaunchConfigs({ controlApiBaseUrl: provisioningEnv.env.CLAUDE_TEAM_CONTROL_URL, cwd: request.cwd, members: effectiveMemberSpecs, run, }); if (nativeBootstrapBuild.diagnostics.warning) { run.progress = { ...run.progress, warnings: mergeProvisioningWarnings( run.progress.warnings, nativeBootstrapBuild.diagnostics.warning ), }; emitProvisioningCheckpoint( run, 'Native bootstrap startup context is large', nativeBootstrapBuild.diagnostics.warning ); } const bootstrapSpec = buildDeterministicLaunchBootstrapSpec( runId, request, effectiveMemberSpecs, nativeBootstrapBuild.specs, memberMcpLaunchConfigs ); emitProvisioningCheckpoint(run, 'Writing deterministic bootstrap spec file'); bootstrapSpecPath = await writeDeterministicBootstrapSpecFile(bootstrapSpec); run.bootstrapSpecPath = bootstrapSpecPath; emitProvisioningCheckpoint( run, 'Writing launch hydration prompt file', `chars=${promptSize.chars} lines=${promptSize.lines}` ); bootstrapUserPromptPath = await writeDeterministicBootstrapUserPromptFile(prompt); run.bootstrapUserPromptPath = bootstrapUserPromptPath; run.requiresFirstRealTurnSuccess = true; emitProvisioningCheckpoint(run, 'Writing MCP config file'); mcpConfigPath = await this.mcpConfigBuilder.writeConfigFile(request.cwd, { controlApiBaseUrl: provisioningEnv.env.CLAUDE_TEAM_CONTROL_URL, }); run.mcpConfigPath = mcpConfigPath; emitProvisioningCheckpoint(run, 'Validating agent-teams MCP runtime'); await this.validateAgentTeamsMcpRuntime(claudePath, request.cwd, shellEnv, mcpConfigPath, { isCancelled: () => run.cancelRequested || run.processKilled || this.stopAllTeamsGeneration !== stopAllGenerationAtStart, }); } catch (error) { this.runs.delete(runId); this.provisioningRunByTeam.delete(request.teamName); if (provisioningEnv.anthropicApiKeyHelper) { await cleanupAnthropicTeamApiKeyHelperMaterial({ directory: provisioningEnv.anthropicApiKeyHelper.directory, }).catch(() => undefined); } await removeDeterministicBootstrapSpecFile(run.bootstrapSpecPath).catch(() => {}); run.bootstrapSpecPath = null; await removeDeterministicBootstrapUserPromptFile(run.bootstrapUserPromptPath).catch( () => {} ); run.bootstrapUserPromptPath = null; if (run.mcpConfigPath) { await this.mcpConfigBuilder.removeConfigFile(run.mcpConfigPath).catch(() => {}); run.mcpConfigPath = null; } await this.removeRunMemberMcpConfigFiles(run).catch(() => {}); await this.restorePrelaunchConfig(request.teamName); throw error; } const launchArgs = [ '--print', '--input-format', 'stream-json', '--output-format', 'stream-json', '--verbose', '--setting-sources', 'user,project,local', '--mcp-config', mcpConfigPath, '--team-bootstrap-spec', bootstrapSpecPath, ...(bootstrapUserPromptPath ? ['--team-bootstrap-user-prompt-file', bootstrapUserPromptPath] : []), '--disallowedTools', APP_TEAM_RUNTIME_DISALLOWED_TOOLS, // Explicit --permission-mode overrides user's defaultMode in ~/.claude/settings.json // (e.g. "acceptEdits") which otherwise takes precedence over CLI flags ...(request.skipPermissions !== false ? ['--dangerously-skip-permissions', '--permission-mode', 'bypassPermissions'] : ['--permission-prompt-tool', 'stdio', '--permission-mode', 'default']), ]; const launchModelArg = getLaunchModelArg( resolveTeamProviderId(request.providerId), request.model, launchIdentity ); const extraCliArgs = parseCliArgs(request.extraCliArgs); const runtimeArgsPlan = await this.buildTeamRuntimeLaunchArgsPlan({ teamName: request.teamName, providerId: resolvedProviderId, launchIdentity, envResolution: { ...provisioningEnv, providerArgs: providerArgsForLaunch }, extraArgs: extraCliArgs, inheritedProviderArgs: crossProviderMemberArgsForLaunch.args, includeAnthropicHelper: resolvedProviderId === 'anthropic', contextLabel: 'Team launch', }); if (launchModelArg) { launchArgs.push('--model', launchModelArg); } if (launchIdentity.resolvedEffort) { launchArgs.push('--effort', launchIdentity.resolvedEffort); } launchArgs.push(...runtimeArgsPlan.fastModeArgs); launchArgs.push(...runtimeArgsPlan.runtimeTurnSettledHookArgs); if (request.worktree) { launchArgs.push('--worktree', request.worktree); } launchArgs.push(...buildDesktopTeammateModeCliArgs(teammateModeDecision)); launchArgs.push(...runtimeArgsPlan.extraArgs); launchArgs.push(...runtimeArgsPlan.providerArgs); launchArgs.push(...runtimeArgsPlan.settingsArgs); // When the lead uses a different provider than some teammates (e.g., anthropic lead // with codex teammates), the lead needs the teammate provider's launch args so they // can be inherited by the teammate subprocess via buildInheritedCliFlags. // Without this, a codex teammate spawned from an anthropic lead has no way to learn // about the required forced_login_method (chatgpt/api) and fails to start. emitProvisioningCheckpoint(run, 'Resolving cross-provider member launch args'); launchArgs.push(...runtimeArgsPlan.inheritedProviderArgs); const finalLaunchArgs = mergeJsonSettingsArgs(launchArgs); applyAppManagedRuntimeSettingsPathEnv(shellEnv, runtimeArgsPlan.appManagedSettingsPath); const runtimeWarning = buildRuntimeLaunchWarning(request, shellEnv, { geminiRuntimeAuth, promptSize, expectedMembersCount: effectiveMemberSpecs.length, }); logRuntimeLaunchSnapshot( logger, request.teamName, claudePath, finalLaunchArgs, request, shellEnv, { geminiRuntimeAuth, promptSize, expectedMembersCount: effectiveMemberSpecs.length, launchIdentity, } ); // Deterministic bootstrap launches fresh because --team-bootstrap-spec and // --resume are not a supported orchestrator combination. emitProvisioningCheckpoint(run, 'Persisting team metadata before spawn'); await this.teamMetaStore.writeMeta(request.teamName, { displayName: syntheticRequest.displayName, description: syntheticRequest.description, color: syntheticRequest.color, cwd: request.cwd, prompt: request.prompt, providerId: request.providerId, providerBackendId: request.providerBackendId, model: request.model, effort: request.effort, fastMode: request.fastMode, skipPermissions: request.skipPermissions, worktree: request.worktree, extraCliArgs: request.extraCliArgs, limitContext: request.limitContext, launchIdentity, createdAt: Date.now(), }); await this.membersMetaStore.writeMembers( request.teamName, this.buildMembersMetaWritePayload(allEffectiveMemberSpecs), { providerBackendId: request.providerBackendId, } ); try { if ( run.cancelRequested || run.processKilled || this.stopAllTeamsGeneration !== stopAllGenerationAtStart ) { throw new Error('Team launch cancelled by app shutdown'); } if (request.skipPermissions === false) { emitProvisioningCheckpoint(run, 'Seeding lead bootstrap permission rules'); await this.seedLeadBootstrapPermissionRules(request.teamName, request.cwd); } emitProvisioningCheckpoint( run, 'Spawning Claude CLI process for team launch', `args=${finalLaunchArgs.length} cwd=${request.cwd}` ); child = spawnCli(claudePath, finalLaunchArgs, { cwd: request.cwd, env: { ...shellEnv }, stdio: ['pipe', 'pipe', 'pipe'], }); } catch (error) { if (run.mcpConfigPath) { await this.mcpConfigBuilder.removeConfigFile(run.mcpConfigPath).catch(() => {}); run.mcpConfigPath = null; } await removeDeterministicBootstrapSpecFile(run.bootstrapSpecPath).catch(() => {}); run.bootstrapSpecPath = null; await removeDeterministicBootstrapUserPromptFile(run.bootstrapUserPromptPath).catch( () => {} ); run.bootstrapUserPromptPath = null; await this.removeRunMemberMcpConfigFiles(run).catch(() => {}); if (provisioningEnv.anthropicApiKeyHelper) { await cleanupAnthropicTeamApiKeyHelperMaterial({ directory: provisioningEnv.anthropicApiKeyHelper.directory, }).catch(() => undefined); } this.runs.delete(runId); this.provisioningRunByTeam.delete(request.teamName); await this.restorePrelaunchConfig(request.teamName); throw error; } updateProgress(run, 'spawning', 'Starting Claude CLI process for team launch', { pid: child.pid ?? undefined, warnings: mergeProvisioningWarnings(run.progress.warnings, runtimeWarning), }); run.onProgress(run.progress); run.child = child; run.processClosed = false; run.spawnContext = { claudePath, args: finalLaunchArgs, cwd: request.cwd, env: { ...shellEnv }, prompt, }; this.attachStdoutHandler(run); this.attachStderrHandler(run); // Reset AFTER spawn — not at run init — because async operations between init // and spawn can take seconds, causing false stall warnings. run.lastDataReceivedAt = Date.now(); run.lastStdoutReceivedAt = Date.now(); this.startStallWatchdog(run); // For launch, skip the filesystem monitor — files (config, inboxes, tasks) // already exist from the previous run and would trigger immediate false // completion on the first poll. Rely on stream-json result.success instead. updateProgress(run, 'configuring', 'CLI running - deterministic launch in progress'); run.onProgress(run.progress); run.timeoutHandle = setTimeout(() => { if (!run.processKilled && !run.provisioningComplete) { run.processKilled = true; run.finalizingByTimeout = true; void (async () => { const readyOnTimeout = await this.tryCompleteAfterTimeout(run); killTeamProcess(run.child); if (readyOnTimeout) { return; } const progress = updateProgress(run, 'failed', 'Timed out waiting for CLI (launch)', { error: 'Timed out waiting for CLI during team launch.', cliLogsTail: extractCliLogsFromRun(run), }); run.onProgress(progress); this.cleanupRun(run); })(); } }, getProvisioningRunTimeoutMs(run)); child.once('error', (error) => { const progress = updateProgress(run, 'failed', 'Failed to start Claude CLI (launch)', { error: error.message, cliLogsTail: extractCliLogsFromRun(run), }); run.onProgress(progress); this.cleanupRun(run); }); child.once('close', (code) => { void this.handleProcessExit(run, code); }); return { runId }; } catch (error) { // Clean up pending key if failure occurred before runId was set if (this.provisioningRunByTeam.get(request.teamName) === pendingKey) { this.provisioningRunByTeam.delete(request.teamName); } throw error; } } async getProvisioningStatus(runId: string): Promise { const run = this.runs.get(runId); if (run) { return run.progress; } const runtimeProgress = this.runtimeAdapterProgressByRunId.get(runId); if (runtimeProgress) { return runtimeProgress; } const retainedProgress = this.getRetainedProvisioningProgressMap().get(runId); if (retainedProgress) { return retainedProgress; } throw new Error('Unknown runId'); } private getRetainedProvisioningProgressMap(): Map { this.retainedProvisioningProgressByRunId ??= new Map(); return this.retainedProvisioningProgressByRunId; } private getRetainedProvisioningProgressTimersMap(): Map> { this.retainedProvisioningProgressTimersByRunId ??= new Map< string, ReturnType >(); return this.retainedProvisioningProgressTimersByRunId; } private retainProvisioningProgress(runId: string, progress: TeamProvisioningProgress): void { const retainedProgress = this.getRetainedProvisioningProgressMap(); const retainedTimers = this.getRetainedProvisioningProgressTimersMap(); const previousTimer = retainedTimers.get(runId); if (previousTimer) { clearTimeout(previousTimer); } retainedProgress.set(runId, { ...progress, warnings: progress.warnings ? [...progress.warnings] : undefined, launchDiagnostics: progress.launchDiagnostics ? [...progress.launchDiagnostics] : undefined, }); const timer = setTimeout(() => { retainedProgress.delete(runId); retainedTimers.delete(runId); }, TeamProvisioningService.RETAINED_PROVISIONING_PROGRESS_TTL_MS); timer.unref?.(); retainedTimers.set(runId, timer); } async cancelProvisioning(runId: string): Promise { const run = this.runs.get(runId); if (!run) { const runtimeProgress = this.runtimeAdapterProgressByRunId.get(runId); if (runtimeProgress) { await this.cancelRuntimeAdapterProvisioning(runId, runtimeProgress); return; } throw new Error('Unknown runId'); } if ( !['spawning', 'configuring', 'assembling', 'finalizing', 'verifying'].includes( run.progress.state ) ) { throw new Error('Provisioning cannot be cancelled in current state'); } run.cancelRequested = true; run.processKilled = true; // SIGKILL: newer Claude CLI versions handle SIGTERM gracefully and delete // team files during cleanup. SIGKILL is uncatchable — files are preserved. killTeamProcess(run.child); if ( this.getTrackedRunId(run.teamName) === run.runId && this.hasSecondaryRuntimeRuns(run.teamName) ) { void this.stopMixedSecondaryRuntimeLanes(run.teamName); } const progress = updateProgress(run, 'cancelled', 'Provisioning cancelled by user'); run.onProgress(progress); this.cleanupRun(run); } private isCancellableRuntimeAdapterProgress(progress: TeamProvisioningProgress): boolean { return [ 'validating', 'spawning', 'configuring', 'assembling', 'finalizing', 'verifying', ].includes(progress.state); } private async cancelRuntimeAdapterProvisioning( runId: string, runtimeProgress: TeamProvisioningProgress ): Promise { if (!this.isCancellableRuntimeAdapterProgress(runtimeProgress)) { throw new Error('Provisioning cannot be cancelled in current state'); } const teamName = runtimeProgress.teamName; const runtimeRun = this.runtimeAdapterRunByTeam.get(teamName); this.cancelledRuntimeAdapterRunIds.add(runId); this.clearOpenCodeRuntimeToolApprovals(teamName, { runId, laneId: 'primary', emitDismiss: true, }); this.runtimeAdapterRunByTeam.delete(teamName); this.deleteAliveRunId(teamName); if (this.provisioningRunByTeam.get(teamName) === runId) { this.provisioningRunByTeam.delete(teamName); } this.invalidateRuntimeSnapshotCaches(teamName); this.setRuntimeAdapterProgress({ ...runtimeProgress, state: 'cancelled', message: 'Provisioning cancelled by user', updatedAt: nowIso(), }); this.teamChangeEmitter?.({ type: 'process', teamName, runId, detail: 'cancelled', }); const previousLaunchState = await this.launchStateStore.read(teamName); const adapter = this.getOpenCodeRuntimeAdapter(); if (adapter) { try { await adapter.stop({ runId, laneId: 'primary', teamName, cwd: runtimeRun?.cwd ?? this.readPersistedTeamProjectPath(teamName) ?? undefined, providerId: 'opencode', reason: 'user_requested', previousLaunchState, force: true, }); } catch (error) { logger.warn( `[${teamName}] Failed to stop OpenCode runtime adapter launch during cancel: ${ error instanceof Error ? error.message : String(error) }` ); } } await clearOpenCodeRuntimeLaneStorage({ teamsBasePath: getTeamsBasePath(), teamName, laneId: 'primary', }).catch(() => undefined); } private getPendingRuntimeAdapterLaunchesForShutdown(): TeamProvisioningProgress[] { return Array.from(this.runtimeAdapterProgressByRunId.values()).filter((progress) => this.isCancellableRuntimeAdapterProgress(progress) ); } private async clearOpenCodeRuntimeAdapterPrimaryLaneIfOwned( teamName: string, runId: string ): Promise { const currentProvisioningRunId = this.provisioningRunByTeam.get(teamName); const currentAliveRunId = this.aliveRunByTeam.get(teamName); const currentRuntimeRun = this.runtimeAdapterRunByTeam.get(teamName); const ownsPrimaryLane = currentProvisioningRunId === runId || currentAliveRunId === runId || currentRuntimeRun?.runId === runId || (!currentProvisioningRunId && !currentAliveRunId && !currentRuntimeRun); if (!ownsPrimaryLane) { return; } await clearOpenCodeRuntimeLaneStorage({ teamsBasePath: getTeamsBasePath(), teamName, laneId: 'primary', }).catch(() => undefined); if (this.runtimeAdapterRunByTeam.get(teamName)?.runId === runId) { this.runtimeAdapterRunByTeam.delete(teamName); } if (this.aliveRunByTeam.get(teamName) === runId) { this.deleteAliveRunId(teamName); } if (this.provisioningRunByTeam.get(teamName) === runId) { this.provisioningRunByTeam.delete(teamName); } this.invalidateRuntimeSnapshotCaches(teamName); } private recordCancelledOpenCodeRuntimeAdapterLaunch( teamName: string, sourceWarning: string | undefined, onProgress: (progress: TeamProvisioningProgress) => void ): TeamLaunchResponse { const runId = randomUUID(); const timestamp = nowIso(); this.provisioningRunByTeam.delete(teamName); this.runtimeAdapterRunByTeam.delete(teamName); this.deleteAliveRunId(teamName); this.invalidateRuntimeSnapshotCaches(teamName); const progress: TeamProvisioningProgress = { runId, teamName, state: 'cancelled', message: 'Provisioning cancelled by user', startedAt: timestamp, updatedAt: timestamp, warnings: sourceWarning ? [sourceWarning] : undefined, }; this.setRuntimeAdapterProgress(progress, onProgress); this.teamChangeEmitter?.({ type: 'process', teamName, runId, detail: 'cancelled', }); return { runId }; } /** * Send a message to the team's lead process via stream-json stdin. * The lead will receive it as a new user turn and can delegate to teammates. */ async sendMessageToTeam( teamName: string, message: string, attachments?: { data: string; mimeType: string; filename?: string }[] ): Promise { const runId = this.getAliveRunId(teamName); if (!runId) { throw new Error(`No active process for team "${teamName}"`); } const run = this.runs.get(runId); if (!run?.child?.stdin?.writable) { throw new Error(`Team "${teamName}" process stdin is not writable`); } await this.sendMessageToRun(run, message, attachments); } private async sendMessageToRun( run: ProvisioningRun, message: string, attachments?: { data: string; mimeType: string; filename?: string }[] ): Promise { if (!this.isCurrentTrackedRun(run)) { throw new Error(`Team "${run.teamName}" run "${run.runId}" is no longer current`); } if (run.processKilled || run.cancelRequested || !run.child?.stdin?.writable) { throw new Error(`Team "${run.teamName}" process stdin is not writable`); } const attachmentPayloads = this.toLeadAttachmentPayloads(attachments); const contentBlocks = normalizeOptionalTeamProviderId(run.request.providerId) === 'codex' && attachmentPayloads.length > 0 ? await this.buildCodexLeadAttachmentContentBlocks(run, message, attachmentPayloads) : (buildClaudeAttachmentDeliveryParts({ text: message, attachments: attachmentPayloads, }).blocks as Record[]); const payload = JSON.stringify({ type: 'user', message: { role: 'user', content: contentBlocks, }, }); const stdin = run.child.stdin; await new Promise((resolve, reject) => { stdin.write(payload + '\n', (err) => { if (err) reject(err); else resolve(); }); }); this.setLeadActivity(run, 'active'); } private toLeadAttachmentPayloads( attachments?: { data: string; mimeType: string; filename?: string }[] ): AttachmentPayload[] { return (attachments ?? []).map((attachment, index) => { const filename = attachment.filename?.trim() || `attachment-${index + 1}`; const bytes = Buffer.from(attachment.data, 'base64'); return { id: `lead_att_${index + 1}`, filename, mimeType: attachment.mimeType, size: bytes.byteLength, data: attachment.data, }; }); } private async buildCodexLeadAttachmentContentBlocks( run: ProvisioningRun, message: string, attachments: AttachmentPayload[] ): Promise[]> { const prepared = await buildCodexNativeAttachmentDeliveryParts({ teamName: run.teamName, messageId: `lead_${run.runId}_${Date.now()}`, text: message, attachments, }); return [ { type: 'text', text: prepared.promptText }, ...prepared.imageParts.map((part) => this.codexImagePartToContentBlock(part)), ]; } private codexImagePartToContentBlock(part: CodexNativeImageArgPart): Record { return { type: 'image', source: { type: 'file', path: part.path, media_type: part.mimeType, }, }; } /** * UNUSED (2026-03-23): teammates read their own inbox files directly via fs.watch, * so forwarding through the lead is unnecessary. Kept for reference — the prompt * pattern here ("MUST: ask teammate to reply back to user") was a useful finding * that informed the direct inbox approach. * * Original purpose: forward a user DM to a teammate by injecting a relay turn * into the lead's stdin and suppressing the lead's textual output. */ async forwardUserDmToTeammate( teamName: string, teammateName: string, userText: string, userSummary?: string ): Promise { const runId = this.getAliveRunId(teamName); if (!runId) { throw new Error(`No active process for team "${teamName}"`); } const run = this.runs.get(runId); if (!run?.child?.stdin?.writable) { throw new Error(`Team "${teamName}" process stdin is not writable`); } if (!run.provisioningComplete) { // Don't inject extra turns during provisioning/bootstrap. return; } this.armSilentTeammateForward(run, teammateName, 'user_dm'); const summaryLine = userSummary?.trim() ? `Summary: ${userSummary.trim()}` : null; const internal = wrapInAgentBlock( [ `UI relay request — forward a direct message to teammate "${teammateName}".`, `MUST: ${getCanonicalSendMessageToolRule(teammateName)}`, `MUST: if they reply to the human, the destination must be to="user" (short answer).`, `CRITICAL: Do NOT send any message to="user" for this turn.`, getCanonicalSendMessageFieldRule(), ].join('\n') ); const message = [ `User DM relay (internal).`, internal, ``, `Message to forward:`, ...(summaryLine ? [summaryLine] : []), userText, ].join('\n'); await this.sendMessageToRun(run, message); } async relayMemberInboxMessages(teamName: string, memberName: string): Promise { if ( this.isCrossTeamPseudoRecipientName(memberName) || this.isCrossTeamToolRecipientName(memberName) ) { return 0; } const relayKey = this.getMemberRelayKey(teamName, memberName); const existing = this.memberInboxRelayInFlight.get(relayKey); if (existing) { try { return await this.waitForInboxRelayInFlight({ promise: existing, relayName: 'member_inbox_relay', relayKey, }); } catch (error) { if (!isInboxRelayInFlightTimeoutError(error)) { throw error; } logger.warn(`[${teamName}] member_inbox_relay_timed_out: ${getErrorMessage(error)}`); return 0; } finally { if (this.memberInboxRelayInFlight.get(relayKey) === existing) { this.memberInboxRelayInFlight.delete(relayKey); } } } const work = (async (): Promise => { const runId = this.getAliveRunId(teamName); if (!runId) return 0; const run = this.runs.get(runId); if (!run?.child || run.processKilled || run.cancelRequested) return 0; if (!run.provisioningComplete) return 0; const isStaleRelayRun = (): boolean => !this.isCurrentTrackedRun(run) || !run.child || run.processKilled || run.cancelRequested; const relayedIds = this.relayedMemberInboxMessageIds.get(relayKey) ?? new Set(); let memberInboxMessages: Awaited> = []; try { memberInboxMessages = await this.inboxReader.getMessagesFor(teamName, memberName); } catch { return 0; } if (isStaleRelayRun()) return 0; const unread = memberInboxMessages .filter((m): m is InboxMessage & { messageId: string } => { if (m.read) return false; if (typeof m.text !== 'string' || m.text.trim().length === 0) return false; if (!this.hasStableMessageId(m)) return false; return !relayedIds.has(m.messageId); }) .sort((a, b) => Date.parse(a.timestamp) - Date.parse(b.timestamp)); if (unread.length === 0) return 0; const relayView = buildRelayInboxView(unread); const silentNoiseUnread = relayView .filter(({ idle, isCoarseNoise }) => { if (idle) return idle.handling === 'silent_noise'; return isCoarseNoise; }) .map(({ message }) => message); const passiveIdleUnread = relayView .filter(({ idle }) => idle?.handling === 'passive_activity') .map(({ message }) => message); const actionableUnread = relayView .filter(({ idle, isCoarseNoise }) => { if (idle) return idle.handling === 'visible_actionable'; return !isCoarseNoise; }) .map(({ message }) => message); const readOnlyIgnoredUnread = [...silentNoiseUnread, ...passiveIdleUnread]; if (isStaleRelayRun()) return 0; if (readOnlyIgnoredUnread.length > 0) { try { await this.markInboxMessagesRead(teamName, memberName, readOnlyIgnoredUnread); if (passiveIdleUnread.length > 0) { logger.debug( `[${teamName}] member relay marked ${passiveIdleUnread.length} passive idle message(s) read without relay for ${memberName}` ); } } catch (error) { logger.debug( `[${teamName}] member relay failed to mark ${readOnlyIgnoredUnread.length} ignored inbox message(s) read for ${memberName}: ${ error instanceof Error ? error.message : String(error) }` ); } } if (actionableUnread.length === 0) return 0; const MAX_RELAY = 10; const batch = [...actionableUnread] .sort(compareMemberInboxRelayMessagesByPriority) .slice(0, MAX_RELAY); this.armSilentTeammateForward(run, memberName, 'member_inbox_relay'); const rememberedRelayIds = this.rememberPendingInboxRelayCandidates(run, memberName, batch); const message = [ `Inbox relay (internal) — forward to "${memberName}".`, wrapInAgentBlock( [ `CRITICAL: Do NOT send any message to="user" for this relay turn. The ONLY valid destination is to="${memberName}".`, getCanonicalSendMessageToolRule(memberName), `If an inbox item has Message kind: member_work_sync_nudge, a member_work_sync_status call alone is incomplete; the recipient must also call member_work_sync_report with the returned agendaFingerprint/reportToken.`, getCanonicalSendMessageFieldRule(), `Preserve task IDs and critical instructions. Do NOT add extra narration outside the SendMessage calls.`, `If an inbox item is marked Source: system_notification, forward that notification exactly once without paraphrasing.`, ].join('\n') ), ``, `Messages to relay (DO NOT respond to user directly):`, ...batch.flatMap((m, idx) => { const summaryLine = m.summary?.trim() ? `Summary: ${m.summary.trim()}` : null; const crossTeamMeta = m.source === 'cross_team' ? { origin: parseCrossTeamPrefix(m.text), sourceTeam: m.from.includes('.') ? m.from.split('.', 1)[0] : null, } : null; const conversationId = m.conversationId ?? crossTeamMeta?.origin?.conversationId; const replyInstructions = crossTeamMeta?.sourceTeam && conversationId ? [ ` Cross-team conversationId: ${conversationId}`, ` Call the MCP tool named cross_team_send with toTeam="${crossTeamMeta.sourceTeam}", conversationId="${conversationId}", and replyToConversationId="${conversationId}". Do NOT put "cross_team_send" into a SendMessage recipient or message_send "to" field.`, ] : []; return [ `${idx + 1}) From: ${m.from || 'unknown'}`, ` Timestamp: ${m.timestamp}`, ` MessageId: ${m.messageId}`, ...(summaryLine ? [` ${summaryLine}`] : []), ...(typeof m.messageKind === 'string' && m.messageKind.trim() ? [` Message kind: ${m.messageKind.trim()}`] : []), ...(typeof m.workSyncIntent === 'string' && m.workSyncIntent.trim() ? [` Work-sync intent: ${m.workSyncIntent.trim()}`] : []), ...(typeof m.source === 'string' && m.source.trim() ? [` Source: ${m.source.trim()}`] : []), ...replyInstructions, ` Text:`, ...m.text.split('\n').map((line) => ` ${line}`), ``, ]; }), ].join('\n'); try { await this.sendMessageToRun(run, message); } catch { this.forgetPendingInboxRelayCandidates(run, memberName, rememberedRelayIds); return 0; } for (const m of batch) { relayedIds.add(m.messageId); } this.relayedMemberInboxMessageIds.set(relayKey, this.trimRelayedSet(relayedIds)); try { await this.markInboxMessagesRead(teamName, memberName, batch); } catch { // Best-effort: relay succeeded; marking read failed. } return batch.length; })(); this.memberInboxRelayInFlight.set(relayKey, work); try { return await this.waitForInboxRelayInFlight({ promise: work, relayName: 'member_inbox_relay', relayKey, }); } catch (error) { if (!isInboxRelayInFlightTimeoutError(error)) { throw error; } logger.warn(`[${teamName}] member_inbox_relay_timed_out: ${getErrorMessage(error)}`); return 0; } finally { if (this.memberInboxRelayInFlight.get(relayKey) === work) { this.memberInboxRelayInFlight.delete(relayKey); } } } async relayInboxFileToLiveRecipient( teamName: string, inboxName: string, options: OpenCodeMemberInboxRelayOptions = {} ): Promise { if ( this.isCrossTeamPseudoRecipientName(inboxName) || this.isCrossTeamToolRecipientName(inboxName) ) { return { kind: 'ignored', relayed: 0 }; } const [config, metaMembers] = await Promise.all([ this.readConfigSnapshot(teamName).catch(() => null), this.membersMetaStore.getMembers(teamName).catch(() => []), ]); const leadName = config?.members?.find((member) => isLeadMember(member))?.name?.trim() || null; const isOpenCodeRecipient = this.isOpenCodeRuntimeRecipientFromSources( inboxName, config, metaMembers ); if (inboxName.trim().toLowerCase() === leadName?.toLowerCase()) { if (isOpenCodeRecipient) { const diagnostic = 'opencode_lead_runtime_session_missing: OpenCode lead inbox relay is unsupported in v1; leaving inbox unread for durable retry/diagnostics.'; logger.warn(`[${teamName}] ${diagnostic} inbox=${inboxName}`); return { kind: 'opencode_lead_unsupported', relayed: 0, diagnostics: [diagnostic], }; } return { kind: 'native_lead', relayed: this.isTeamAlive(teamName) ? await this.relayLeadInboxMessages(teamName) : 0, }; } if (isOpenCodeRecipient) { const relayOptions: OpenCodeMemberInboxRelayOptions = { source: options.source ?? 'watcher', ...(options.onlyMessageId ? { onlyMessageId: options.onlyMessageId } : {}), ...(options.deliveryMetadata ? { deliveryMetadata: options.deliveryMetadata } : {}), }; const relay = await this.relayOpenCodeMemberInboxMessages(teamName, inboxName, relayOptions); return { kind: 'opencode_member', relayed: relay.relayed, diagnostics: relay.diagnostics, lastDelivery: relay.lastDelivery, }; } return { kind: 'native_member_noop', relayed: 0 }; } private async resolveOpenCodeInboxAttachmentPayloads(input: { teamName: string; message: InboxMessage & { messageId: string }; }): Promise< | { ok: true; attachments?: AttachmentPayload[] } | { ok: false; reason: string; diagnostics: string[] } > { const metas = input.message.attachments ?? []; if (metas.length === 0) { return { ok: true }; } let fileDataById: Map | null = null; const payloads: AttachmentPayload[] = []; const missingIds: string[] = []; for (const meta of metas) { const inlinePayload = this.asOpenCodeAttachmentPayload(meta); if (inlinePayload) { payloads.push(inlinePayload); continue; } if (!fileDataById) { let fileData: Awaited>; try { fileData = await this.attachmentStore.getAttachments( input.teamName, input.message.messageId ); } catch (error) { const reason = `opencode_inbox_attachment_payload_read_failed: ${getErrorMessage(error)}`; return { ok: false, reason, diagnostics: [reason] }; } fileDataById = new Map( fileData.map((attachment) => [ attachment.id, { data: attachment.data, mimeType: attachment.mimeType }, ]) ); } const data = fileDataById.get(meta.id); if (!data) { missingIds.push(meta.id); continue; } payloads.push({ ...meta, mimeType: meta.mimeType || data.mimeType, data: data.data, }); } if (missingIds.length > 0) { const reason = `opencode_inbox_attachment_payload_unavailable: ${missingIds.join(', ')}`; return { ok: false, reason, diagnostics: [reason] }; } return { ok: true, attachments: payloads }; } private asOpenCodeAttachmentPayload(meta: AttachmentMeta): AttachmentPayload | null { const data = (meta as Partial).data; return typeof data === 'string' ? { ...meta, data, } : null; } private async inferOpenCodeInboxMessageTaskRefs( teamName: string, message: InboxMessage, readTasks?: () => Promise ): Promise { if (Array.isArray(message.taskRefs) && message.taskRefs.length > 0) { return message.taskRefs; } const tasks = await (readTasks?.() ?? new TeamTaskReader().getTasks(teamName).catch(() => [])); if (tasks.length === 0) { return []; } return inferOpenCodeTaskRefsFromInboxMessage({ teamName, message, tasks, }); } async relayOpenCodeMemberInboxMessages( teamName: string, memberName: string, options: OpenCodeMemberInboxRelayOptions = {} ): Promise { const relayKey = this.getOpenCodeMemberRelayKey(teamName, memberName); const existing = this.openCodeMemberInboxRelayInFlight.get(relayKey); if (existing) { const onlyMessageId = options.onlyMessageId?.trim(); if (!onlyMessageId) { try { return await this.waitForInboxRelayInFlight({ promise: existing, relayName: 'opencode_member_inbox_relay', relayKey, }); } catch (error) { if (!isInboxRelayInFlightTimeoutError(error)) { throw error; } const diagnostic = `opencode_member_inbox_relay_timed_out: ${getErrorMessage(error)}`; logger.warn(`[${teamName}] ${diagnostic}`); return { relayed: 0, attempted: 0, delivered: 0, failed: 1, lastDelivery: { delivered: false, accepted: false, responsePending: false, reason: 'opencode_member_inbox_relay_timed_out', diagnostics: [diagnostic], }, diagnostics: [diagnostic], }; } finally { if (this.openCodeMemberInboxRelayInFlight.get(relayKey) === existing) { this.openCodeMemberInboxRelayInFlight.delete(relayKey); } } } const inboxMessages = await this.inboxReader .getMessagesFor(teamName, memberName) .catch(() => []); const targetMessage = inboxMessages.find((message) => message.messageId === onlyMessageId); if (targetMessage?.read) { if (targetMessage.messageKind === 'member_work_sync_nudge') { this.scheduleOpenCodeMemberInboxDeliveryWake({ teamName, memberName, messageId: onlyMessageId, delayMs: 500, }); const diagnostic = `opencode_work_sync_read_commit_waiting_for_active_relay: ${onlyMessageId}`; return { relayed: 0, attempted: 1, delivered: 0, failed: 0, lastDelivery: { delivered: true, accepted: false, responsePending: true, reason: 'opencode_work_sync_read_commit_waiting_for_active_relay', diagnostics: [diagnostic], }, diagnostics: [diagnostic], }; } return { relayed: 0, attempted: 1, delivered: 1, failed: 0, lastDelivery: { delivered: true }, }; } if (!targetMessage) { const diagnostic = `opencode_inbox_message_missing_after_inflight_relay: ${onlyMessageId}`; return { relayed: 0, attempted: 1, delivered: 0, failed: 1, lastDelivery: { delivered: false, reason: 'opencode_inbox_message_missing_after_inflight_relay', diagnostics: [diagnostic], }, diagnostics: [diagnostic], }; } const diagnostic = `opencode_inbox_relay_queued_behind_active_relay: ${relayKey}/${onlyMessageId}`; this.scheduleOpenCodeMemberInboxDeliveryWake({ teamName, memberName, messageId: onlyMessageId, delayMs: 500, }); return { relayed: 0, attempted: 1, delivered: 0, failed: 0, lastDelivery: { delivered: true, accepted: false, responsePending: true, queuedBehindMessageId: onlyMessageId, reason: 'opencode_inbox_relay_queued_behind_active_relay', diagnostics: [diagnostic], }, diagnostics: [diagnostic], }; } const work = (async (): Promise => { const result: OpenCodeMemberInboxRelayResult = { relayed: 0, attempted: 0, delivered: 0, failed: 0, }; if (!(await this.isOpenCodeRuntimeRecipient(teamName, memberName))) { result.lastDelivery = { delivered: false, reason: 'recipient_is_not_opencode' }; return result; } const memberIdentity = await this.resolveOpenCodeMemberDeliveryIdentity(teamName, memberName); if (!memberIdentity.ok) { result.lastDelivery = { delivered: false, reason: memberIdentity.reason }; return result; } const promptLedger = this.createOpenCodePromptDeliveryLedger(teamName, memberIdentity.laneId); let inboxMessages: Awaited> = []; try { inboxMessages = await this.inboxReader.getMessagesFor(teamName, memberName); } catch (error) { const diagnostic = `opencode_inbox_read_failed: ${getErrorMessage(error)}`; result.lastDelivery = { delivered: false, reason: 'opencode_inbox_read_failed', diagnostics: [diagnostic], }; result.diagnostics = [diagnostic]; return result; } const onlyMessageId = options.onlyMessageId?.trim(); if (onlyMessageId) { const targetMessage = inboxMessages.find((message) => message.messageId === onlyMessageId); if (targetMessage?.read && targetMessage.messageKind !== 'member_work_sync_nudge') { return { relayed: 0, attempted: 1, delivered: 1, failed: 0, lastDelivery: { delivered: true }, }; } if (!targetMessage) { const diagnostic = `opencode_inbox_message_missing: ${onlyMessageId}`; return { relayed: 0, attempted: 1, delivered: 0, failed: 1, lastDelivery: { delivered: false, reason: 'opencode_inbox_message_missing', diagnostics: [diagnostic], }, diagnostics: [diagnostic], }; } } const unread = inboxMessages .filter((message): message is InboxMessage & { messageId: string } => { if (onlyMessageId && message.messageId !== onlyMessageId) return false; if ( message.read && (!onlyMessageId || message.messageKind !== 'member_work_sync_nudge') ) { return false; } if (typeof message.text !== 'string' || message.text.trim().length === 0) return false; return this.hasStableMessageId(message); }) .sort(compareOpenCodeInboxRelayMessagesByPriority) .slice(0, 10); let taskRefInferenceTasks: Promise | null = null; const readTaskRefInferenceTasks = (): Promise => { taskRefInferenceTasks ??= new TeamTaskReader().getTasks(teamName).catch(() => []); return taskRefInferenceTasks; }; for (const message of unread) { let existingRecord = await promptLedger .getByInboxMessage({ teamName, memberName: memberIdentity.canonicalMemberName, laneId: memberIdentity.laneId, inboxMessageId: message.messageId, }) .catch(() => null); if (existingRecord?.status === 'failed_terminal') { const requeuedRecord = await this.requeueOpenCodeRuntimeManifestWatermarkDeliveryIfNeeded( { ledger: promptLedger, ledgerRecord: existingRecord, } ); if (requeuedRecord.status !== 'failed_terminal') { existingRecord = requeuedRecord; } } if (existingRecord?.status === 'failed_terminal') { const requeuedRecord = await this.requeueOpenCodeNoAssistantTerminalDeliveryIfNeeded({ ledger: promptLedger, ledgerRecord: existingRecord, }); if (requeuedRecord.status !== 'failed_terminal') { existingRecord = requeuedRecord; } } if (existingRecord?.status === 'failed_terminal') { let recoveredRecord: OpenCodePromptDeliveryLedgerRecord | null = null; let recoveredVisibleReply: OpenCodeVisibleReplyProof | null = null; if (typeof promptLedger.applyDestinationProof === 'function') { try { const proof = await this.applyOpenCodeVisibleDestinationProof({ ledger: promptLedger, ledgerRecord: existingRecord, teamName, replyRecipient: existingRecord.replyRecipient, memberName: memberIdentity.canonicalMemberName, }); recoveredRecord = proof.ledgerRecord; recoveredVisibleReply = proof.visibleReply; } catch { recoveredRecord = null; recoveredVisibleReply = null; } } const recoveredReadAllowed = recoveredRecord ? await this.isOpenCodeDeliveryResponseReadCommitAllowed({ teamName, memberName: memberIdentity.canonicalMemberName, responseState: recoveredRecord.responseState, actionMode: recoveredRecord.actionMode ?? undefined, taskRefs: recoveredRecord.taskRefs, visibleReply: recoveredVisibleReply, ledgerRecord: recoveredRecord, }) : false; if (recoveredRecord && recoveredReadAllowed) { try { await this.markInboxMessagesRead(teamName, memberName, [message]); const committed = await promptLedger.markInboxReadCommitted({ id: recoveredRecord.id, committedAt: nowIso(), }); this.logOpenCodePromptDeliveryEvent( 'opencode_prompt_delivery_inbox_committed_read', committed, { recoveredTerminal: true } ); result.delivered += 1; result.relayed += 1; result.lastDelivery = { delivered: true, accepted: true, responsePending: false, responseState: committed.responseState, ledgerStatus: committed.status, ledgerRecordId: committed.id, laneId: memberIdentity.laneId, visibleReplyMessageId: committed.visibleReplyMessageId ?? undefined, visibleReplyCorrelation: committed.visibleReplyCorrelation ?? undefined, diagnostics: committed.diagnostics, }; break; } catch (error) { const diagnostic = `opencode_inbox_mark_read_failed_after_terminal_recovery: ${getErrorMessage( error )}`; result.failed += 1; result.lastDelivery = { delivered: false, reason: 'opencode_inbox_mark_read_failed_after_terminal_recovery', diagnostics: [diagnostic], }; result.diagnostics = [...(result.diagnostics ?? []), diagnostic]; break; } } const diagnostic = existingRecord.lastReason ?? `opencode_prompt_delivery_failed_terminal: ${message.messageId}`; result.diagnostics = [...(result.diagnostics ?? []), diagnostic]; if (onlyMessageId) { result.failed += 1; result.lastDelivery = { delivered: false, accepted: false, ledgerStatus: existingRecord.status, ledgerRecordId: existingRecord.id, laneId: memberIdentity.laneId, reason: existingRecord.lastReason ?? 'opencode_prompt_delivery_failed_terminal', diagnostics: existingRecord.diagnostics.length ? existingRecord.diagnostics : [diagnostic], }; } continue; } const fallbackReplyRecipient = typeof message.from === 'string' && message.from.trim() && message.from.trim().toLowerCase() !== memberName.trim().toLowerCase() ? message.from.trim() : 'user'; const effectiveReplyRecipient = existingRecord?.replyRecipient ?? options.deliveryMetadata?.replyRecipient ?? fallbackReplyRecipient; const effectiveActionMode = existingRecord?.actionMode ?? options.deliveryMetadata?.actionMode ?? message.actionMode ?? null; const existingTaskRefs = existingRecord?.taskRefs?.length ? existingRecord.taskRefs : undefined; const metadataTaskRefs = options.deliveryMetadata?.taskRefs?.length ? options.deliveryMetadata.taskRefs : undefined; const messageTaskRefs = message.taskRefs?.length ? message.taskRefs : undefined; const inferredTaskRefs = existingTaskRefs || metadataTaskRefs || messageTaskRefs ? [] : await this.inferOpenCodeInboxMessageTaskRefs( teamName, message, readTaskRefInferenceTasks ); const effectiveTaskRefs = existingTaskRefs ?? metadataTaskRefs ?? messageTaskRefs ?? inferredTaskRefs; const effectiveSource = existingRecord?.source ?? options.source ?? 'watcher'; result.attempted += 1; const attachmentPayloads = await this.resolveOpenCodeInboxAttachmentPayloads({ teamName, message, }); if (!attachmentPayloads.ok) { let failedRecord: OpenCodePromptDeliveryLedgerRecord | null = null; try { const markedAt = nowIso(); const pendingRecord = existingRecord ?? (await promptLedger.ensurePending({ teamName, memberName: memberIdentity.canonicalMemberName, laneId: memberIdentity.laneId, runId: await this.resolveCurrentOpenCodeRuntimeRunId( teamName, memberIdentity.laneId ), inboxMessageId: message.messageId, inboxTimestamp: message.timestamp, source: effectiveSource, replyRecipient: effectiveReplyRecipient, actionMode: effectiveActionMode ?? null, messageKind: message.messageKind ?? null, workSyncIntent: message.workSyncIntent ?? null, taskRefs: effectiveTaskRefs, payloadHash: hashOpenCodePromptDeliveryPayload({ text: message.text, replyRecipient: effectiveReplyRecipient, actionMode: effectiveActionMode ?? null, taskRefs: effectiveTaskRefs, attachments: message.attachments, source: effectiveSource, }), now: markedAt, })); if (pendingRecord.createdAt === markedAt) { this.logOpenCodePromptDeliveryEvent( 'opencode_prompt_delivery_ledger_created', pendingRecord ); } failedRecord = await this.markOpenCodePromptLedgerFailedTerminal({ ledger: promptLedger, id: pendingRecord.id, reason: attachmentPayloads.reason, diagnostics: attachmentPayloads.diagnostics, failedAt: nowIso(), eventContext: { attachmentPayloadUnavailable: true }, }); } catch (error) { const diagnostic = `opencode_inbox_attachment_terminal_ledger_failed: ${getErrorMessage( error )}`; result.diagnostics = [...(result.diagnostics ?? []), diagnostic]; } result.failed += 1; result.diagnostics = [...(result.diagnostics ?? []), ...attachmentPayloads.diagnostics]; result.lastDelivery = { delivered: false, reason: attachmentPayloads.reason, accepted: false, ledgerStatus: failedRecord?.status, ledgerRecordId: failedRecord?.id, laneId: memberIdentity.laneId, diagnostics: attachmentPayloads.diagnostics, }; break; } const delivery = await this.deliverOpenCodeMemberMessage(teamName, { memberName, text: message.text, messageId: message.messageId, replyRecipient: effectiveReplyRecipient, actionMode: effectiveActionMode ?? undefined, messageKind: message.messageKind, workSyncIntent: message.workSyncIntent, workSyncReviewRequestEventIds: message.workSyncReviewRequestEventIds, taskRefs: effectiveTaskRefs, attachments: attachmentPayloads.attachments, source: effectiveSource, inboxTimestamp: message.timestamp, }); result.lastDelivery = delivery; if (!delivery.delivered) { if (delivery.accepted === true) { const diagnostics = delivery.diagnostics ?? [ delivery.reason ?? 'opencode_delivery_response_pending', ]; result.diagnostics = [...(result.diagnostics ?? []), ...diagnostics]; result.lastDelivery = { ...delivery, diagnostics, }; break; } result.failed += 1; result.diagnostics = [ ...(result.diagnostics ?? []), ...(delivery.diagnostics ?? [delivery.reason ?? 'opencode_message_delivery_failed']), ]; if ( !this.isOpenCodeAttachmentDeliveryFailureReason(delivery.reason) && (delivery.reason !== 'opencode_runtime_not_active' || !this.cleanedStoppedTeamOpenCodeRuntimeLanes.has(teamName)) ) { logger.warn( `[${teamName}] OpenCode inbox relay failed for ${memberName}/${message.messageId}: ${ delivery.reason ?? 'unknown error' }` ); } break; } if (delivery.responsePending) { result.diagnostics = [ ...(result.diagnostics ?? []), ...(delivery.diagnostics ?? [delivery.reason ?? 'opencode_delivery_response_pending']), ]; break; } try { await this.markInboxMessagesRead(teamName, memberName, [message]); if (delivery.ledgerRecordId && delivery.laneId) { const committed = await this.createOpenCodePromptDeliveryLedger( teamName, delivery.laneId ).markInboxReadCommitted({ id: delivery.ledgerRecordId, committedAt: nowIso(), }); this.logOpenCodePromptDeliveryEvent( 'opencode_prompt_delivery_inbox_committed_read', committed ); } } catch (error) { const diagnostic = `opencode_inbox_mark_read_failed_after_delivery: ${getErrorMessage( error )}`; if (delivery.ledgerRecordId && delivery.laneId) { const failedCommit = await this.createOpenCodePromptDeliveryLedger( teamName, delivery.laneId ).markInboxReadCommitFailed({ id: delivery.ledgerRecordId, error: diagnostic, failedAt: nowIso(), }); this.logOpenCodePromptDeliveryEvent( 'opencode_prompt_delivery_response_observed', failedCommit, { inboxReadCommitError: diagnostic } ); } result.failed += 1; result.lastDelivery = { delivered: false, reason: 'opencode_inbox_mark_read_failed_after_delivery', diagnostics: [diagnostic], }; result.diagnostics = [...(result.diagnostics ?? []), diagnostic]; logger.warn(`[${teamName}] ${diagnostic}`); break; } result.delivered += 1; result.relayed += 1; break; } if (result.diagnostics?.length) { result.diagnostics = [...new Set(result.diagnostics)]; } return result; })(); this.openCodeMemberInboxRelayInFlight.set(relayKey, work); try { return await this.waitForInboxRelayInFlight({ promise: work, relayName: 'opencode_member_inbox_relay', relayKey, }); } catch (error) { if (!isInboxRelayInFlightTimeoutError(error)) { throw error; } const diagnostic = `opencode_member_inbox_relay_timed_out: ${getErrorMessage(error)}`; logger.warn(`[${teamName}] ${diagnostic}`); return { relayed: 0, attempted: options.onlyMessageId ? 1 : 0, delivered: 0, failed: 1, lastDelivery: { delivered: false, accepted: false, responsePending: false, reason: 'opencode_member_inbox_relay_timed_out', diagnostics: [diagnostic], }, diagnostics: [diagnostic], }; } finally { if (this.openCodeMemberInboxRelayInFlight.get(relayKey) === work) { this.openCodeMemberInboxRelayInFlight.delete(relayKey); } } } /** * Relay unread inbox messages addressed to the team lead into the live lead process. * * Why: teammates (and the UI) write to `inboxes/.json`, but the live lead CLI * process consumes new turns via stream-json stdin. Without relaying, the lead * appears unresponsive to direct messages. * * Returns the number of messages relayed. */ private hasStableMessageId( message: InboxMessage ): message is InboxMessage & { messageId: string } { return typeof message.messageId === 'string' && message.messageId.trim().length > 0; } private async getOpenCodeAgendaSyncRecoveryBypassMessageIds(input: { teamName: string; memberName: string; workSyncIntent?: 'agenda_sync' | 'review_pickup'; taskRefs?: TaskRef[]; foregroundMessages: InboxMessage[]; }): Promise> { const bypassMessageIds = new Set(); if (input.workSyncIntent !== 'agenda_sync') { return bypassMessageIds; } const expectedRefs = this.normalizeOpenCodeTaskRefsForComparison(input.taskRefs); if (expectedRefs.length === 0) { return bypassMessageIds; } const candidateMessages = input.foregroundMessages.filter( (message): message is InboxMessage & { messageId: string } => { if (!this.hasStableMessageId(message)) { return false; } if (typeof message.text !== 'string' || message.text.trim().length === 0) { return false; } if (Array.isArray(message.attachments) && message.attachments.length > 0) { return false; } return true; } ); if (candidateMessages.length === 0) { return bypassMessageIds; } const identity = await this.resolveOpenCodeMemberDeliveryIdentity( input.teamName, input.memberName ).catch(() => null); if (!identity?.ok) { return bypassMessageIds; } const laneIndex = await readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), input.teamName).catch( () => null ); if (!laneIndex) { return bypassMessageIds; } const laneActive = laneIndex.lanes[identity.laneId]?.state === 'active' || (await this.tryRecoverOpenCodeRuntimeLaneForConfiguredMemberAndVerifyActive({ teamName: input.teamName, memberName: identity.canonicalMemberName, laneId: identity.laneId, }).catch(() => false)); if (!laneActive) { return bypassMessageIds; } const records = await this.createOpenCodePromptDeliveryLedger(input.teamName, identity.laneId) .list() .catch(() => null); const proofMissingRecords = (records ?? []).filter( (record) => record.teamName.trim().toLowerCase() === input.teamName.trim().toLowerCase() && record.memberName.trim().toLowerCase() === identity.canonicalMemberName.trim().toLowerCase() && record.laneId === identity.laneId && record.status === 'failed_terminal' && !record.inboxReadCommittedAt && this.openCodeTaskRefsOverlap(record.taskRefs, expectedRefs) && this.isOpenCodeProtocolProofMissingRecord(record) ); if (proofMissingRecords.length === 0) { return bypassMessageIds; } const proofMissingMessageIds = new Set( proofMissingRecords.map((record) => record.inboxMessageId.trim()).filter(Boolean) ); for (const message of candidateMessages) { const messageId = message.messageId.trim(); if (proofMissingMessageIds.has(messageId)) { bypassMessageIds.add(messageId); } } return bypassMessageIds; } private isOpenCodeProtocolProofMissingRecord( record: OpenCodePromptDeliveryLedgerRecord ): boolean { return [record.lastReason, ...record.diagnostics].some( (reason) => typeof reason === 'string' && classifyOpenCodeRuntimeDeliveryReasonCode(reason) === 'protocol_proof_missing' ); } private openCodeTaskRefsOverlap( left: readonly TaskRef[] | undefined, right: readonly TaskRef[] | undefined ): boolean { const leftRefs = this.normalizeOpenCodeTaskRefsForComparison(left); const rightRefs = this.normalizeOpenCodeTaskRefsForComparison(right); if (leftRefs.length === 0 || rightRefs.length === 0) { return false; } const rightKeys = new Set(rightRefs.map((taskRef) => this.openCodeTaskRefKey(taskRef))); return leftRefs.some((taskRef) => rightKeys.has(this.openCodeTaskRefKey(taskRef))); } private isCurrentReviewPickupRequestForegroundMessage( message: InboxMessage, input: { workSyncIntent?: 'agenda_sync' | 'review_pickup'; taskRefs?: TaskRef[] } ): boolean { if (input.workSyncIntent !== 'review_pickup') { return false; } if (message.source !== 'system_notification') { return false; } const expectedRefs = this.normalizeOpenCodeTaskRefsForComparison(input.taskRefs); if (expectedRefs.length === 0) { return false; } const summary = typeof message.summary === 'string' ? message.summary.trim() : ''; const text = typeof message.text === 'string' ? message.text : ''; const looksLikeReviewRequest = summary.startsWith('Review request for #') || (text.includes('**Please review**') && text.includes('review_start')); if (!looksLikeReviewRequest) { return false; } const messageRefs = this.normalizeOpenCodeTaskRefsForComparison(message.taskRefs); if (messageRefs.length > 0) { const expectedKeys = new Set(expectedRefs.map((taskRef) => this.openCodeTaskRefKey(taskRef))); return messageRefs.some((taskRef) => expectedKeys.has(this.openCodeTaskRefKey(taskRef))); } return expectedRefs.some((taskRef) => this.openCodeReviewPickupRequestTextMentionsTask({ summary, text, taskRef }) ); } private isCurrentProofMissingRecoveryForegroundMessage( message: InboxMessage, input: { workSyncIntent?: 'agenda_sync' | 'review_pickup'; workSyncIntentKey?: string } ): boolean { if (input.workSyncIntent !== 'agenda_sync') { return false; } const prefix = 'proof-missing:'; const intentKey = input.workSyncIntentKey?.trim(); if (!intentKey?.startsWith(prefix)) { return false; } const originalMessageId = intentKey.slice(prefix.length).trim(); return this.hasStableMessageId(message) && message.messageId.trim() === originalMessageId; } private openCodeReviewPickupRequestTextMentionsTask(input: { summary: string; text: string; taskRef: TaskRef; }): boolean { const displayId = input.taskRef.displayId.trim(); const taskId = input.taskRef.taskId.trim(); const haystack = `${input.summary}\n${input.text}`; return ( (displayId.length > 0 && (haystack.includes(`#${displayId}`) || haystack.includes(`task #${displayId}`))) || (taskId.length > 0 && haystack.includes(taskId)) ); } private isUserOriginatedLeadRelayMessage(message: InboxMessage): boolean { const from = typeof message.from === 'string' ? message.from.trim().toLowerCase() : ''; return from === 'user' || message.source === 'user_sent'; } private async hasAcceptedMemberWorkSyncReport(input: { teamName: string; memberName: string; }): Promise { const checker = this.memberWorkSyncAcceptedReportChecker; if (!checker) { return false; } try { return ( (await checker({ teamName: input.teamName, memberName: input.memberName, })) === true ); } catch (error) { logger.warn( `[${input.teamName}] Failed to check accepted work sync report for ${input.memberName}: ${getErrorMessage(error)}` ); return false; } } private async hasAcceptedLeadWorkSyncReport(input: { teamName: string; leadName: string; }): Promise { return this.hasAcceptedMemberWorkSyncReport({ teamName: input.teamName, memberName: input.leadName, }); } private async scheduleLeadProofMissingWorkSyncRecovery(input: { teamName: string; leadName: string; message: InboxMessage & { messageId: string }; }): Promise { const scheduler = this.memberWorkSyncProofMissingRecoveryScheduler; if (!scheduler) { return false; } try { const result = (await scheduler({ teamName: input.teamName, memberName: input.leadName, originalMessageId: input.message.messageId, taskRefs: input.message.taskRefs, reason: 'lead_member_work_sync_report_required', })) as { scheduled?: boolean; reason?: string } | null | undefined; return result?.scheduled === true || result?.reason === 'coalesced_recent'; } catch (error) { logger.warn( `[${input.teamName}] Failed to schedule lead proof-missing work sync recovery for ${input.leadName}: ${getErrorMessage(error)}` ); return false; } } private async getLeadRelayReadCommitBatch(input: { teamName: string; leadName: string; batch: (InboxMessage & { messageId: string })[]; }): Promise<(InboxMessage & { messageId: string })[]> { const readCommitBatch: (InboxMessage & { messageId: string })[] = []; for (const message of input.batch) { if (message.messageKind !== 'member_work_sync_nudge') { readCommitBatch.push(message); continue; } if ( await this.hasAcceptedLeadWorkSyncReport({ teamName: input.teamName, leadName: input.leadName, }) ) { readCommitBatch.push(message); continue; } const recoveryScheduled = await this.scheduleLeadProofMissingWorkSyncRecovery({ teamName: input.teamName, leadName: input.leadName, message, }); if (recoveryScheduled) { readCommitBatch.push(message); } } return readCommitBatch; } async relayLeadInboxMessages(teamName: string): Promise { const existing = this.leadInboxRelayInFlight.get(teamName); if (existing) { try { return await this.waitForInboxRelayInFlight({ promise: existing, relayName: 'lead_inbox_relay', relayKey: teamName, }); } catch (error) { if (!isInboxRelayInFlightTimeoutError(error)) { throw error; } logger.warn(`[${teamName}] lead_inbox_relay_timed_out: ${getErrorMessage(error)}`); return 0; } finally { if (this.leadInboxRelayInFlight.get(teamName) === existing) { this.leadInboxRelayInFlight.delete(teamName); } } } const work = (async (): Promise => { const runId = this.getAliveRunId(teamName) ?? this.getProvisioningRunId(teamName); if (!runId) return 0; const run = this.runs.get(runId); if (!run?.child || run.processKilled || run.cancelRequested) return 0; const isStaleRelayRun = (): boolean => !this.isCurrentTrackedRun(run) || !run.child || run.processKilled || run.cancelRequested; // Permission request scan runs even during provisioning — teammates may need // tool approval before the lead's first turn completes. CLI marks inbox messages // as read after native delivery, so we must scan ALL messages (including read). let config: Awaited> | null = null; try { config = await this.readConfigForObservation(teamName); } catch { // config not ready yet during early provisioning — skip scan } if (isStaleRelayRun()) return 0; if (config) { const leadName = config.members?.find((m) => isLeadMember(m))?.name?.trim() || 'team-lead'; try { const leadInboxMessages = await this.inboxReader.getMessagesFor(teamName, leadName); if (isStaleRelayRun()) return 0; const permMsgsToMarkRead: { messageId: string }[] = []; const runStartedAtMs = Date.parse(run.startedAt); for (const msg of leadInboxMessages) { if (typeof msg.text !== 'string') continue; const perm = parsePermissionRequest(msg.text); if (!perm) continue; // Skip permission_requests from previous runs — they're stale const msgTs = Date.parse(msg.timestamp); if ( Number.isFinite(msgTs) && Number.isFinite(runStartedAtMs) && msgTs < runStartedAtMs ) { continue; } // Dedup is handled inside handleTeammatePermissionRequest via processedPermissionRequestIds this.handleTeammatePermissionRequest(run, perm, msg.timestamp); // Mark unread permission_request messages as read to prevent stale unread indicators if (!msg.read && this.hasStableMessageId(msg)) { permMsgsToMarkRead.push({ messageId: msg.messageId }); } } if (permMsgsToMarkRead.length > 0) { await this.markInboxMessagesRead(teamName, leadName, permMsgsToMarkRead).catch( () => {} ); } } catch { // best-effort — inbox may not exist yet } } if (!run.provisioningComplete) return 0; const relayedIds = this.relayedLeadInboxMessageIds.get(teamName) ?? new Set(); // Re-read config if needed (already fetched above but guard provisioningComplete path) if (!config) { try { config = await this.readConfigForObservation(teamName); } catch { return 0; } } if (isStaleRelayRun()) return 0; if (!config) return 0; const leadName = config.members?.find((m) => isLeadMember(m))?.name?.trim() || 'team-lead'; let leadInboxMessages: Awaited> = []; try { leadInboxMessages = await this.inboxReader.getMessagesFor(teamName, leadName); } catch { return 0; } if (isStaleRelayRun()) return 0; await this.refreshMemberSpawnStatusesFromLeadInbox(run); if (isStaleRelayRun()) return 0; const unread = leadInboxMessages .filter((m): m is InboxMessage & { messageId: string } => { if (m.read) return false; if (typeof m.text !== 'string' || m.text.trim().length === 0) return false; if (!this.hasStableMessageId(m)) return false; return !relayedIds.has(m.messageId); }) .sort((a, b) => Date.parse(a.timestamp) - Date.parse(b.timestamp)); if (unread.length === 0) return 0; const relayView = buildRelayInboxView(unread); const silentIdleIds = new Set( relayView .filter(({ idle }) => idle?.handling === 'silent_noise') .map(({ message }) => message.messageId) ); const passiveIdleIds = new Set( relayView .filter(({ idle }) => idle?.handling === 'passive_activity') .map(({ message }) => message.messageId) ); const coarseNonIdleNoiseIds = new Set( relayView .filter(({ idle, isCoarseNoise }) => idle === null && isCoarseNoise) .map(({ message }) => message.messageId) ); const latestOutboundByConversation = new Map(); const latestReadInboundByConversation = new Map(); for (const message of leadInboxMessages) { const timestampMs = Date.parse(message.timestamp); if (!Number.isFinite(timestampMs)) continue; if (message.source === CROSS_TEAM_SENT_SOURCE) { const conversationId = message.conversationId?.trim(); const targetTeam = this.parseCrossTeamTargetTeam(message.to); if (!conversationId || !targetTeam) continue; const key = this.buildCrossTeamConversationKey(targetTeam, conversationId); latestOutboundByConversation.set( key, Math.max(latestOutboundByConversation.get(key) ?? 0, timestampMs) ); continue; } if (message.source === CROSS_TEAM_SOURCE && message.read) { const conversationId = message.replyToConversationId?.trim() ?? message.conversationId?.trim() ?? parseCrossTeamPrefix(message.text)?.conversationId; const sourceTeam = this.getCrossTeamSourceTeam(message.from); if (!conversationId || !sourceTeam) continue; const key = this.buildCrossTeamConversationKey(sourceTeam, conversationId); latestReadInboundByConversation.set( key, Math.max(latestReadInboundByConversation.get(key) ?? 0, timestampMs) ); } } const pendingHistoricalReplies = new Set( Array.from(latestOutboundByConversation.entries()) .filter(([key, sentAtMs]) => sentAtMs > (latestReadInboundByConversation.get(key) ?? 0)) .map(([key]) => key) ); const pendingTransientReplies = this.getPendingCrossTeamReplyExpectationKeys(teamName); const matchedTransientReplyKeys = new Set(); const wasRecentlyDeliveredCrossTeam = (message: InboxMessage): boolean => { if (message.source !== CROSS_TEAM_SOURCE) return false; if (!this.hasStableMessageId(message)) return false; return this.wasRecentlyDeliveredToLead(teamName, message.messageId); }; const isCrossTeamReplyToOwnOutbound = (message: InboxMessage): boolean => { if (message.source !== CROSS_TEAM_SOURCE) return false; const conversationId = message.replyToConversationId?.trim() ?? message.conversationId?.trim() ?? parseCrossTeamPrefix(message.text)?.conversationId; if (!conversationId) return false; const sourceTeam = this.getCrossTeamSourceTeam(message.from); if (!sourceTeam) return false; const key = this.buildCrossTeamConversationKey(sourceTeam, conversationId); if (pendingHistoricalReplies.has(key)) { return true; } if (pendingTransientReplies.has(key)) { matchedTransientReplyKeys.add(key); return true; } return false; }; // Category 1: permanently ignored → mark as read. // Includes noise (idle/shutdown), cross-team sender copies, cross-team reply dedup. const permanentlyIgnored = unread.filter( (m) => silentIdleIds.has(m.messageId) || coarseNonIdleNoiseIds.has(m.messageId) || m.source === CROSS_TEAM_SENT_SOURCE || isCrossTeamReplyToOwnOutbound(m) || wasRecentlyDeliveredCrossTeam(m) ); if (permanentlyIgnored.length > 0) { try { await this.markInboxMessagesRead(teamName, leadName, permanentlyIgnored); } catch { // best-effort } for (const key of matchedTransientReplyKeys) { const [otherTeam, conversationId] = key.split('\0'); if (otherTeam && conversationId) { this.clearPendingCrossTeamReplyExpectation(teamName, otherTeam, conversationId); } } } const passiveIdleUnread = unread.filter((m) => passiveIdleIds.has(m.messageId)); if (passiveIdleUnread.length > 0) { try { await this.markInboxMessagesRead(teamName, leadName, passiveIdleUnread); logger.debug( `[${teamName}] lead relay marked ${passiveIdleUnread.length} passive idle message(s) read without relay` ); } catch (error) { logger.debug( `[${teamName}] lead relay failed to mark ${passiveIdleUnread.length} passive idle message(s) read: ${ error instanceof Error ? error.message : String(error) }` ); } } const readOnlyIgnoredIds = new Set([ ...permanentlyIgnored.map((m) => m.messageId), ...passiveIdleUnread.map((m) => m.messageId), ]); const remainingUnread = unread.filter((m) => !readOnlyIgnoredIds.has(m.messageId)); if (isStaleRelayRun()) return 0; // Category 2: same-team native delivery confirmation (one-to-one pairing). const { nativeMatchedMessageIds, persisted: sameTeamPersisted } = await this.confirmSameTeamNativeMatches(teamName, leadName, remainingUnread); // Category 3: deferred by age — source-less messages within grace window of CURRENT run. // NOT marked read (crash safety: if native delivery fails, retry will relay). const runStartedAtMs = Date.parse(run.startedAt); const deferredByAge = remainingUnread.filter( (m) => !nativeMatchedMessageIds.has(m.messageId) && this.shouldDeferSameTeamMessage(m, leadName, runStartedAtMs) ); const deferredIds = new Set(deferredByAge.map((m) => m.messageId)); // Category 4: teammate permission requests — filter from actionable so they're // NOT relayed to the lead. The actual interception + ToolApprovalRequest emission // is handled by the early scan above (which checks processedPermissionRequestIds). const permissionRequestIds = new Set( remainingUnread .filter((m) => !deferredIds.has(m.messageId) && parsePermissionRequest(m.text) !== null) .map((m) => m.messageId) ); // Actionable: everything not in any category. const actionableUnread = remainingUnread.filter( (m) => !nativeMatchedMessageIds.has(m.messageId) && !deferredIds.has(m.messageId) && !permissionRequestIds.has(m.messageId) ); // Layer 3: schedule retry timers. if (nativeMatchedMessageIds.size > 0 && !sameTeamPersisted) { this.scheduleSameTeamPersistRetry(teamName); } if (deferredByAge.length > 0) { this.scheduleSameTeamDeferredRetry(teamName); } if (actionableUnread.length === 0) return 0; const MAX_RELAY = 10; const prioritizedActionableUnread = [...actionableUnread].sort( compareLeadInboxRelayMessagesByPriority ); const priorityUnread = prioritizedActionableUnread.filter( (message) => getLeadInboxRelayPriority(message) > 0 ); const userOriginatedUnread = prioritizedActionableUnread.filter((message) => this.isUserOriginatedLeadRelayMessage(message) ); const batchSource = priorityUnread.length > 0 ? priorityUnread : userOriginatedUnread.length > 0 ? userOriginatedUnread : prioritizedActionableUnread; const batch = batchSource.slice(0, MAX_RELAY); const replyVisibility: 'user' | 'internal_activity' = priorityUnread.length === 0 && userOriginatedUnread.length > 0 ? 'user' : 'internal_activity'; const batchIds = new Set(batch.map((message) => message.messageId)); const hasPendingFollowUpRelay = unread.some( (message) => !batchIds.has(message.messageId) && !readOnlyIgnoredIds.has(message.messageId) ); const teammateRoster = (config.members ?? []) .filter((member) => { const name = member.name?.trim(); return name && name !== leadName; }) .map((member) => ({ name: member.name.trim(), ...(member.role?.trim() ? { role: member.role.trim() } : {}), })); const rosterContextBlock = buildLeadRosterContextBlock(teamName, leadName, teammateRoster); const workSyncControlUrl = await this.resolveControlApiBaseUrl(); const workSyncControlUrlClause = workSyncControlUrl ? `, controlUrl="${workSyncControlUrl}"` : ''; run.activeCrossTeamReplyHints = batch.flatMap((m) => { if (m.source !== 'cross_team') return []; const sourceTeam = m.from.includes('.') ? m.from.split('.', 1)[0] : ''; const conversationId = m.conversationId ?? parseCrossTeamPrefix(m.text)?.conversationId; if (!sourceTeam || !conversationId) return []; return [{ toTeam: sourceTeam, conversationId }]; }); const replyVisibilityInstruction = replyVisibility === 'user' ? [ `Plain text reply visibility for this batch: user-visible.`, `These inbox rows originated from the human user, so a concise plain text reply is allowed and will be shown to the user.`, `If a visible reply is needed for a teammate or another team, use the appropriate messaging tool; plain text is only for the human response.`, ] : [ `Plain text reply visibility for this batch: internal lead activity only.`, `Do NOT write a user-facing summary for teammate/system/cross-team relay traffic. If the human user must be notified, explicitly call SendMessage with recipient "user".`, `If you take action and no visible message/tool result already records it, you may write one terse internal status line for the team activity log.`, `Do not use that internal status line to confirm, correct, or relay task, kanban, review, PR, branch, merge, or queue state unless you verified it with the source-of-truth tool in this turn.`, `If a visible reply is needed for a teammate, another team, or the human user, use the appropriate messaging tool instead of relying on plain text.`, ]; const message = [ `You have new inbox messages addressed to you (team lead "${leadName}").`, `Process them in the listed order. High-priority work-sync control messages may appear before older routine rows.`, `If action is required, delegate via task creation or SendMessage, and keep responses minimal.`, ...replyVisibilityInstruction, `If there is no action to take, produce ZERO text output. Do NOT write "No action needed.", status echoes, or any other no-op summary.`, `For pure system notifications, comment notifications, or routine teammate availability updates that require no reply/comment/action, say nothing.`, `Do NOT respond with only an agent-only block.`, ...(rosterContextBlock ? [rosterContextBlock] : []), wrapAgentBlock( [ `Internal note: for task assignments, prefer task_create and rely on the board/runtime notification path instead of sending a separate SendMessage for the same assignment.`, `For any MCP board tool call in this turn, teamName MUST be "${teamName}". Never use the lead/member name "${leadName}" as teamName.`, `Treat teammate/system/cross-team claims about task, kanban, review, PR, branch, merge, or queue state as unverified until checked. Before confirming, correcting, relaying, or acting on that state, call the relevant source-of-truth tool first (task_get/task_list/review/kanban tooling, or an available repository/GitHub command/tool). If you have not verified it in this turn, say verification is needed instead of stating the claim as fact.`, `A member_work_sync_status call alone is incomplete for Message kind: member_work_sync_nudge. Do not stop until member_work_sync_report succeeds or a real blocker is recorded.`, `Use task_create_from_message only for messages below that explicitly say "Eligible for task_create_from_message: yes" and provide a User MessageId. Never use task_create_from_message for teammate messages, system notifications, cross-team messages, or any inbox row that is not explicitly marked eligible.`, `If a message below is marked Source: system_notification and its summary looks like "Comment on #...", reply via task_add_comment only when you have a substantive board update (decision, blocker, clarification answer, review result, or concrete next-step change).`, `If a message below has Message kind: member_work_sync_nudge, it is actionable work-sync control traffic, not routine notification noise. Do NOT ignore it as a pure system notification. Call member_work_sync_status with teamName="${teamName}", memberName="${leadName}"${workSyncControlUrlClause}, then call member_work_sync_report with the same teamName/memberName${workSyncControlUrlClause}, the returned agendaFingerprint/reportToken, and taskIds from the nudge task refs. Do not use provider names, runtime names, or team names as memberName. If the agenda still has actionable work you are continuing, use state "still_working"; if blocked, use state "blocked" and record the blocker on the task.`, `Do NOT post acknowledgement-only task comments such as "Принято", "Ок", "На связи", "Жду", or similar low-signal echoes. If the task comment notification is FYI and no durable update is needed, say nothing.`, `If a message below includes a hidden structured task-context block, treat that block as authoritative for teamName/taskId/commentId. Do NOT infer alternate ids or namespaces from visible prose.`, `If a message below is marked Source: cross_team, CALL the MCP tool named cross_team_send. Do NOT use SendMessage or message_send for cross-team replies.`, `NEVER set recipient="cross_team_send" or to="cross_team_send". "cross_team_send" is a tool name, not a teammate.`, ].join('\n') ), ``, `Messages:`, ...batch.flatMap((m, idx) => { const summaryLine = m.summary?.trim() ? `Summary: ${m.summary.trim()}` : null; const isTaskCreateFromMessageEligible = m.source === 'user_sent'; const provenanceLines = isTaskCreateFromMessageEligible ? [` Eligible for task_create_from_message: yes`, ` User MessageId: ${m.messageId}`] : [` Eligible for task_create_from_message: no`]; const crossTeamMeta = m.source === 'cross_team' ? { origin: parseCrossTeamPrefix(m.text), sourceTeam: m.from.includes('.') ? m.from.split('.', 1)[0] : null, } : null; const conversationId = m.replyToConversationId?.trim() ?? m.conversationId ?? crossTeamMeta?.origin?.conversationId; const replyInstructions = crossTeamMeta?.sourceTeam && conversationId ? [ ` Cross-team conversationId: ${conversationId}`, ` Call the MCP tool named cross_team_send with toTeam="${crossTeamMeta.sourceTeam}", conversationId="${conversationId}", and replyToConversationId="${conversationId}". Do NOT use SendMessage or message_send. NEVER set recipient/to to "cross_team_send".`, ] : []; const structuredTaskContextBlock = buildLeadInboxTaskContextBlock(m); return [ `${idx + 1}) From: ${m.from || 'unknown'}`, ` Timestamp: ${m.timestamp}`, ...(summaryLine ? [` ${summaryLine}`] : []), ...(typeof m.messageKind === 'string' && m.messageKind.trim() ? [` Message kind: ${m.messageKind.trim()}`] : []), ...(typeof m.workSyncIntent === 'string' && m.workSyncIntent.trim() ? [` Work-sync intent: ${m.workSyncIntent.trim()}`] : []), ...(typeof m.source === 'string' && m.source.trim() ? [` Source: ${m.source.trim()}`] : []), ...provenanceLines, ...replyInstructions, ...(structuredTaskContextBlock ? [structuredTaskContextBlock] : []), ` Text:`, ...m.text.split('\n').map((line) => ` ${line}`), ``, ]; }), ].join('\n'); const captureTimeoutMs = 15_000; const captureIdleMs = 800; const capturePromise = new Promise((resolve, reject) => { const timeoutHandle = setTimeout(() => { reject(new Error('Timed out waiting for lead reply')); }, captureTimeoutMs); const capture = { leadName, startedAt: nowIso(), textParts: [] as string[], replyVisibility, hasVisibleSendMessage: false, hasUserVisibleSendMessage: false, settled: false, idleHandle: null as NodeJS.Timeout | null, idleMs: captureIdleMs, timeoutHandle, resolveOnce: (text: string) => { if (capture.settled) return; capture.settled = true; if (capture.idleHandle) { clearTimeout(capture.idleHandle); capture.idleHandle = null; } clearTimeout(capture.timeoutHandle); resolve(text); }, rejectOnce: (error: string) => { if (capture.settled) return; capture.settled = true; if (capture.idleHandle) { clearTimeout(capture.idleHandle); capture.idleHandle = null; } clearTimeout(capture.timeoutHandle); reject(new Error(error)); }, }; run.leadRelayCapture = capture; }); try { await this.sendMessageToRun(run, message); } catch { if (run.leadRelayCapture) { clearTimeout(run.leadRelayCapture.timeoutHandle); run.leadRelayCapture = null; } return 0; } this.rememberRecentCrossTeamLeadDeliveryMessageIds( teamName, batch .filter((message) => message.source === CROSS_TEAM_SOURCE) .map((message) => message.messageId) ); let replyText: string | null = null; let capturedVisibleSendMessage = false; let capturedUserVisibleSendMessage = false; try { replyText = (await capturePromise).trim() || null; } catch { // Best-effort: if we captured some text but never got result.success, keep it. const partial = run.leadRelayCapture ? this.joinLeadRelayCaptureText(run.leadRelayCapture) : null; replyText = partial && partial.length > 0 ? partial : null; } finally { if (run.leadRelayCapture) { capturedVisibleSendMessage = run.leadRelayCapture.hasVisibleSendMessage === true; capturedUserVisibleSendMessage = run.leadRelayCapture.hasUserVisibleSendMessage === true; if (run.leadRelayCapture.idleHandle) { clearTimeout(run.leadRelayCapture.idleHandle); run.leadRelayCapture.idleHandle = null; } clearTimeout(run.leadRelayCapture.timeoutHandle); run.leadRelayCapture = null; } } const readCommitBatch = await this.getLeadRelayReadCommitBatch({ teamName, leadName, batch, }); for (const m of readCommitBatch) { relayedIds.add(m.messageId); } this.relayedLeadInboxMessageIds.set(teamName, this.trimRelayedSet(relayedIds)); if (readCommitBatch.length > 0) { try { await this.markInboxMessagesRead(teamName, leadName, readCommitBatch); } catch { // Best-effort: relay succeeded; marking read failed. } } // Strip agent-only blocks — lead may respond with pure coordination content // that is not meant for the human user. const cleanReply = replyText ? stripExactInternalControlEchoPrefix( stripAgentBlocks(replyText), stripAgentBlocks(message) ) : null; if (cleanReply) { if (isTeamInternalControlMessageText(cleanReply)) { logger.debug(`[${teamName}] Suppressed internal lead relay echo`); } else if ( (replyVisibility === 'internal_activity' && capturedVisibleSendMessage) || (replyVisibility === 'user' && capturedUserVisibleSendMessage) ) { logger.debug(`[${teamName}] Suppressed lead relay text duplicated by visible message`); } else if ( replyVisibility === 'internal_activity' && shouldSuppressUnverifiedLeadRelayStateLine(cleanReply) ) { logger.debug(`[${teamName}] Suppressed unverified lead relay state claim`); } else if (replyVisibility === 'internal_activity') { this.pushLiveLeadTextMessage( run, cleanReply, `lead-relay-${runId}-${Date.now()}`, nowIso() ); } else { const relayMsg: InboxMessage = { from: leadName, to: 'user', text: cleanReply, timestamp: nowIso(), read: true, summary: cleanReply.length > 60 ? cleanReply.slice(0, 57) + '...' : cleanReply, messageId: `lead-process-${runId}-${Date.now()}`, source: 'lead_process', }; this.pushLiveLeadProcessMessage(teamName, relayMsg); // Persist to disk so relayed replies survive app restart and trigger FileWatcher this.persistSentMessage(teamName, relayMsg); this.teamChangeEmitter?.({ type: 'inbox', teamName, detail: 'lead-process-reply', }); } } if (hasPendingFollowUpRelay) { this.scheduleLeadInboxFollowUpRelay(teamName); } return batch.length; })(); this.leadInboxRelayInFlight.set(teamName, work); try { return await this.waitForInboxRelayInFlight({ promise: work, relayName: 'lead_inbox_relay', relayKey: teamName, }); } catch (error) { if (!isInboxRelayInFlightTimeoutError(error)) { throw error; } logger.warn(`[${teamName}] lead_inbox_relay_timed_out: ${getErrorMessage(error)}`); return 0; } finally { if (this.leadInboxRelayInFlight.get(teamName) === work) { this.leadInboxRelayInFlight.delete(teamName); } } } /** * Check if a team has an active provisioning run (started but not yet finished). */ hasProvisioningRun(teamName: string): boolean { return this.provisioningRunByTeam.has(teamName); } /** * Check if a team has a live process. */ isTeamAlive(teamName: string): boolean { const runId = this.getAliveRunId(teamName); if (!runId) return false; const run = this.runs.get(runId); if (!run && this.runtimeAdapterRunByTeam.get(teamName)?.runId === runId) { return true; } return run?.child != null && !run.processKilled && !run.cancelRequested; } /** * Get list of teams with active processes. */ getAliveTeams(): string[] { return Array.from(this.aliveRunByTeam.keys()).filter((name) => this.isTeamAlive(name)); } /** * True when shutdown has team runtime state that must not be left headless. * Includes active leads, provisioning runs, runtime-adapter runs, secondary lanes, * and in-flight team operations that may expose a runtime shortly. */ hasActiveTeamRuntimes(): boolean { return this.getShutdownTrackedTeamNames().length > 0; } async getRuntimeState(teamName: string): Promise { const runId = this.getTrackedRunId(teamName); const run = runId ? (this.runs.get(runId) ?? null) : null; if (!run) { const recovered = await readBootstrapRuntimeState(teamName); if (recovered) { return recovered; } } return { teamName, isAlive: this.isTeamAlive(teamName), runId: run?.runId ?? runId ?? null, progress: run?.progress ?? (runId ? (this.runtimeAdapterProgressByRunId.get(runId) ?? null) : null), }; } private languageChangeInFlight: Promise = Promise.resolve(); /** * Notify alive teams when the agent language setting changes. * Compares each team's stored `config.language` with the new code and sends * a message to the team lead if they differ. * * Serialised: rapid language switches (e.g. ru → en → ru) are queued so that * only the latest value is applied to each team. */ async notifyLanguageChange(newLangCode: string): Promise { this.languageChangeInFlight = this.languageChangeInFlight.then(() => this.doNotifyLanguageChange(newLangCode) ); return this.languageChangeInFlight; } private async doNotifyLanguageChange(newLangCode: string): Promise { const aliveTeams = this.getAliveTeams(); if (aliveTeams.length === 0) return; const systemLocale = getSystemLocale(); const newResolved = resolveLanguageName(newLangCode, systemLocale); for (const teamName of aliveTeams) { try { const config = await this.readConfigForStrictDecision(teamName); if (!config) continue; const oldCode = config.language || 'system'; if (oldCode === newLangCode) continue; // Compare resolved names to avoid spurious notifications // e.g. switching from 'ru' to 'system' when system locale is Russian const oldResolved = resolveLanguageName(oldCode, systemLocale); if (oldResolved === newResolved) { // Effective language unchanged — just update stored code silently await this.configReader.updateConfig(teamName, { language: newLangCode }); continue; } const message = `The user has changed the preferred communication language from "${oldResolved}" to "${newResolved}". ` + `Please switch to ${newResolved} for all future responses and broadcast this change to all teammates ` + `so they also switch to ${newResolved}.`; await this.sendMessageToTeam(teamName, message); await this.configReader.updateConfig(teamName, { language: newLangCode }); logger.info(`[${teamName}] Notified about language change: ${oldCode} → ${newLangCode}`); } catch (error) { logger.warn( `[${teamName}] Failed to notify language change: ${ error instanceof Error ? error.message : String(error) }` ); } } } private async markInboxMessagesRead( teamName: string, member: string, messages: { messageId: string }[] ): Promise { const inboxPath = path.join(getTeamsBasePath(), teamName, 'inboxes', `${member}.json`); await withFileLock(inboxPath, async () => { await withInboxLock(inboxPath, async () => { const raw = await tryReadRegularFileUtf8(inboxPath, { timeoutMs: TEAM_JSON_READ_TIMEOUT_MS, maxBytes: TEAM_INBOX_MAX_BYTES, }); if (!raw) { return; } let parsed: unknown; try { parsed = JSON.parse(raw) as unknown; } catch { return; } if (!Array.isArray(parsed)) return; const ids = new Set(messages.map((m) => m.messageId).filter((id) => id.trim().length > 0)); let changed = false; for (const item of parsed) { if (!item || typeof item !== 'object') continue; const row = item as Record; const msgId = getEffectiveInboxMessageId(row); if (!msgId || !ids.has(msgId)) continue; if (row.read !== true) { row.read = true; changed = true; } } if (!changed) return; await atomicWriteAsync(inboxPath, JSON.stringify(parsed, null, 2)); }); }); } private trimRelayedSet(set: Set): Set { const MAX_IDS = 2000; if (set.size <= MAX_IDS) return set; const next = new Set(); const tail = Array.from(set).slice(-MAX_IDS); for (const id of tail) next.add(id); return next; } /** * Intercept SendMessage tool_use blocks from the lead's stream-json output. * * Claude Code's internal teamContext may be lost after session resume (--resume), causing * SendMessage routing to drift away from our canonical team artifacts. By capturing tool_use * calls directly from stdout, we persist a durable message row under the correct team name so * Messages stays accurate even if Claude's own routing is flaky. */ /** * Intercept Task tool_use blocks that spawn team members. * Sets member spawn status to 'spawning' when the lead issues a Task call with team_name + name. */ private captureTeamSpawnEvents(run: ProvisioningRun, content: Record[]): void { for (const part of content) { if (part.type !== 'tool_use' || part.name !== 'Agent') continue; const input = part.input; if (!input || typeof input !== 'object') continue; const inp = input as Record; const teamName = typeof inp.team_name === 'string' ? inp.team_name.trim() : ''; const memberName = typeof inp.name === 'string' ? inp.name.trim() : ''; if (teamName && !memberName) { logger.warn( `[captureTeamSpawnEvents] Agent call for team "${run.teamName}" is missing name - ` + `runtime will spawn an ephemeral subagent instead of a persistent teammate` ); continue; } if (!memberName) continue; if (!teamName) { logger.warn( `[captureTeamSpawnEvents] Agent call for "${memberName}" is missing team_name - ` + `teammate will be an ephemeral subagent, not a persistent member of "${run.teamName}"` ); this.setMemberSpawnStatus( run, memberName, 'error', `Agent spawn for "${memberName}" is missing team_name - spawned as ephemeral subagent instead of persistent teammate` ); continue; } // Only track spawns for this team if (teamName !== run.teamName) continue; const existing = run.memberSpawnStatuses.get(memberName); if ( existing && !existing.hardFailure && (existing.bootstrapConfirmed || existing.runtimeAlive || existing.agentToolAccepted) ) { this.appendMemberBootstrapDiagnostic( run, memberName, 'respawn blocked as duplicate - teammate already online' ); continue; } this.setMemberSpawnStatus(run, memberName, 'spawning'); const toolUseId = typeof part.id === 'string' ? part.id.trim() : ''; if (toolUseId) { run.memberSpawnToolUseIds.set(toolUseId, memberName); } // Advance stepper to "Members joining" when first member spawn is detected if ( !run.provisioningComplete && (run.progress.state === 'configuring' || run.progress.state === 'spawning') ) { const progress = updateProgress(run, 'assembling', `Spawning member ${memberName}...`); run.onProgress(progress); } } } /** * Post-provisioning audit: read config.json members and flag any expectedMember * that was NOT registered by Claude Code as a team member. * * This is the ground-truth check — when Agent(team_name=X, name=Y) succeeds, * the CLI adds Y to config.json members[]. If a member is missing, the spawn * was incorrect (e.g., missing team_name/name params) and the agent ran as a * one-shot subagent instead of a persistent teammate. */ private async getRegisteredTeamMemberNames(teamName: string): Promise | null> { const configPath = path.join(getTeamsBasePath(), teamName, 'config.json'); try { const raw = await tryReadRegularFileUtf8(configPath, { timeoutMs: TEAM_JSON_READ_TIMEOUT_MS, maxBytes: TEAM_CONFIG_MAX_BYTES, }); if (!raw) { return null; } const config = JSON.parse(raw) as { members?: { name?: string; agentType?: string }[]; }; return new Set( (config.members ?? []) .map((m) => (typeof m.name === 'string' ? m.name.trim() : '')) .filter(Boolean) ); } catch { return null; } } private async auditMemberSpawnStatuses(run: ProvisioningRun): Promise { if (!run.expectedMembers || run.expectedMembers.length === 0) return; // Read config.json to get the actual registered members const registeredNames = await this.getRegisteredTeamMemberNames(run.teamName); if (!registeredNames) { try { await fs.promises.access(path.join(getTeamsBasePath(), run.teamName)); } catch { return; } const now = Date.now(); if ( shouldWarnOnUnreadableMemberAuditConfig({ nowMs: now, lastWarnAt: run.lastMemberSpawnAuditConfigReadWarningAt, expectedMembers: run.expectedMembers, memberSpawnStatuses: run.memberSpawnStatuses, }) ) { run.lastMemberSpawnAuditConfigReadWarningAt = now; logger.debug(`[${run.teamName}] auditMemberSpawnStatuses: config.json not readable`); } return; } const liveAgentNames = await this.getLiveTeamAgentNames(run.teamName); // Flag any expected member not found in config.json (excluding the lead) for (const expected of run.expectedMembers) { const current = run.memberSpawnStatuses.get(expected); if ( current?.launchState === 'failed_to_start' || current?.launchState === 'confirmed_alive' || current?.launchState === 'skipped_for_launch' || current?.skippedForLaunch === true ) { continue; } const matchedRuntimeNames = [...registeredNames].filter((name) => { if (name === expected) return true; const parsed = parseNumericSuffixName(name); return parsed !== null && parsed.suffix >= 2 && parsed.base === expected; }); const runtimeAlive = liveAgentNames.has(expected) || matchedRuntimeNames.some((runtimeName) => liveAgentNames.has(runtimeName)); // A teammate may intentionally stay silent after bootstrap. If Claude Code // registered the runtime and the OS process is still alive, treat it as // process-confirmed running. Keep this distinct from heartbeat-confirmed online. if (runtimeAlive) { if (this.isOpenCodeSecondaryLaneMemberInRun(run, expected)) { const base = current ?? createInitialMemberSpawnStatusEntry(); const bootstrapStalled = base.bootstrapStalled === true || this.isOpenCodeBootstrapStallWindowElapsed(base.firstSpawnAcceptedAt); const stalledDiagnostic = bootstrapStalled ? await this.buildOpenCodeSecondaryBootstrapStallDiagnostic(run, expected, base) : null; const runtimeProcessStallDiagnostic = stalledDiagnostic === 'OpenCode bootstrap did not complete runtime_bootstrap_checkin after 5 min.' ? 'Runtime process is alive, but no bootstrap check-in after 5 min.' : stalledDiagnostic; this.setOpenCodeRuntimePendingBootstrapStatus(run, expected, base, { bootstrapStalled, runtimeDiagnostic: bootstrapStalled ? (runtimeProcessStallDiagnostic ?? 'Runtime process is alive, but no bootstrap check-in after 5 min.') : (base.runtimeDiagnostic ?? 'OpenCode runtime process is alive, waiting for bootstrap check-in.'), runtimeDiagnosticSeverity: bootstrapStalled ? 'warning' : (base.runtimeDiagnosticSeverity ?? 'info'), }); if (bootstrapStalled) { await this.maybeSendOpenCodeSecondaryBootstrapCheckinRetryPrompt({ run, memberName: expected, current: base, runtimeDiagnostic: runtimeProcessStallDiagnostic ?? 'Runtime process is alive, but no bootstrap check-in after 5 min.', }); } continue; } this.setMemberSpawnStatus(run, expected, 'online', undefined, 'process'); continue; } if (matchedRuntimeNames.length > 0) { if (current?.agentToolAccepted) { if ( this.isOpenCodeSecondaryLaneMemberInRun(run, expected) && current.launchState === 'runtime_pending_bootstrap' && current.bootstrapConfirmed !== true && current.hardFailure !== true && this.isOpenCodeBootstrapStallWindowElapsed(current.firstSpawnAcceptedAt) ) { const diagnostic = await this.buildOpenCodeSecondaryBootstrapStallDiagnostic( run, expected, current ); this.setOpenCodeSecondaryBootstrapStalledStatus(run, expected, current, diagnostic); await this.maybeSendOpenCodeSecondaryBootstrapCheckinRetryPrompt({ run, memberName: expected, current, runtimeDiagnostic: diagnostic, }); continue; } this.setMemberSpawnStatus(run, expected, 'waiting'); } continue; } if (run.pendingMemberRestarts?.has(expected) === true) { continue; } const acceptedAtMs = current?.firstSpawnAcceptedAt != null ? Date.parse(current.firstSpawnAcceptedAt) : NaN; const graceExpired = current?.agentToolAccepted === true && Number.isFinite(acceptedAtMs) && Date.now() - acceptedAtMs >= MEMBER_LAUNCH_GRACE_MS; if (current?.agentToolAccepted && !graceExpired) { this.setMemberSpawnStatus(run, expected, 'waiting'); continue; } const now = Date.now(); const lastWarnAt = run.lastMemberSpawnAuditMissingWarningAt.get(expected) ?? 0; if ( shouldWarnOnMissingRegisteredMember({ nowMs: now, lastWarnAt, graceExpired, }) ) { run.lastMemberSpawnAuditMissingWarningAt.set(expected, now); logger.warn( `[${run.teamName}] Member "${expected}" not found in config.json members after provisioning` ); } if (graceExpired) { this.setMemberSpawnStatus( run, expected, 'error', 'Teammate not registered after provisioning within the launch grace window.' ); } } } private async finalizeMissingRegisteredMembersAsFailed(run: ProvisioningRun): Promise { if (!run.expectedMembers || run.expectedMembers.length === 0) return; const registeredNames = await this.getRegisteredTeamMemberNames(run.teamName); if (!registeredNames) { return; } for (const expected of run.expectedMembers) { const matchedRuntimeNames = [...registeredNames].filter((name) => { if (name === expected) return true; const parsed = parseNumericSuffixName(name); return parsed !== null && parsed.suffix >= 2 && parsed.base === expected; }); if (matchedRuntimeNames.length > 0) { continue; } if (this.isMemberLifecycleOperationActive(run.teamName, expected)) { continue; } if (run.pendingMemberRestarts?.has(expected) === true) { continue; } const current = run.memberSpawnStatuses.get(expected); if ( current?.launchState === 'failed_to_start' || current?.launchState === 'skipped_for_launch' || current?.skippedForLaunch === true || current?.bootstrapConfirmed || current?.runtimeAlive ) { continue; } this.setMemberSpawnStatus( run, expected, 'error', 'Teammate was not registered in config.json during launch. Persistent spawn failed.' ); } } private markUnconfirmedBootstrapMembersFailed( run: ProvisioningRun, reason: string, options?: { cleanupRequested?: boolean; preserveExistingFailure?: boolean } ): void { const failedAt = nowIso(); const baseReason = reason.trim() || 'Deterministic bootstrap failed before teammate check-in.'; for (const expected of run.expectedMembers) { const prev = run.memberSpawnStatuses.get(expected) ?? createInitialMemberSpawnStatusEntry(); if (prev.bootstrapConfirmed || prev.skippedForLaunch) { continue; } if (this.isMemberLifecycleOperationActive(run.teamName, expected)) { continue; } if (run.pendingMemberRestarts?.has(expected) === true) { continue; } const hasExistingTerminalFailure = prev.status === 'error' || prev.launchState === 'failed_to_start' || prev.hardFailure === true || Boolean(prev.hardFailureReason); const preservedFailureReason = options?.preserveExistingFailure && hasExistingTerminalFailure ? (prev.hardFailureReason ?? prev.error)?.trim() : undefined; const runtimeWasAlive = prev.runtimeAlive === true || prev.livenessSource === 'process'; const fallbackFailureReason = runtimeWasAlive ? `${baseReason} Runtime process was alive after bootstrap failure${ options?.cleanupRequested ? '; launch-owned cleanup requested.' : '.' }` : baseReason; const hardFailureReason = preservedFailureReason || fallbackFailureReason; const next: MemberSpawnStatusEntry = { ...prev, status: 'error', updatedAt: failedAt, error: hardFailureReason, hardFailure: true, hardFailureReason, bootstrapConfirmed: false, bootstrapStalled: undefined, runtimeAlive: options?.cleanupRequested ? false : prev.runtimeAlive, livenessSource: options?.cleanupRequested ? undefined : prev.livenessSource, runtimeDiagnostic: runtimeWasAlive ? options?.cleanupRequested ? 'Bootstrap failed before teammate check-in; launch-owned runtime cleanup requested.' : 'Bootstrap failed before teammate check-in while runtime process was still alive.' : prev.runtimeDiagnostic, runtimeDiagnosticSeverity: runtimeWasAlive ? 'warning' : prev.runtimeDiagnosticSeverity, launchState: 'failed_to_start', }; this.syncMemberTaskActivityForRuntimeTransition(run, expected, prev, next, failedAt); run.memberSpawnStatuses.set(expected, next); this.appendMemberBootstrapDiagnostic(run, expected, hardFailureReason); if (this.isCurrentTrackedRun(run)) { this.emitMemberSpawnChange(run, expected); } } } private async attachLiveRuntimeMetadataToStatuses( teamName: string, statuses: Record, options?: { openCodeSecondaryBootstrapPendingMembers?: ReadonlySet; } ): Promise> { const runtimeByMember = await this.getLiveTeamAgentRuntimeMetadata(teamName); const nextStatuses = { ...statuses }; for (const [memberName, metadata] of runtimeByMember.entries()) { const resolvedStatusKey = nextStatuses[memberName] != null ? memberName : (() => { const matches = Object.keys(nextStatuses).filter((candidateName) => matchesObservedMemberNameForExpected(memberName, candidateName) ); return matches.length === 1 ? matches[0] : null; })(); if (!resolvedStatusKey) { continue; } const current = nextStatuses[resolvedStatusKey]; if (!current) { continue; } const openCodeSecondaryBootstrapPending = options?.openCodeSecondaryBootstrapPendingMembers?.has(resolvedStatusKey) === true && current.launchState === 'runtime_pending_bootstrap' && current.bootstrapConfirmed !== true && current.hardFailure !== true; const openCodeBootstrapStalled = openCodeSecondaryBootstrapPending && (current.bootstrapStalled === true || this.isOpenCodeBootstrapStallWindowElapsed(current.firstSpawnAcceptedAt)); if (current.launchState === 'skipped_for_launch' || current.skippedForLaunch === true) { nextStatuses[resolvedStatusKey] = { ...current, status: 'skipped', launchState: 'skipped_for_launch', skippedForLaunch: true, runtimeAlive: false, bootstrapConfirmed: false, hardFailure: false, hardFailureReason: undefined, error: undefined, livenessSource: undefined, livenessLastCheckedAt: nowIso(), }; continue; } const shouldPreserveProcessBootstrapTransportDiagnostic = current.bootstrapConfirmed !== true && (current.launchState === 'runtime_pending_bootstrap' || current.launchState === 'failed_to_start') && isProcessBootstrapTransportDiagnostic(current.runtimeDiagnostic); const hasStrongEvidence = isStrongRuntimeEvidence(metadata); const hasConfirmedBootstrap = current.bootstrapConfirmed === true || current.launchState === 'confirmed_alive'; const shouldSuppressWeakRuntimeMetadataForConfirmedBootstrap = hasConfirmedBootstrap && !hasStrongEvidence; const failureReason = current.hardFailureReason ?? current.error ?? current.runtimeDiagnostic; const bootstrapProofClearableFailure = isBootstrapProofClearableLaunchFailureReason(failureReason); const metadataRuntimeDiagnosticForUnsafe = buildRuntimeDiagnosticForSpawn(metadata); const unsafeRuntimeDiagnosticEvidence = metadataRuntimeDiagnosticForUnsafe && current.runtimeDiagnostic && metadataRuntimeDiagnosticForUnsafe !== current.runtimeDiagnostic ? `${metadataRuntimeDiagnosticForUnsafe}; ${current.runtimeDiagnostic}` : (metadataRuntimeDiagnosticForUnsafe ?? current.runtimeDiagnostic); const hasUnsafeProvisionedButNotAliveFailure = isProvisionedButNotAliveFailureReason(failureReason) && hasUnsafeProvisionedButNotAliveRuntimeEvidence({ ...current, runtimeDiagnostic: unsafeRuntimeDiagnosticEvidence, runtimeDiagnosticSeverity: metadata.runtimeDiagnosticSeverity ?? current.runtimeDiagnosticSeverity, livenessKind: metadata.livenessKind ?? current.livenessKind, }); const shouldPreserveConfirmedBootstrapRuntimeError = hasConfirmedBootstrap && metadata.alive === false && metadata.runtimeDiagnosticSeverity === 'error'; const shouldPreserveUnsafeMetadataLivenessKind = hasUnsafeProvisionedButNotAliveFailure && (metadata.livenessKind === 'not_found' || metadata.livenessKind === 'shell_only' || metadata.livenessKind === 'runtime_process_candidate' || ((metadata.livenessKind === 'registered_only' || metadata.livenessKind === 'stale_metadata') && (metadata.runtimeDiagnosticSeverity ?? current.runtimeDiagnosticSeverity) !== 'error' && !mentionsProcessTableUnavailable(unsafeRuntimeDiagnosticEvidence) && !mentionsProcessTableUnavailable(failureReason))); let runtimeDiagnostic: string | undefined; let runtimeDiagnosticSeverity: TeamAgentRuntimeDiagnosticSeverity | undefined; if (shouldPreserveProcessBootstrapTransportDiagnostic) { runtimeDiagnostic = current.runtimeDiagnostic; runtimeDiagnosticSeverity = current.runtimeDiagnosticSeverity; } else if (shouldSuppressWeakRuntimeMetadataForConfirmedBootstrap) { if ( current.runtimeDiagnostic && !shouldClearRuntimeDiagnosticAfterBootstrapConfirmation(current.runtimeDiagnostic) ) { runtimeDiagnostic = current.runtimeDiagnostic; runtimeDiagnosticSeverity = current.runtimeDiagnosticSeverity; } else { const metadataRuntimeDiagnostic = metadataRuntimeDiagnosticForUnsafe; if ( metadataRuntimeDiagnostic && !shouldClearRuntimeDiagnosticAfterBootstrapConfirmation(metadataRuntimeDiagnostic) ) { runtimeDiagnostic = metadataRuntimeDiagnostic; runtimeDiagnosticSeverity = metadata.runtimeDiagnosticSeverity; } } } else { runtimeDiagnostic = buildRuntimeDiagnosticForSpawn(metadata); runtimeDiagnosticSeverity = metadata.runtimeDiagnosticSeverity; } const metadataLivenessKind = hasConfirmedBootstrap ? metadata.livenessKind === 'runtime_process' || metadata.livenessKind === 'confirmed_bootstrap' || shouldPreserveConfirmedBootstrapRuntimeError || shouldPreserveUnsafeMetadataLivenessKind ? metadata.livenessKind : current.livenessKind === 'stale_metadata' || current.livenessKind === 'registered_only' ? 'confirmed_bootstrap' : (current.livenessKind ?? 'confirmed_bootstrap') : metadata.livenessKind; const nextEntry: MemberSpawnStatusEntry = { ...current, ...(metadata.model ? { runtimeModel: metadata.model } : {}), ...(metadataLivenessKind ? { livenessKind: metadataLivenessKind } : {}), ...(runtimeDiagnostic || shouldSuppressWeakRuntimeMetadataForConfirmedBootstrap ? { runtimeDiagnostic } : {}), ...(shouldPreserveProcessBootstrapTransportDiagnostic ? { runtimeDiagnosticSeverity } : runtimeDiagnosticSeverity || shouldSuppressWeakRuntimeMetadataForConfirmedBootstrap ? { runtimeDiagnosticSeverity } : {}), livenessLastCheckedAt: nowIso(), }; const hasWeakEvidence = metadata.livenessKind != null && !hasStrongEvidence && current.bootstrapConfirmed !== true; if ( hasStrongEvidence && !openCodeSecondaryBootstrapPending && current.bootstrapStalled !== true && current.hardFailure !== true && current.launchState !== 'failed_to_start' ) { nextEntry.status = 'online'; nextEntry.agentToolAccepted = true; nextEntry.runtimeAlive = true; nextEntry.hardFailure = false; nextEntry.hardFailureReason = undefined; nextEntry.error = undefined; nextEntry.livenessSource = current.bootstrapConfirmed ? current.livenessSource : 'process'; nextEntry.launchState = deriveMemberLaunchState(nextEntry); } if ( (current.bootstrapStalled === true || openCodeSecondaryBootstrapPending) && hasStrongEvidence && current.bootstrapConfirmed !== true && current.launchState !== 'failed_to_start' ) { nextEntry.status = 'waiting'; nextEntry.agentToolAccepted = true; nextEntry.runtimeAlive = true; nextEntry.hardFailure = false; nextEntry.hardFailureReason = undefined; nextEntry.error = undefined; nextEntry.livenessSource = undefined; nextEntry.bootstrapStalled = openCodeBootstrapStalled ? true : undefined; if (openCodeBootstrapStalled) { nextEntry.runtimeDiagnostic = isExplicitLegacyOpenCodeBootstrap(current) ? 'Runtime process is alive, but no bootstrap check-in after 5 min.' : OPENCODE_APP_MANAGED_BOOTSTRAP_STALLED_DIAGNOSTIC; nextEntry.runtimeDiagnosticSeverity = 'warning'; } nextEntry.launchState = deriveMemberLaunchState(nextEntry); } if ( hasStrongEvidence && current.launchState === 'failed_to_start' && bootstrapProofClearableFailure && !hasUnsafeProvisionedButNotAliveFailure ) { nextEntry.status = 'online'; nextEntry.agentToolAccepted = true; nextEntry.runtimeAlive = true; nextEntry.hardFailure = false; nextEntry.hardFailureReason = undefined; nextEntry.error = undefined; nextEntry.livenessSource = current.bootstrapConfirmed ? current.livenessSource : 'process'; nextEntry.launchState = deriveMemberLaunchState(nextEntry); } if ( hasConfirmedBootstrap && current.hardFailure === true && bootstrapProofClearableFailure && !hasUnsafeProvisionedButNotAliveFailure ) { nextEntry.status = 'online'; nextEntry.agentToolAccepted = true; nextEntry.runtimeAlive = true; nextEntry.bootstrapConfirmed = true; nextEntry.hardFailure = false; nextEntry.hardFailureReason = undefined; nextEntry.error = undefined; nextEntry.bootstrapStalled = undefined; nextEntry.launchState = deriveMemberLaunchState(nextEntry); } const healedConfirmedBootstrapFailure = hasConfirmedBootstrap && current.hardFailure === true && bootstrapProofClearableFailure && !hasUnsafeProvisionedButNotAliveFailure; if (shouldPreserveConfirmedBootstrapRuntimeError) { nextEntry.runtimeAlive = false; if (nextEntry.livenessSource === 'process') { nextEntry.livenessSource = undefined; } } if (hasWeakEvidence && !healedConfirmedBootstrapFailure) { nextEntry.runtimeAlive = false; if (nextEntry.livenessSource === 'process') { nextEntry.livenessSource = undefined; } if ( current.launchState === 'runtime_pending_bootstrap' || current.launchState === 'runtime_pending_permission' ) { nextEntry.agentToolAccepted = true; } if ( current.status === 'online' && current.hardFailure !== true && current.launchState !== 'failed_to_start' ) { nextEntry.status = nextEntry.agentToolAccepted ? 'waiting' : 'spawning'; } nextEntry.launchState = deriveMemberLaunchState(nextEntry); } nextStatuses[resolvedStatusKey] = nextEntry; } for (const [memberName, current] of Object.entries(nextStatuses)) { const openCodeSecondaryBootstrapPending = options?.openCodeSecondaryBootstrapPendingMembers?.has(memberName) === true && current.launchState === 'runtime_pending_bootstrap' && current.bootstrapConfirmed !== true && current.hardFailure !== true; if ( !openCodeSecondaryBootstrapPending || current.bootstrapStalled === true || !this.isOpenCodeBootstrapStallWindowElapsed(current.firstSpawnAcceptedAt) ) { continue; } const runtimeProcessAlive = current.runtimeAlive === true && current.livenessKind === 'runtime_process'; const runtimeDiagnostic = isExplicitLegacyOpenCodeBootstrap(current) ? runtimeProcessAlive ? 'Runtime process is alive, but no bootstrap check-in after 5 min.' : 'OpenCode bootstrap did not complete runtime_bootstrap_checkin after 5 min.' : OPENCODE_APP_MANAGED_BOOTSTRAP_STALLED_DIAGNOSTIC; const nextEntry: MemberSpawnStatusEntry = { ...current, status: 'waiting', launchState: 'runtime_pending_bootstrap', agentToolAccepted: true, runtimeAlive: runtimeProcessAlive, bootstrapConfirmed: false, hardFailure: false, hardFailureReason: undefined, error: undefined, livenessSource: undefined, livenessKind: current.livenessKind ?? (runtimeProcessAlive ? 'runtime_process' : 'registered_only'), runtimeDiagnostic, runtimeDiagnosticSeverity: 'warning', bootstrapStalled: true, livenessLastCheckedAt: nowIso(), updatedAt: nowIso(), }; nextEntry.launchState = deriveMemberLaunchState(nextEntry); nextStatuses[memberName] = nextEntry; } return nextStatuses; } private getOpenCodeSecondaryBootstrapPendingMemberNames( snapshot: PersistedTeamLaunchSnapshot | null | undefined ): ReadonlySet { if (!snapshot) { return new Set(); } const names = Object.entries(snapshot.members) .filter(([, member]) => { return ( member.providerId === 'opencode' && member.laneKind === 'secondary' && member.laneOwnerProviderId === 'opencode' && member.launchState === 'runtime_pending_bootstrap' && member.bootstrapConfirmed !== true && member.hardFailure !== true ); }) .map(([name]) => name); return new Set(names); } private applyOpenCodeSecondaryBootstrapStallOverlay( snapshot: PersistedTeamLaunchSnapshot | null ): PersistedTeamLaunchSnapshot | null { if (!snapshot) { return null; } const nowMs = Date.now(); const updatedAt = nowIso(); let changed = false; const members: Record = { ...snapshot.members }; for (const memberName of this.getPersistedLaunchMemberNames(snapshot)) { let current = members[memberName]; if (!current) { continue; } const stableFirstSpawnAcceptedAt = isPersistedOpenCodeSecondaryLaneMember(current) ? resolveOpenCodeBootstrapAcceptedAt(current) : undefined; if ( stableFirstSpawnAcceptedAt && stableFirstSpawnAcceptedAt !== current.firstSpawnAcceptedAt ) { current = { ...current, firstSpawnAcceptedAt: stableFirstSpawnAcceptedAt, }; members[memberName] = current; changed = true; } if (!shouldMarkPersistedOpenCodeBootstrapStalled(current, nowMs)) { continue; } const runtimeDiagnostic = getOpenCodeSecondaryBootstrapStallDiagnosticFromPersisted(current); members[memberName] = { ...current, launchState: 'runtime_pending_bootstrap', agentToolAccepted: true, runtimeAlive: current.runtimeAlive === true && current.livenessKind === 'runtime_process', bootstrapConfirmed: false, hardFailure: false, hardFailureReason: undefined, livenessKind: current.livenessKind ?? 'registered_only', runtimeDiagnostic, runtimeDiagnosticSeverity: 'warning', bootstrapStalled: true, firstSpawnAcceptedAt: stableFirstSpawnAcceptedAt ?? current.firstSpawnAcceptedAt, lastEvaluatedAt: updatedAt, diagnostics: mergeRuntimeDiagnostics(current.diagnostics, [ runtimeDiagnostic, 'opencode_bootstrap_stalled', ]), }; changed = true; } if (!changed) { return snapshot; } return createPersistedLaunchSnapshot({ teamName: snapshot.teamName, expectedMembers: snapshot.expectedMembers, bootstrapExpectedMembers: snapshot.bootstrapExpectedMembers, leadSessionId: snapshot.leadSessionId, launchPhase: snapshot.launchPhase, members, updatedAt, }); } private async getLiveTeamAgentNames(teamName: string): Promise> { const runtimeByMember = await this.getLiveTeamAgentRuntimeMetadata(teamName); return new Set( [...runtimeByMember.entries()] .filter(([, metadata]) => metadata.alive) .map(([memberName]) => memberName) ); } private findConfiguredMemberModel( configuredMembers: TeamConfig['members'] | undefined, memberName: string ): string | undefined { for (const member of configuredMembers ?? []) { const candidateName = typeof member?.name === 'string' ? member.name.trim() : ''; if (!candidateName || !matchesExactTeamMemberName(candidateName, memberName)) { continue; } const model = member.model?.trim(); if (model) { return model; } } return undefined; } private findMetaMemberModel( metaMembers: Awaited>, memberName: string ): string | undefined { for (const member of metaMembers) { const candidateName = member.name?.trim() ?? ''; if (!candidateName || !matchesExactTeamMemberName(candidateName, memberName)) { continue; } const model = member.model?.trim(); if (model) { return model; } } return undefined; } private resolveEffectiveConfiguredMember( configuredMembers: TeamConfig['members'] | undefined, metaMembers: Awaited>, memberName: string ): { name: string; role?: string; workflow?: string; isolation?: 'worktree'; providerId?: TeamProviderId; providerBackendId?: TeamProviderBackendId; model?: string; effort?: EffortLevel; fastMode?: TeamFastMode; mcpPolicy?: ReturnType; cwd?: string; agentType?: string; removedAt?: number | string; } | null { const configuredMember = (configuredMembers ?? []).find((member) => { const candidateName = typeof member?.name === 'string' ? member.name.trim() : ''; return candidateName.length > 0 && matchesExactTeamMemberName(candidateName, memberName); }); const metaMember = metaMembers.find((member) => { const candidateName = member.name?.trim() ?? ''; return candidateName.length > 0 && matchesExactTeamMemberName(candidateName, memberName); }); if (!configuredMember && !metaMember) { return null; } const name = metaMember?.name?.trim() || configuredMember?.name?.trim() || memberName.trim() || memberName; const role = metaMember?.role?.trim() || configuredMember?.role?.trim() || undefined; const workflow = metaMember?.workflow?.trim() || configuredMember?.workflow?.trim() || undefined; const isolation = metaMember?.isolation === 'worktree' || configuredMember?.isolation === 'worktree' ? 'worktree' : undefined; const providerId = normalizeTeamMemberProviderId(metaMember?.providerId) ?? normalizeTeamMemberProviderId(configuredMember?.providerId); const providerBackendId = migrateProviderBackendId(metaMember?.providerId, metaMember?.providerBackendId) ?? migrateProviderBackendId(configuredMember?.providerId, configuredMember?.providerBackendId); const model = metaMember?.model?.trim() || configuredMember?.model?.trim() || undefined; const effort = isTeamEffortLevel(metaMember?.effort) ? metaMember.effort : isTeamEffortLevel(configuredMember?.effort) ? configuredMember.effort : undefined; const fastMode = metaMember?.fastMode === 'inherit' || metaMember?.fastMode === 'on' || metaMember?.fastMode === 'off' ? metaMember.fastMode : configuredMember?.fastMode === 'inherit' || configuredMember?.fastMode === 'on' || configuredMember?.fastMode === 'off' ? configuredMember.fastMode : undefined; const agentType = metaMember?.agentType?.trim() || configuredMember?.agentType?.trim() || undefined; const mcpPolicy = normalizeTeamMemberMcpPolicy(metaMember?.mcpPolicy) ?? normalizeTeamMemberMcpPolicy(configuredMember?.mcpPolicy); const cwd = metaMember?.cwd?.trim() || configuredMember?.cwd?.trim() || undefined; const removedAt = metaMember?.removedAt ?? configuredMember?.removedAt; return { name, ...(role ? { role } : {}), ...(workflow ? { workflow } : {}), ...(isolation ? { isolation } : {}), ...(providerId ? { providerId } : {}), ...(providerBackendId ? { providerBackendId } : {}), ...(model ? { model } : {}), ...(effort ? { effort } : {}), ...(fastMode ? { fastMode } : {}), ...(mcpPolicy ? { mcpPolicy } : {}), ...(cwd ? { cwd } : {}), ...(agentType ? { agentType } : {}), ...(removedAt != null ? { removedAt } : {}), }; } private resolveLeadMemberName( configuredMembers: TeamConfig['members'] | undefined, metaMembers: Awaited> ): string { const configuredLead = (configuredMembers ?? []).find((member) => isLeadMember(member)); const configuredLeadName = configuredLead?.name?.trim(); if (configuredLeadName) { return configuredLeadName; } const metaLead = metaMembers.find((member) => isLeadMember(member)); const metaLeadName = metaLead?.name?.trim(); if (metaLeadName) { return metaLeadName; } return 'team-lead'; } private isMemberRemovedInMeta( metaMembers: Awaited>, memberName: string ): boolean { const normalizedMemberName = memberName.trim().toLowerCase(); if (!normalizedMemberName) { return false; } return metaMembers.some((member) => { const candidateName = member.name?.trim().toLowerCase() ?? ''; return ( candidateName.length > 0 && candidateName === normalizedMemberName && Boolean(member.removedAt) ); }); } private filterRemovedMembersFromLaunchSnapshot( snapshot: PersistedTeamLaunchSnapshot, metaMembers: Awaited> ): PersistedTeamLaunchSnapshot { const removedNames = new Set( metaMembers .filter((member) => Boolean(member.removedAt)) .map((member) => member.name?.trim().toLowerCase() ?? '') .filter((name) => name.length > 0) ); if (removedNames.size === 0) { return snapshot; } const isRemoved = (name: string | undefined): boolean => { const normalized = name?.trim().toLowerCase() ?? ''; return normalized.length > 0 && removedNames.has(normalized); }; const expectedMembers = this.getPersistedLaunchMemberNames(snapshot).filter( (name) => !isRemoved(name) ); const members: Record = {}; for (const [memberName, member] of Object.entries(snapshot.members)) { if (isRemoved(memberName) || isRemoved(member.name)) { continue; } members[memberName] = { ...member }; } return createPersistedLaunchSnapshot({ teamName: snapshot.teamName, expectedMembers, bootstrapExpectedMembers: snapshot.bootstrapExpectedMembers?.filter( (name) => !isRemoved(name) ), leadSessionId: snapshot.leadSessionId, launchPhase: snapshot.launchPhase, members, updatedAt: snapshot.updatedAt, }); } private findEffectiveRunMemberModel( run: ProvisioningRun | null, memberName: string ): string | undefined { const member = this.findEffectiveRunMember(run, memberName); const model = member?.model?.trim(); return model || undefined; } private findEffectiveRunMember( run: ProvisioningRun | null, memberName: string ): TeamCreateRequest['members'][number] | undefined { if (!run) { return undefined; } for (const member of [...(run.allEffectiveMembers ?? []), ...(run.effectiveMembers ?? [])]) { const candidateName = member.name?.trim() ?? ''; if (!candidateName || !matchesTeamMemberIdentity(candidateName, memberName)) { continue; } return member; } return undefined; } private findTrackedMemberSpawnStatus( run: ProvisioningRun | null, memberName: string ): MemberSpawnStatusEntry | undefined { if (!run) { return undefined; } const statusMap = run.memberSpawnStatuses instanceof Map ? run.memberSpawnStatuses : undefined; if (!statusMap) { return undefined; } const direct = statusMap.get(memberName); if (direct) { return direct; } for (const [candidateName, entry] of statusMap.entries()) { if (matchesTeamMemberIdentity(candidateName, memberName)) { return entry; } } return undefined; } private buildLaunchMemberSpawnStatus( member: PersistedTeamLaunchMemberState | undefined, runtimeModel?: string ): MemberSpawnStatusEntry | undefined { if (!member) { return undefined; } return { status: member.hardFailure ? 'error' : member.bootstrapConfirmed || member.launchState === 'confirmed_alive' ? 'online' : member.agentToolAccepted ? 'waiting' : 'spawning', launchState: member.launchState, ...(member.hardFailureReason ? { hardFailureReason: member.hardFailureReason } : {}), ...(member.pendingPermissionRequestIds?.length ? { pendingPermissionRequestIds: member.pendingPermissionRequestIds } : {}), agentToolAccepted: member.agentToolAccepted, runtimeAlive: member.runtimeAlive, bootstrapConfirmed: member.bootstrapConfirmed, hardFailure: member.hardFailure, ...(runtimeModel ? { runtimeModel } : {}), ...(member.livenessKind ? { livenessKind: member.livenessKind } : {}), ...(member.runtimeDiagnostic ? { runtimeDiagnostic: member.runtimeDiagnostic } : {}), ...(member.runtimeDiagnosticSeverity ? { runtimeDiagnosticSeverity: member.runtimeDiagnosticSeverity } : {}), ...(member.bootstrapStalled ? { bootstrapStalled: true } : {}), ...(member.firstSpawnAcceptedAt ? { firstSpawnAcceptedAt: member.firstSpawnAcceptedAt } : {}), ...(member.lastHeartbeatAt ? { lastHeartbeatAt: member.lastHeartbeatAt } : {}), updatedAt: member.lastEvaluatedAt, }; } private shouldPreferCurrentLaunchMemberStatus( trackedStatus: MemberSpawnStatusEntry | undefined, launchStatus: MemberSpawnStatusEntry | undefined ): boolean { if (!launchStatus?.bootstrapConfirmed && launchStatus?.launchState !== 'confirmed_alive') { return false; } if (!trackedStatus) { return true; } return ( trackedStatus.hardFailure !== true && trackedStatus.launchState !== 'failed_to_start' && trackedStatus.launchState !== 'runtime_pending_permission' ); } private isLaunchMemberStatusRelevantToRuntimeRun( member: PersistedTeamLaunchMemberState | undefined, activeRuntimeRunId: string ): boolean { if (!member || activeRuntimeRunId.length === 0) { return false; } const memberRuntimeRunId = member.runtimeRunId?.trim() ?? ''; if (member.providerId === 'opencode') { return memberRuntimeRunId.length > 0 && memberRuntimeRunId === activeRuntimeRunId; } return memberRuntimeRunId.length === 0 || memberRuntimeRunId === activeRuntimeRunId; } private async getLiveTeamAgentRuntimeMetadata( teamName: string ): Promise> { const runId = this.getTrackedRunId(teamName); const cached = this.liveTeamAgentRuntimeMetadataCache.get(teamName); if (cached && cached.expiresAtMs > Date.now() && cached.runId === runId) { return this.cloneLiveTeamAgentRuntimeMetadata(cached.metadata); } const generationAtStart = this.getRuntimeSnapshotCacheGeneration(teamName); const existingRequest = this.liveTeamAgentRuntimeMetadataInFlightByTeam.get(teamName); if ( existingRequest?.generationAtStart === generationAtStart && existingRequest.runIdAtStart === runId ) { return this.cloneLiveTeamAgentRuntimeMetadata(await existingRequest.promise); } const request = this.buildLiveTeamAgentRuntimeMetadata( teamName, runId, generationAtStart ).finally(() => { if (this.liveTeamAgentRuntimeMetadataInFlightByTeam.get(teamName)?.promise === request) { this.liveTeamAgentRuntimeMetadataInFlightByTeam.delete(teamName); } }); this.liveTeamAgentRuntimeMetadataInFlightByTeam.set(teamName, { generationAtStart, runIdAtStart: runId, promise: request, }); return this.cloneLiveTeamAgentRuntimeMetadata(await request); } private async buildLiveTeamAgentRuntimeMetadata( teamName: string, runId: string | null, generationAtStart: number ): Promise> { const run = runId ? (this.runs.get(runId) ?? null) : null; let configuredMembers: TeamConfig['members'] = []; try { configuredMembers = (await this.readConfigSnapshot(teamName))?.members ?? []; } catch { configuredMembers = []; } let metaMembers: Awaited> = []; try { metaMembers = await this.membersMetaStore.getMembers(teamName); } catch { metaMembers = []; } const persistedRuntimeMembers = this.readPersistedRuntimeMembers(teamName); const metadataByMember = new Map(); const upsertMetadata = ( memberName: string, patch: Partial ): void => { const current = metadataByMember.get(memberName) ?? { alive: false }; metadataByMember.set(memberName, { ...current, ...patch, alive: patch.alive ?? current.alive, }); }; for (const member of persistedRuntimeMembers) { const memberName = typeof member.name === 'string' ? member.name.trim() : ''; if ( !memberName || this.isMemberRemovedInMeta(metaMembers, memberName) || isLeadMember({ name: memberName }) ) { continue; } const runtimeModel = this.findEffectiveRunMemberModel(run, memberName) ?? this.findConfiguredMemberModel(configuredMembers, memberName) ?? this.findMetaMemberModel(metaMembers, memberName); upsertMetadata(memberName, { backendType: normalizeTeamAgentRuntimeBackendType(member.backendType, false), providerId: normalizeOptionalTeamProviderId(member.providerId), agentId: typeof member.agentId === 'string' ? member.agentId.trim() || undefined : undefined, tmuxPaneId: typeof member.tmuxPaneId === 'string' ? member.tmuxPaneId.trim() || undefined : undefined, ...(normalizeRuntimePositiveInteger(member.runtimePid) ? { metricsPid: normalizeRuntimePositiveInteger(member.runtimePid) } : {}), ...(typeof member.runtimeSessionId === 'string' && member.runtimeSessionId.trim() ? { runtimeSessionId: member.runtimeSessionId.trim() } : {}), ...(typeof member.cwd === 'string' && member.cwd.trim() ? { cwd: member.cwd.trim() } : {}), ...(runtimeModel ? { model: runtimeModel } : {}), }); } for (const member of configuredMembers) { const memberName = typeof member?.name === 'string' ? member.name.trim() : ''; if ( !memberName || this.isMemberRemovedInMeta(metaMembers, memberName) || isLeadMember({ name: memberName, agentType: member.agentType }) ) { continue; } const configuredRuntimeMember = member as unknown as Record; const configuredAgentId = typeof configuredRuntimeMember.agentId === 'string' ? configuredRuntimeMember.agentId.trim() : ''; const configuredTmuxPaneId = typeof configuredRuntimeMember.tmuxPaneId === 'string' ? configuredRuntimeMember.tmuxPaneId.trim() : ''; const configuredBackendType = typeof configuredRuntimeMember.backendType === 'string' ? configuredRuntimeMember.backendType : undefined; const runtimeModel = this.findEffectiveRunMemberModel(run, memberName) || member.model?.trim() || this.findMetaMemberModel(metaMembers, memberName); upsertMetadata(memberName, { ...(runtimeModel ? { model: runtimeModel } : {}), ...(configuredAgentId ? { agentId: configuredAgentId } : {}), ...(configuredTmuxPaneId ? { tmuxPaneId: configuredTmuxPaneId } : {}), ...(normalizeOptionalTeamProviderId(member.providerId) ? { providerId: normalizeOptionalTeamProviderId(member.providerId) } : {}), ...(typeof member.cwd === 'string' && member.cwd.trim() ? { cwd: member.cwd.trim() } : {}), ...(normalizeTeamAgentRuntimeBackendType(configuredBackendType, false) ? { backendType: normalizeTeamAgentRuntimeBackendType(configuredBackendType, false), } : {}), }); } for (const member of metaMembers) { const memberName = typeof member?.name === 'string' ? member.name.trim() : ''; if ( !memberName || member.removedAt || isLeadMember({ name: memberName, agentType: member.agentType }) ) { continue; } const runtimeModel = this.findEffectiveRunMemberModel(run, memberName) || member.model?.trim() || this.findConfiguredMemberModel(configuredMembers, memberName); upsertMetadata(memberName, { ...(runtimeModel ? { model: runtimeModel } : {}), ...(normalizeOptionalTeamProviderId(member.providerId) ? { providerId: normalizeOptionalTeamProviderId(member.providerId) } : {}), ...(typeof member.agentId === 'string' && member.agentId.trim() ? { agentId: member.agentId.trim() } : {}), ...(typeof member.cwd === 'string' && member.cwd.trim() ? { cwd: member.cwd.trim() } : {}), }); } for (const member of run?.effectiveMembers ?? []) { const memberName = member.name?.trim() ?? ''; if (!memberName || isLeadMember(member) || memberName.toLowerCase() === 'user') { continue; } const providerId = normalizeOptionalTeamProviderId(member.providerId); upsertMetadata(memberName, { ...(member.model?.trim() ? { model: member.model.trim() } : {}), ...(providerId ? { providerId } : {}), }); } for (const lane of run?.mixedSecondaryLanes ?? []) { const memberName = lane.member.name?.trim() ?? ''; if (!memberName || this.isMemberRemovedInMeta(metaMembers, memberName)) { continue; } const evidence = lane.result?.members[memberName]; const runtimeModel = lane.member.model?.trim() || undefined; const laneMemberCwd = typeof (lane.member as { cwd?: unknown }).cwd === 'string' ? (lane.member as { cwd?: string }).cwd?.trim() : ''; const laneCwd = laneMemberCwd || run?.request.cwd; upsertMetadata(memberName, { backendType: 'process', providerId: 'opencode', alive: false, livenessKind: evidence?.livenessKind, pidSource: evidence?.pidSource, runtimeDiagnostic: evidence?.runtimeDiagnostic, ...(laneCwd ? { cwd: laneCwd } : {}), ...(runtimeModel ? { model: runtimeModel } : {}), ...(typeof evidence?.runtimePid === 'number' && evidence.runtimePid > 0 ? { metricsPid: evidence.runtimePid } : {}), ...(evidence?.sessionId ? { runtimeSessionId: evidence.sessionId } : {}), }); } const currentRuntimeAdapterRun = this.runtimeAdapterRunByTeam.get(teamName); const persistedLaunchSnapshot = choosePreferredLaunchSnapshot( await readBootstrapLaunchSnapshot(teamName).catch(() => null), await this.launchStateStore.read(teamName).catch(() => null) ); const activeRuntimeRunId = run?.runId?.trim() || currentRuntimeAdapterRun?.runId?.trim() || runId?.trim() || ''; for (const persistedMember of Object.values(persistedLaunchSnapshot?.members ?? {})) { const memberName = persistedMember.name?.trim() ?? ''; if (!memberName || this.isMemberRemovedInMeta(metaMembers, memberName)) { continue; } const activeRunMember = this.findEffectiveRunMember(run, memberName); const activeRunModel = activeRunMember?.model?.trim(); const evidenceModel = currentRuntimeAdapterRun?.members?.[memberName]?.model?.trim(); const activeRunProviderId = normalizeOptionalTeamProviderId(activeRunMember?.providerId) ?? inferTeamProviderIdFromModel(activeRunModel ?? evidenceModel); const effectiveProviderId = activeRunProviderId ?? persistedMember.providerId; const currentRuntimeAdapterEvidence = currentRuntimeAdapterRun?.members?.[memberName]; upsertMetadata(memberName, { backendType: effectiveProviderId === 'opencode' ? 'process' : metadataByMember.get(memberName)?.backendType, providerId: effectiveProviderId, alive: false, livenessKind: currentRuntimeAdapterEvidence?.livenessKind ?? persistedMember.livenessKind, pidSource: currentRuntimeAdapterEvidence?.pidSource ?? persistedMember.pidSource, runtimeDiagnostic: currentRuntimeAdapterEvidence?.runtimeDiagnostic ?? persistedMember.runtimeDiagnostic, runtimeDiagnosticSeverity: persistedMember.runtimeDiagnosticSeverity, runtimeLastSeenAt: persistedMember.runtimeLastSeenAt ?? persistedMember.lastHeartbeatAt ?? persistedMember.lastRuntimeAliveAt, ...(activeRunModel ? { model: activeRunModel } : evidenceModel ? { model: evidenceModel } : persistedMember.model?.trim() ? { model: persistedMember.model.trim() } : {}), ...(typeof currentRuntimeAdapterEvidence?.runtimePid === 'number' && currentRuntimeAdapterEvidence.runtimePid > 0 ? { metricsPid: currentRuntimeAdapterEvidence.runtimePid } : typeof persistedMember.runtimePid === 'number' && persistedMember.runtimePid > 0 ? { metricsPid: persistedMember.runtimePid } : {}), ...(currentRuntimeAdapterEvidence?.sessionId ? { runtimeSessionId: currentRuntimeAdapterEvidence.sessionId } : persistedMember.runtimeSessionId ? { runtimeSessionId: persistedMember.runtimeSessionId } : {}), }); } const paneIds = [...metadataByMember.values()] .filter((metadata) => metadata.backendType === 'tmux' || metadata.backendType === undefined) .map((metadata) => metadata.tmuxPaneId?.trim() ?? '') .filter((paneId) => paneId.length > 0 && !paneId.startsWith('process:')); let paneInfoById = new Map(); if (paneIds.length > 0) { try { paneInfoById = await listTmuxPaneRuntimeInfoForCurrentPlatform(paneIds); } catch (error) { logger.debug( `[${teamName}] Failed to read tmux pane info for runtime snapshot: ${ error instanceof Error ? error.message : String(error) }` ); } } let processRows: RuntimeTelemetryProcessTableRow[] = []; let processTableAvailable = true; let processRowsReadForMetadata = false; const shouldReadProcessTable = this.shouldReadProcessTableForLiveRuntimeMetadata({ metadataByMember, launchSnapshot: persistedLaunchSnapshot, paneInfoById, }); if (shouldReadProcessTable) { const cachedRows = this.readCachedRuntimeProcessRowsForLiveRuntimeMetadata(teamName, runId); if (cachedRows) { processTableAvailable = cachedRows.rows !== null; processRows = cachedRows.rows ?? []; } else { processRowsReadForMetadata = true; try { processRows = this.normalizeRuntimeProcessRowsForTelemetry( await this.withRuntimeTelemetryTimeout( listRuntimeProcessTableForCurrentPlatform(), TeamProvisioningService.RUNTIME_PROCESS_TABLE_TIMEOUT_MS, 'process table runtime snapshot' ), process.platform === 'win32' ? 'wsl' : 'native' ) ?? []; } catch (error) { processTableAvailable = false; logger.debug( `[${teamName}] Failed to read process table for runtime snapshot: ${ error instanceof Error ? error.message : String(error) }` ); } } } if ( processRowsReadForMetadata && this.getRuntimeSnapshotCacheGeneration(teamName) === generationAtStart ) { const sampledAtMs = Date.now(); this.runtimeProcessRowsForUsageSnapshotByTeam.set(teamName, { expiresAtMs: sampledAtMs + (processTableAvailable ? TeamProvisioningService.RUNTIME_RESOURCE_TELEMETRY_CACHE_TTL_MS : TeamProvisioningService.RUNTIME_RESOURCE_TELEMETRY_FAILURE_CACHE_TTL_MS), generation: generationAtStart, runId, sampledAtMs, rows: processTableAvailable ? processRows : null, includesWindowsHostRows: false, }); } let windowsHostProcessRows: RuntimeTelemetryProcessTableRow[] | null = null; let windowsHostProcessTableAvailable = false; const getWindowsHostProcessRows = async (): Promise => { if (windowsHostProcessRows) { return windowsHostProcessRows; } try { windowsHostProcessRows = this.normalizeRuntimeProcessRowsForTelemetry( await this.withRuntimeTelemetryTimeout( listWindowsProcessTable( TeamProvisioningService.RUNTIME_WINDOWS_PROCESS_TABLE_TIMEOUT_MS ), TeamProvisioningService.RUNTIME_WINDOWS_PROCESS_TABLE_TIMEOUT_MS, 'Windows process table runtime snapshot' ), 'windows-host' ) ?? []; windowsHostProcessTableAvailable = true; } catch (error) { windowsHostProcessRows = []; logger.debug( `[${teamName}] Failed to read Windows host process table for runtime snapshot: ${ error instanceof Error ? error.message : String(error) }` ); } return windowsHostProcessRows; }; for (const [memberName, metadata] of metadataByMember.entries()) { const paneId = metadata.tmuxPaneId?.trim() ?? ''; const launchMember = persistedLaunchSnapshot?.members[memberName]; const adapterEvidence = currentRuntimeAdapterRun?.members?.[memberName]; const adapterStatus: MemberSpawnStatusEntry | undefined = adapterEvidence ? { status: adapterEvidence.hardFailure ? 'error' : adapterEvidence.bootstrapConfirmed ? 'online' : adapterEvidence.agentToolAccepted ? 'waiting' : 'spawning', launchState: adapterEvidence.launchState, ...(adapterEvidence.hardFailureReason ? { hardFailureReason: adapterEvidence.hardFailureReason } : {}), ...(adapterEvidence.pendingPermissionRequestIds?.length ? { pendingPermissionRequestIds: adapterEvidence.pendingPermissionRequestIds } : {}), agentToolAccepted: adapterEvidence.agentToolAccepted, runtimeAlive: adapterEvidence.runtimeAlive, bootstrapConfirmed: adapterEvidence.bootstrapConfirmed, hardFailure: adapterEvidence.hardFailure, ...(metadata.model ? { runtimeModel: metadata.model } : {}), ...(adapterEvidence.livenessKind ? { livenessKind: adapterEvidence.livenessKind } : {}), ...(adapterEvidence.runtimeDiagnostic ? { runtimeDiagnostic: adapterEvidence.runtimeDiagnostic } : {}), updatedAt: persistedLaunchSnapshot?.updatedAt ?? nowIso(), } : undefined; const shouldUseWindowsHostRows = process.platform === 'win32' && (metadata.providerId === 'opencode' || launchMember?.providerId === 'opencode' || metadata.backendType !== 'tmux') && currentRuntimeAdapterRun?.members?.[memberName]?.runtimeAlive !== true && currentRuntimeAdapterRun?.members?.[memberName]?.bootstrapConfirmed !== true; const hostProcessRows = shouldUseWindowsHostRows ? await getWindowsHostProcessRows() : []; const memberProcessRows = shouldUseWindowsHostRows ? [...hostProcessRows, ...processRows] : processRows; const memberProcessTableAvailable = shouldUseWindowsHostRows ? windowsHostProcessTableAvailable || processTableAvailable : processTableAvailable; const trackedStatus = this.findTrackedMemberSpawnStatus(run, memberName); const launchStatus = this.isLaunchMemberStatusRelevantToRuntimeRun(launchMember, activeRuntimeRunId) && launchMember ? this.buildLaunchMemberSpawnStatus(launchMember, metadata.model) : undefined; const status = this.shouldPreferCurrentLaunchMemberStatus(trackedStatus, launchStatus) ? launchStatus : this.shouldPreferCurrentLaunchMemberStatus(trackedStatus, adapterStatus) ? adapterStatus : (trackedStatus ?? adapterStatus ?? launchStatus); const resolved = resolveTeamMemberRuntimeLiveness({ teamName, memberName, agentId: metadata.agentId, backendType: metadata.backendType, providerId: metadata.providerId ?? launchMember?.providerId, tmuxPaneId: metadata.tmuxPaneId, persistedRuntimePid: launchMember?.runtimePid ?? metadata.metricsPid, persistedRuntimeSessionId: launchMember?.runtimeSessionId ?? metadata.runtimeSessionId, trackedSpawnStatus: status, runtimePid: metadata.metricsPid, runtimeSessionId: metadata.runtimeSessionId, pane: paneId ? paneInfoById.get(paneId) : undefined, processRows: memberProcessRows, processTableAvailable: memberProcessTableAvailable, nowIso: nowIso(), }); const bootstrapTransportDiagnostic = status?.runtimeDiagnostic ?? launchMember?.runtimeDiagnostic; const bootstrapTransportDiagnosticSeverity = status?.runtimeDiagnosticSeverity ?? launchMember?.runtimeDiagnosticSeverity; const bootstrapTransportLaunchState = status?.launchState ?? launchMember?.launchState; const bootstrapTransportConfirmed = status?.bootstrapConfirmed === true || launchMember?.bootstrapConfirmed === true; const hasProcessBootstrapTransportDiagnostic = (metadata.backendType === 'process' || metadata.tmuxPaneId?.startsWith('process:')) && !bootstrapTransportConfirmed && (bootstrapTransportLaunchState === 'runtime_pending_bootstrap' || bootstrapTransportLaunchState === 'failed_to_start') && isProcessBootstrapTransportDiagnostic(bootstrapTransportDiagnostic); // Prefer bootstrap transport diagnostics over generic pid/liveness text // while launch is unconfirmed, otherwise the UI hides the exact stage // where process bootstrap got stuck. const runtimeDiagnostic = hasProcessBootstrapTransportDiagnostic ? bootstrapTransportDiagnostic : resolved.runtimeDiagnostic; const runtimeDiagnosticSeverity = hasProcessBootstrapTransportDiagnostic ? (bootstrapTransportDiagnosticSeverity ?? resolved.runtimeDiagnosticSeverity) : resolved.runtimeDiagnosticSeverity; metadataByMember.set(memberName, { ...metadata, alive: resolved.alive, ...(typeof resolved.pid === 'number' && resolved.pid > 0 ? { pid: resolved.pid } : {}), ...(typeof (resolved.metricsPid ?? metadata.metricsPid) === 'number' && Number.isFinite(resolved.metricsPid ?? metadata.metricsPid) && (resolved.metricsPid ?? metadata.metricsPid)! > 0 ? { metricsPid: resolved.metricsPid ?? metadata.metricsPid } : {}), livenessKind: resolved.livenessKind, ...(resolved.pidSource ? { pidSource: resolved.pidSource } : {}), ...(resolved.processCommand ? { processCommand: resolved.processCommand } : {}), ...(resolved.panePid ? { panePid: resolved.panePid } : {}), ...(resolved.paneCurrentCommand ? { paneCurrentCommand: resolved.paneCurrentCommand } : {}), ...(resolved.runtimeSessionId ? { runtimeSessionId: resolved.runtimeSessionId } : {}), ...(resolved.runtimeLastSeenAt ? { runtimeLastSeenAt: resolved.runtimeLastSeenAt } : {}), runtimeDiagnostic, runtimeDiagnosticSeverity, diagnostics: hasProcessBootstrapTransportDiagnostic ? mergeRuntimeDiagnostics(resolved.diagnostics, [bootstrapTransportDiagnostic]) : resolved.diagnostics, }); } if ( this.getRuntimeSnapshotCacheGeneration(teamName) === generationAtStart && this.getTrackedRunId(teamName) === runId ) { this.liveTeamAgentRuntimeMetadataCache.set(teamName, { expiresAtMs: Date.now() + this.getAgentRuntimeSnapshotCacheTtlMs(teamName, runId), metadata: this.cloneLiveTeamAgentRuntimeMetadata(metadataByMember), runId, }); } return metadataByMember; } private buildAgentRuntimeResourceHistoryKey(params: { memberName: string; pid?: number; runtimePid?: number; pidSource?: TeamAgentRuntimePidSource; }): string | null { const memberName = params.memberName.trim(); const usagePid = typeof params.pid === 'number' && Number.isFinite(params.pid) && params.pid > 0 ? params.pid : typeof params.runtimePid === 'number' && Number.isFinite(params.runtimePid) && params.runtimePid > 0 ? params.runtimePid : null; if (!memberName || usagePid == null) { return null; } return [memberName, usagePid, params.pidSource ?? 'unknown'].join('\0'); } private recordAgentRuntimeResourceSample(params: { teamName: string; memberName: string; timestamp: string; cpuPercent?: number; rssBytes?: number; primaryCpuPercent?: number; primaryRssBytes?: number; childCpuPercent?: number; childRssBytes?: number; processCount?: number; runtimeLoadScope?: TeamAgentRuntimeLoadScope; runtimeLoadTruncated?: boolean; pidSource?: TeamAgentRuntimePidSource; pid?: number; runtimePid?: number; activeKeys?: Set; }): TeamAgentRuntimeResourceSample[] | undefined { const key = this.buildAgentRuntimeResourceHistoryKey(params); if (!key) { return undefined; } params.activeKeys?.add(key); const cpuPercent = typeof params.cpuPercent === 'number' && Number.isFinite(params.cpuPercent) && params.cpuPercent >= 0 ? params.cpuPercent : undefined; const rssBytes = typeof params.rssBytes === 'number' && Number.isFinite(params.rssBytes) && params.rssBytes >= 0 ? params.rssBytes : undefined; const primaryCpuPercent = typeof params.primaryCpuPercent === 'number' && Number.isFinite(params.primaryCpuPercent) && params.primaryCpuPercent >= 0 ? params.primaryCpuPercent : undefined; const primaryRssBytes = typeof params.primaryRssBytes === 'number' && Number.isFinite(params.primaryRssBytes) && params.primaryRssBytes >= 0 ? params.primaryRssBytes : undefined; const childCpuPercent = typeof params.childCpuPercent === 'number' && Number.isFinite(params.childCpuPercent) && params.childCpuPercent >= 0 ? params.childCpuPercent : undefined; const childRssBytes = typeof params.childRssBytes === 'number' && Number.isFinite(params.childRssBytes) && params.childRssBytes >= 0 ? params.childRssBytes : undefined; const processCount = typeof params.processCount === 'number' && Number.isFinite(params.processCount) && params.processCount > 0 ? Math.floor(params.processCount) : undefined; let historyByKey = this.agentRuntimeResourceHistoryByTeam.get(params.teamName); if (!historyByKey) { historyByKey = new Map(); this.agentRuntimeResourceHistoryByTeam.set(params.teamName, historyByKey); } const existingHistory = historyByKey.get(key) ?? []; if (cpuPercent == null && rssBytes == null) { return existingHistory.length > 0 ? existingHistory.map((sample) => ({ ...sample })) : undefined; } const sample: TeamAgentRuntimeResourceSample = { timestamp: params.timestamp, ...(cpuPercent != null ? { cpuPercent } : {}), ...(rssBytes != null ? { rssBytes } : {}), ...(primaryCpuPercent != null ? { primaryCpuPercent } : {}), ...(primaryRssBytes != null ? { primaryRssBytes } : {}), ...(childCpuPercent != null ? { childCpuPercent } : {}), ...(childRssBytes != null ? { childRssBytes } : {}), ...(processCount != null ? { processCount } : {}), ...(params.runtimeLoadScope ? { runtimeLoadScope: params.runtimeLoadScope } : {}), ...(params.runtimeLoadTruncated ? { runtimeLoadTruncated: true } : {}), ...(params.pidSource ? { pidSource: params.pidSource } : {}), ...(params.pid ? { pid: params.pid } : {}), ...(params.runtimePid ? { runtimePid: params.runtimePid } : {}), }; const lastSample = existingHistory.at(-1); const lastSampleMs = lastSample ? Date.parse(lastSample.timestamp) : Number.NaN; const sampleMs = Date.parse(sample.timestamp); const sampledRecently = Number.isFinite(lastSampleMs) && Number.isFinite(sampleMs) && sampleMs - lastSampleMs >= 0 && sampleMs - lastSampleMs < TeamProvisioningService.RUNTIME_RESOURCE_SAMPLE_MIN_INTERVAL_MS; if (sampledRecently) { return existingHistory.map((entry) => ({ ...entry })); } const nextHistory = [...existingHistory, sample].slice( -TeamProvisioningService.AGENT_RUNTIME_RESOURCE_HISTORY_LIMIT ); historyByKey.set(key, nextHistory); return nextHistory.map((entry) => ({ ...entry })); } private recordAgentRuntimeResourceSampleSafely(params: { teamName: string; memberName: string; timestamp: string; cpuPercent?: number; rssBytes?: number; primaryCpuPercent?: number; primaryRssBytes?: number; childCpuPercent?: number; childRssBytes?: number; processCount?: number; runtimeLoadScope?: TeamAgentRuntimeLoadScope; runtimeLoadTruncated?: boolean; pidSource?: TeamAgentRuntimePidSource; pid?: number; runtimePid?: number; activeKeys?: Set; }): TeamAgentRuntimeResourceSample[] | undefined { try { return this.recordAgentRuntimeResourceSample(params); } catch (error) { logger.debug( `[${params.teamName}] Failed to record runtime telemetry sample for ${ params.memberName }; continuing without history: ${error instanceof Error ? error.message : String(error)}` ); return undefined; } } private pruneAgentRuntimeResourceHistory( teamName: string, activeKeys: ReadonlySet ): void { const historyByKey = this.agentRuntimeResourceHistoryByTeam.get(teamName); if (!historyByKey) { return; } for (const key of historyByKey.keys()) { if (!activeKeys.has(key)) { historyByKey.delete(key); } } if (historyByKey.size === 0) { this.agentRuntimeResourceHistoryByTeam.delete(teamName); } } private normalizeRuntimeProcessUsageStats(stat: unknown): RuntimeProcessUsageStats | undefined { if (!stat || typeof stat !== 'object') { return undefined; } const candidate = stat as { memory?: unknown; cpu?: unknown; rssBytes?: unknown; cpuPercent?: unknown; }; const rssBytes = normalizeRuntimeTelemetryNumber(candidate.memory) ?? normalizeRuntimeTelemetryNumber(candidate.rssBytes); const cpuPercent = normalizeRuntimeTelemetryNumber(candidate.cpu) ?? normalizeRuntimeTelemetryNumber(candidate.cpuPercent); const normalized: RuntimeProcessUsageStats = { ...(rssBytes != null && rssBytes >= 0 ? { rssBytes } : {}), ...(cpuPercent != null && cpuPercent >= 0 ? { cpuPercent } : {}), }; return normalized.rssBytes != null || normalized.cpuPercent != null ? normalized : undefined; } private normalizeRuntimeProcessRowsForTelemetry( rows: unknown, source?: RuntimeTelemetryProcessSource ): RuntimeTelemetryProcessTableRow[] | null { if (!Array.isArray(rows)) { return null; } const normalizedRows: RuntimeTelemetryProcessTableRow[] = []; for (const row of rows) { if (!row || typeof row !== 'object') { continue; } const candidate = row as Partial; const pid = normalizeRuntimeTelemetryNumber(candidate.pid); const ppid = normalizeRuntimeTelemetryNumber(candidate.ppid); const command = typeof candidate.command === 'string' ? candidate.command.trim() : ''; if (pid != null && pid > 0 && ppid != null && ppid >= 0 && command.length > 0) { const runtimeTelemetrySource = source ?? (candidate.runtimeTelemetrySource === 'native' || candidate.runtimeTelemetrySource === 'wsl' || candidate.runtimeTelemetrySource === 'windows-host' ? candidate.runtimeTelemetrySource : undefined); const usageStats = this.normalizeRuntimeProcessUsageStats(candidate); normalizedRows.push({ pid: Math.floor(pid), ppid: Math.floor(ppid), command, ...(usageStats ?? {}), ...(runtimeTelemetrySource ? { runtimeTelemetrySource } : {}), }); } } return normalizedRows; } private shouldReadProcessTableForLiveRuntimeMetadata(params: { metadataByMember: ReadonlyMap; launchSnapshot: PersistedTeamLaunchSnapshot | null | undefined; paneInfoById: ReadonlyMap; }): boolean { for (const [memberName, metadata] of params.metadataByMember.entries()) { if (metadata.agentId?.trim()) { return true; } const paneId = metadata.tmuxPaneId?.trim() ?? ''; if (paneId && params.paneInfoById.has(paneId)) { return true; } const launchRuntimePid = params.launchSnapshot?.members[memberName]?.runtimePid; if ( (typeof metadata.metricsPid === 'number' && Number.isFinite(metadata.metricsPid) && metadata.metricsPid > 0) || (typeof launchRuntimePid === 'number' && Number.isFinite(launchRuntimePid) && launchRuntimePid > 0) ) { return true; } } return false; } private readCachedRuntimeProcessRowsForLiveRuntimeMetadata( teamName: string, runId: string | null ): { rows: RuntimeTelemetryProcessTableRow[] | null } | null { const cached = this.runtimeProcessRowsForUsageSnapshotByTeam.get(teamName); const nowMs = Date.now(); if (!cached || cached.expiresAtMs <= nowMs || cached.runId !== runId) { return null; } // Process table rows are sampled global OS state. Do not tie reuse to // runtime snapshot generation: launch progress invalidates runtime metadata // frequently, and the age gate below keeps liveness freshness bounded. const sampledAtMs = typeof cached.sampledAtMs === 'number' && Number.isFinite(cached.sampledAtMs) ? cached.sampledAtMs : 0; const maxAgeMs = cached.rows === null ? TeamProvisioningService.RUNTIME_LIVENESS_PROCESS_TABLE_FAILURE_CACHE_TTL_MS : TeamProvisioningService.RUNTIME_LIVENESS_PROCESS_TABLE_CACHE_TTL_MS; if (sampledAtMs <= 0 || nowMs - sampledAtMs > maxAgeMs) { return null; } if (cached.rows === null) { return { rows: null }; } const rows = this.normalizeRuntimeProcessRowsForTelemetry(cached.rows)?.filter( (row) => row.runtimeTelemetrySource !== 'windows-host' ) ?? []; return { rows }; } private async readRuntimeProcessRowsForUsageSnapshot( teamName: string, options: { includeWindowsHostRows?: boolean } = {} ): Promise { const includeWindowsHostRows = process.platform === 'win32' && options.includeWindowsHostRows === true; const cached = this.runtimeProcessRowsForUsageSnapshotByTeam.get(teamName); const canUseCached = cached && cached.expiresAtMs > Date.now() && cached.runId === this.getTrackedRunId(teamName); if (canUseCached && (!includeWindowsHostRows || cached.includesWindowsHostRows)) { return cached.rows; } let rows = canUseCached && cached.rows ? cached.rows : null; let runtimeProcessTableAvailable = rows != null; try { if (!rows) { rows = this.normalizeRuntimeProcessRowsForTelemetry( await this.withRuntimeTelemetryTimeout( listRuntimeProcessTableForCurrentPlatform(), TeamProvisioningService.RUNTIME_PROCESS_TABLE_TIMEOUT_MS, 'process table runtime telemetry' ), process.platform === 'win32' ? 'wsl' : 'native' ) ?? []; runtimeProcessTableAvailable = true; } } catch (error) { logger.debug( `[${teamName}] Failed to read process table for runtime usage snapshot: ${ error instanceof Error ? error.message : String(error) }` ); rows = null; runtimeProcessTableAvailable = false; } let includesWindowsHostRows = false; if (includeWindowsHostRows) { try { const windowsHostRows = await this.withRuntimeTelemetryTimeout( listWindowsProcessTable(TeamProvisioningService.RUNTIME_WINDOWS_PROCESS_TABLE_TIMEOUT_MS), TeamProvisioningService.RUNTIME_WINDOWS_PROCESS_TABLE_TIMEOUT_MS, 'Windows process table runtime telemetry' ); rows = [ ...(rows ?? []), ...(this.normalizeRuntimeProcessRowsForTelemetry(windowsHostRows, 'windows-host') ?? []), ]; includesWindowsHostRows = true; } catch (error) { logger.debug( `[${teamName}] Failed to read Windows host process table for runtime usage snapshot: ${ error instanceof Error ? error.message : String(error) }` ); } } const resultRows = rows && rows.length > 0 ? rows : runtimeProcessTableAvailable ? [] : null; const sampledAtMs = Date.now(); this.runtimeProcessRowsForUsageSnapshotByTeam.set(teamName, { expiresAtMs: sampledAtMs + (resultRows === null ? TeamProvisioningService.RUNTIME_RESOURCE_TELEMETRY_FAILURE_CACHE_TTL_MS : TeamProvisioningService.RUNTIME_RESOURCE_TELEMETRY_CACHE_TTL_MS), generation: this.getRuntimeSnapshotCacheGeneration(teamName), runId: this.getTrackedRunId(teamName), sampledAtMs, rows: resultRows, includesWindowsHostRows, }); return resultRows; } private buildRuntimeProcessChildrenByParent( processRows: readonly RuntimeTelemetryProcessTableRow[] ): Map { const childrenByParent = new Map(); for (const row of processRows) { const current = childrenByParent.get(row.ppid) ?? []; current.push(row); childrenByParent.set(row.ppid, current); } return childrenByParent; } private addRuntimeRootOwnersFromProcessRows( teamName: string, processRows: readonly RuntimeTelemetryProcessTableRow[] | null, rootOwnersByPid: Map> ): void { if (!processRows || processRows.length === 0) { return; } for (const row of processRows) { if (process.platform === 'win32' && row.runtimeTelemetrySource === 'wsl') { continue; } if (!commandArgEquals(row.command, '--team-name', teamName)) { continue; } const agentNames = extractCliArgValues(row.command, '--agent-name'); const agentIds = extractCliArgValues(row.command, '--agent-id'); const ownerKey = agentNames.find((value) => value.trim().length > 0)?.trim() ?? agentIds .map((value) => value.split('@', 1)[0]?.trim() ?? '') .find((value) => value.length > 0); if (!ownerKey) { continue; } const owners = rootOwnersByPid.get(row.pid) ?? new Set(); owners.add(ownerKey); rootOwnersByPid.set(row.pid, owners); } } private buildRuntimeUsageProcessTrees( rootPids: readonly number[], processRows: readonly RuntimeTelemetryProcessTableRow[] | null, rootOwnersByPid?: ReadonlyMap> ): Map { const uniqueRoots = [...new Set(rootPids.filter((pid) => Number.isFinite(pid) && pid > 0))]; const rootOwnerKeysByPid = new Map>(); for (const [pid, owners] of rootOwnersByPid ?? []) { if (Number.isFinite(pid) && pid > 0 && owners.size > 0) { rootOwnerKeysByPid.set(pid, owners); } } for (const rootPid of uniqueRoots) { if (!rootOwnerKeysByPid.has(rootPid)) { rootOwnerKeysByPid.set(rootPid, new Set([String(rootPid)])); } } const haveSameRootOwners = ( left: ReadonlySet | undefined, right: ReadonlySet | undefined ): boolean => { if (!left || left.size !== right?.size) { return false; } for (const value of left) { if (!right.has(value)) { return false; } } return true; }; const usageTreesByRootPid = new Map(); const scheduledPids = new Set(); const normalizedProcessRows = this.normalizeRuntimeProcessRowsForTelemetry(processRows); if (!normalizedProcessRows || normalizedProcessRows.length === 0) { for (const rootPid of uniqueRoots) { if (scheduledPids.size >= TeamProvisioningService.MAX_RUNTIME_USAGE_PIDS_PER_SNAPSHOT) { usageTreesByRootPid.set(rootPid, { pids: [], truncated: true }); continue; } scheduledPids.add(rootPid); usageTreesByRootPid.set(rootPid, { pids: [rootPid], truncated: false }); } return usageTreesByRootPid; } const childrenByParent = this.buildRuntimeProcessChildrenByParent(normalizedProcessRows); const rowByPid = new Map(normalizedProcessRows.map((row) => [row.pid, row])); const missingRootPids: number[] = []; for (const rootPid of uniqueRoots) { const pids: number[] = []; let truncated = false; const rootProcessRow = rowByPid.get(rootPid); if (!rootProcessRow) { missingRootPids.push(rootPid); usageTreesByRootPid.set(rootPid, { pids: [], truncated: false }); continue; } if (process.platform === 'win32' && rootProcessRow?.runtimeTelemetrySource === 'wsl') { usageTreesByRootPid.set(rootPid, { pids: [], truncated: false }); continue; } const rootProcessSource = rootProcessRow?.runtimeTelemetrySource; const addPid = (pid: number): boolean => { if (pids.includes(pid)) { return true; } if ( pids.length >= TeamProvisioningService.MAX_RUNTIME_TREE_PIDS_PER_ROOT || (!scheduledPids.has(pid) && scheduledPids.size >= TeamProvisioningService.MAX_RUNTIME_USAGE_PIDS_PER_SNAPSHOT) ) { truncated = true; return false; } pids.push(pid); scheduledPids.add(pid); return true; }; if (!addPid(rootPid)) { usageTreesByRootPid.set(rootPid, { pids, truncated }); continue; } const queue = [...(childrenByParent.get(rootPid) ?? [])]; const seen = new Set(); const rootOwners = rootOwnerKeysByPid.get(rootPid); while (queue.length > 0) { const row = queue.shift(); if (!row || seen.has(row.pid)) { continue; } seen.add(row.pid); if ( rootProcessSource && row.runtimeTelemetrySource && row.runtimeTelemetrySource !== rootProcessSource ) { continue; } const candidateRootOwners = rootOwnerKeysByPid.get(row.pid); if ( row.pid !== rootPid && candidateRootOwners && !haveSameRootOwners(rootOwners, candidateRootOwners) ) { continue; } if (!addPid(row.pid)) { break; } queue.push(...(childrenByParent.get(row.pid) ?? [])); } if (queue.length > 0) { truncated = true; } usageTreesByRootPid.set(rootPid, { pids, truncated }); } for (const rootPid of missingRootPids) { if (scheduledPids.size >= TeamProvisioningService.MAX_RUNTIME_USAGE_PIDS_PER_SNAPSHOT) { usageTreesByRootPid.set(rootPid, { pids: [], truncated: true }); continue; } scheduledPids.add(rootPid); usageTreesByRootPid.set(rootPid, { pids: [rootPid], truncated: false }); } return usageTreesByRootPid; } private buildRuntimeProcessLoadStats(params: { rootPid: number | undefined; usageStatsByPid: ReadonlyMap; processTree?: { pids: number[]; truncated: boolean }; scope?: TeamAgentRuntimeLoadScope; }): RuntimeProcessLoadStats | undefined { const rootPid = typeof params.rootPid === 'number' && Number.isFinite(params.rootPid) && params.rootPid > 0 ? params.rootPid : undefined; if (!rootPid) { return undefined; } if (params.processTree && params.processTree.pids.length === 0) { return undefined; } const processPids = params.processTree ? params.processTree.pids : [rootPid]; const primaryStats = params.usageStatsByPid.get(rootPid); let childCpuPercent = 0; let childRssBytes = 0; let hasChildCpu = false; let hasChildRss = false; let sampledProcessCount = primaryStats ? 1 : 0; for (const pid of processPids) { if (pid === rootPid) { continue; } const childStats = params.usageStatsByPid.get(pid); if (!childStats) { continue; } sampledProcessCount += 1; if (typeof childStats.cpuPercent === 'number') { childCpuPercent += childStats.cpuPercent; hasChildCpu = true; } if (typeof childStats.rssBytes === 'number') { childRssBytes += childStats.rssBytes; hasChildRss = true; } } if (!primaryStats && sampledProcessCount === 0) { return undefined; } const primaryCpuPercent = primaryStats?.cpuPercent; const primaryRssBytes = primaryStats?.rssBytes; const cpuPercent = primaryCpuPercent != null || hasChildCpu ? (primaryCpuPercent ?? 0) + childCpuPercent : undefined; const rssBytes = primaryRssBytes != null || hasChildRss ? (primaryRssBytes ?? 0) + childRssBytes : undefined; const hasSampledChildren = hasChildCpu || hasChildRss; const hasProcessTree = processPids.length > 1 && hasSampledChildren; const runtimeLoadScope = params.scope ?? (hasProcessTree ? 'process-tree' : ('single-process' as const)); return { ...(rssBytes != null ? { rssBytes } : {}), ...(cpuPercent != null ? { cpuPercent } : {}), ...(primaryCpuPercent != null ? { primaryCpuPercent } : {}), ...(primaryRssBytes != null ? { primaryRssBytes } : {}), ...(hasChildCpu ? { childCpuPercent } : {}), ...(hasChildRss ? { childRssBytes } : {}), ...(sampledProcessCount > 0 ? { processCount: sampledProcessCount } : {}), runtimeLoadScope, ...(params.processTree?.truncated ? { runtimeLoadTruncated: true } : {}), }; } private buildRuntimeProcessLoadStatsSafely( teamName: string, memberName: string, params: { rootPid: number | undefined; usageStatsByPid: ReadonlyMap; processTree?: { pids: number[]; truncated: boolean }; scope?: TeamAgentRuntimeLoadScope; } ): RuntimeProcessLoadStats | undefined { try { return this.buildRuntimeProcessLoadStats(params); } catch (error) { logger.debug( `[${teamName}] Failed to build runtime telemetry stats for ${memberName}; continuing without metrics: ${ error instanceof Error ? error.message : String(error) }` ); return undefined; } } private async withRuntimeTelemetryTimeout( promise: Promise, timeoutMs: number, label: string ): Promise { let timeout: NodeJS.Timeout | undefined; try { return await Promise.race([ promise, new Promise((_resolve, reject) => { timeout = setTimeout(() => { reject(new RuntimeTelemetryTimeoutError(`${label} timed out after ${timeoutMs}ms`)); }, timeoutMs); timeout.unref?.(); }), ]); } finally { if (timeout) { clearTimeout(timeout); } } } private buildProcessUsageStatsFromRows( processRows: readonly RuntimeTelemetryProcessTableRow[] | null, pids: readonly number[] ): Map { const usageStatsByPid = new Map(); const requestedPids = new Set(pids.filter((pid) => Number.isFinite(pid) && pid > 0)); if (!Array.isArray(processRows) || requestedPids.size === 0) { return usageStatsByPid; } for (const row of processRows) { if (!requestedPids.has(row.pid)) { continue; } const usageStats = this.normalizeRuntimeProcessUsageStats(row); if (usageStats) { usageStatsByPid.set(row.pid, usageStats); } } return usageStatsByPid; } private shouldSampleMissingRuntimeUsageStatsWithPidusage(): boolean { // CPU/RSS telemetry already comes from the enriched process table in the // default path. If this opt-in is enabled, preserve the older fallback for // missing rows across platforms. return this.isRuntimePidusageTelemetryEnabled(); } private isRuntimePidusageTelemetryEnabled(): boolean { const value = process.env.CLAUDE_TEAM_RUNTIME_PIDUSAGE_ENABLED?.trim().toLowerCase(); return value === '1' || value === 'true' || value === 'yes'; } private async readProcessUsageStatsByPid( pids: readonly number[], cacheOptions: { ignoreCachedMisses?: boolean } = {} ): Promise> { const pidCandidates = Array.isArray(pids) ? pids : []; const uniquePids = [...new Set(pidCandidates.filter((pid) => Number.isFinite(pid) && pid > 0))]; if (uniquePids.length === 0) { return new Map(); } const usageStatsByPid = new Map(); const pidsToRead: number[] = []; const now = Date.now(); for (const pid of uniquePids) { const cached = this.runtimeProcessUsageStatsCacheByPid.get(pid); if (cached && cached.expiresAtMs > now) { if (cached.stats) { usageStatsByPid.set(pid, { ...cached.stats }); continue; } if (!cacheOptions.ignoreCachedMisses) { continue; } } if (cached) { this.runtimeProcessUsageStatsCacheByPid.delete(pid); } pidsToRead.push(pid); } if (pidsToRead.length === 0) { return usageStatsByPid; } if (!this.isRuntimePidusageTelemetryEnabled()) { return usageStatsByPid; } const rememberUsageStats = ( pid: number, stats: RuntimeProcessUsageStats | null | undefined ): void => { const normalized = stats ? { ...stats } : null; const nowMs = Date.now(); for (const [cachedPid, cached] of this.runtimeProcessUsageStatsCacheByPid) { if (cached.expiresAtMs <= nowMs) { this.runtimeProcessUsageStatsCacheByPid.delete(cachedPid); } } while ( !this.runtimeProcessUsageStatsCacheByPid.has(pid) && this.runtimeProcessUsageStatsCacheByPid.size >= TeamProvisioningService.RUNTIME_PROCESS_USAGE_CACHE_MAX_ENTRIES ) { const oldestPid = this.runtimeProcessUsageStatsCacheByPid.keys().next().value; if (oldestPid == null) { break; } this.runtimeProcessUsageStatsCacheByPid.delete(oldestPid); } this.runtimeProcessUsageStatsCacheByPid.set(pid, { expiresAtMs: nowMs + TeamProvisioningService.RUNTIME_PROCESS_USAGE_CACHE_TTL_MS, stats: normalized, }); if (normalized) { usageStatsByPid.set(pid, { ...normalized }); } }; const options = RUNTIME_PIDUSAGE_OPTIONS; try { const statsByPid = await this.withRuntimeTelemetryTimeout( pidusage(pidsToRead, options), TeamProvisioningService.RUNTIME_PIDUSAGE_BATCH_TIMEOUT_MS, 'pidusage batch runtime telemetry' ); const observedPids = new Set(); for (const [rawPid, stat] of Object.entries( statsByPid && typeof statsByPid === 'object' ? statsByPid : {} )) { const pid = Number.parseInt(rawPid, 10); const usageStats = this.normalizeRuntimeProcessUsageStats(stat); if (Number.isFinite(pid) && pid > 0) { observedPids.add(pid); rememberUsageStats(pid, usageStats); } } for (const pid of pidsToRead) { if (!observedPids.has(pid)) { rememberUsageStats(pid, null); } } return usageStatsByPid; } catch (error) { if (error instanceof RuntimeTelemetryTimeoutError) { logger.debug(`${error.message}; continuing without runtime resource metrics`); return usageStatsByPid; } logger.debug( `pidusage batch runtime snapshot failed; falling back to per-pid reads: ${ error instanceof Error ? error.message : String(error) }` ); } for ( let offset = 0; offset < pidsToRead.length; offset += TeamProvisioningService.RUNTIME_PIDUSAGE_FALLBACK_CONCURRENCY ) { const chunk = pidsToRead.slice( offset, offset + TeamProvisioningService.RUNTIME_PIDUSAGE_FALLBACK_CONCURRENCY ); await Promise.all( chunk.map(async (pid) => { try { const stat = await this.withRuntimeTelemetryTimeout( pidusage(pid, options), TeamProvisioningService.RUNTIME_PIDUSAGE_SINGLE_TIMEOUT_MS, `pidusage runtime telemetry pid=${pid}` ); const usageStats = this.normalizeRuntimeProcessUsageStats(stat); rememberUsageStats(pid, usageStats); } catch (error) { if (error instanceof RuntimeTelemetryTimeoutError) { logger.debug(error.message); } rememberUsageStats(pid, null); // Process likely exited between discovery and sampling. } }) ); } return usageStatsByPid; } private async clearPersistedLaunchState( teamName: string, options?: { expectedRunId?: string } ): Promise { await this.enqueueLaunchStateStoreOperation(teamName, () => this.clearPersistedLaunchStateNow(teamName, options) ); } private canClearPersistedLaunchStateForRun( teamName: string, expectedRunId: string | undefined ): boolean { if (!expectedRunId) { return true; } const trackedRunId = this.getTrackedRunId(teamName); if (trackedRunId !== expectedRunId) { return false; } const lastWrittenRunId = this.launchStateWrittenRunIdByTeam.get(teamName); if (lastWrittenRunId && lastWrittenRunId !== expectedRunId) { return false; } return true; } private async clearPersistedLaunchStateNow( teamName: string, options?: { expectedRunId?: string } ): Promise { if (!this.canClearPersistedLaunchStateForRun(teamName, options?.expectedRunId)) { logger.debug( `[${teamName}] Skipping stale launch-state clear for run ${options?.expectedRunId}` ); return; } await this.launchStateStore.clear(teamName); this.launchStateWrittenRunIdByTeam.delete(teamName); await clearBootstrapState(teamName); this.invalidateRuntimeSnapshotCaches(teamName); } private async applyOpenCodeSecondaryEvidenceOverlay(params: { teamName: string; snapshot: PersistedTeamLaunchSnapshot; previousSnapshot?: PersistedTeamLaunchSnapshot | null; metaMembers?: Awaited>; }): Promise { const candidates = this.collectOpenCodeSecondaryOverlayCandidates( params.snapshot, params.previousSnapshot ?? null ); if (candidates.length === 0) { return params.snapshot; } const laneIndex = await readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), params.teamName).catch( () => null ); let changed = false; const nextMembers: Record = { ...params.snapshot.members, }; const metaMembers = params.metaMembers ?? []; for (const memberName of candidates) { const current = nextMembers[memberName]; const previous = params.previousSnapshot?.members[memberName] ?? null; const baseMember = current ?? previous; if (!baseMember || !isPersistedOpenCodeSecondaryLaneMember(baseMember)) { continue; } const laneId = baseMember.laneId?.trim(); if (!laneId) { continue; } const laneEntry = laneIndex?.lanes[laneId] ?? null; const evidence = await readCommittedOpenCodeBootstrapSessionEvidence({ teamsBasePath: getTeamsBasePath(), teamName: params.teamName, laneId, }).catch((error: unknown) => ({ state: 'invalid_store' as const, committed: false, activeRunId: null, sessions: [], diagnostics: [ `OpenCode committed bootstrap evidence read failed: ${getErrorMessage(error)}`, ], })); const decision = await this.classifyOpenCodeSecondaryEvidenceOverlay({ teamName: params.teamName, memberName, current: baseMember, previous, laneEntry, metaMembers, activeRunId: evidence.activeRunId, sessions: evidence.committed ? evidence.sessions : [], diagnostics: evidence.diagnostics, }); if (decision.kind !== 'confirmed_bootstrap') { continue; } const promoted = promoteOpenCodeSecondaryMemberFromCommittedBootstrapEvidence({ current: baseMember, previous, session: decision.session, now: nowIso(), }); if (!current || JSON.stringify(promoted) !== JSON.stringify(current)) { nextMembers[memberName] = promoted; changed = true; } } if (!changed) { return params.snapshot; } return createPersistedLaunchSnapshot({ teamName: params.snapshot.teamName, expectedMembers: params.snapshot.expectedMembers, bootstrapExpectedMembers: params.snapshot.bootstrapExpectedMembers, leadSessionId: params.snapshot.leadSessionId, launchPhase: params.snapshot.launchPhase, members: nextMembers, updatedAt: nowIso(), }); } private hasCommittedOpenCodeSecondaryEvidenceOverlayDelta( snapshot: PersistedTeamLaunchSnapshot | null, previousSnapshot: PersistedTeamLaunchSnapshot | null ): boolean { if (!snapshot) { return false; } return Object.entries(snapshot.members).some(([memberName, member]) => { if (!member.diagnostics?.includes('opencode_bootstrap_evidence_committed')) { return false; } const previous = previousSnapshot?.members[memberName]; return ( previous?.launchState !== member.launchState || previous?.bootstrapConfirmed !== member.bootstrapConfirmed || previous?.runtimeSessionId !== member.runtimeSessionId || previous?.livenessKind !== member.livenessKind ); }); } private collectOpenCodeSecondaryOverlayCandidates( snapshot: PersistedTeamLaunchSnapshot, previousSnapshot: PersistedTeamLaunchSnapshot | null ): string[] { const names = new Set(); const allNames = new Set([ ...Object.keys(snapshot.members), ...Object.keys(previousSnapshot?.members ?? {}), ]); for (const memberName of allNames) { const current = snapshot.members[memberName]; const previous = previousSnapshot?.members[memberName]; const candidate = current ?? previous; if (!isPersistedOpenCodeSecondaryLaneMember(candidate)) { continue; } if (!current || this.needsOpenCodeSecondaryEvidenceOverlay(current, previous ?? null)) { names.add(memberName); } } return [...names].sort((left, right) => left.localeCompare(right)); } private needsOpenCodeSecondaryEvidenceOverlay( current: PersistedTeamLaunchMemberState, previous: PersistedTeamLaunchMemberState | null ): boolean { if (current.launchState === 'confirmed_alive' && current.bootstrapConfirmed) { return ( current.livenessKind !== 'confirmed_bootstrap' && current.livenessKind !== 'runtime_process' ); } if ( previous?.launchState === 'confirmed_alive' && previous.bootstrapConfirmed && current.launchState !== 'confirmed_alive' ) { return true; } if ( current.launchState === 'starting' || current.launchState === 'runtime_pending_bootstrap' || current.launchState === 'runtime_pending_permission' ) { return true; } return ( current.launchState === 'failed_to_start' && hasStaleOpenCodeSecondaryLaunchDiagnostic(current) ); } private async classifyOpenCodeSecondaryEvidenceOverlay(params: { teamName: string; memberName: string; current: PersistedTeamLaunchMemberState; previous: PersistedTeamLaunchMemberState | null; laneEntry: OpenCodeRuntimeLaneIndexEntry | null; metaMembers: Awaited>; activeRunId: string | null; sessions: OpenCodeCommittedBootstrapSessionRecord[]; diagnostics: readonly string[]; }): Promise< | { kind: 'blocked' | 'none' | 'ambiguous' | 'conflict'; diagnostics: string[] } | { kind: 'confirmed_bootstrap'; session: OpenCodeCommittedBootstrapSessionRecord } > { if (isOpenCodeOverlayMemberRemoved(params.metaMembers, params.memberName)) { return { kind: 'blocked', diagnostics: ['opencode_overlay_member_removed'] }; } if (params.laneEntry?.state === 'stopped') { return { kind: 'blocked', diagnostics: ['opencode_overlay_lane_stopped'] }; } if (hasRealOpenCodeLaunchDiagnostic(params.current)) { return { kind: 'blocked', diagnostics: ['opencode_overlay_real_failure_preserved'] }; } if ( params.current.launchState === 'failed_to_start' && !hasStaleOpenCodeSecondaryLaunchDiagnostic(params.current) ) { return { kind: 'blocked', diagnostics: ['opencode_overlay_real_failure_preserved'] }; } if ( params.laneEntry?.state === 'degraded' && !hasStaleOpenCodeSecondaryLaunchDiagnostic(params.current) && !hasStaleOpenCodeDiagnostics(params.laneEntry.diagnostics) ) { return { kind: 'blocked', diagnostics: ['opencode_overlay_degraded_lane_preserved'] }; } const memberSessions = params.sessions.filter((session) => namesMatchCaseInsensitive(session.memberName, params.memberName) ); if (memberSessions.length === 0) { return { kind: 'none', diagnostics: [...params.diagnostics, 'opencode_overlay_no_session'] }; } const expectedSessionId = params.current.runtimeSessionId?.trim() || params.previous?.runtimeSessionId?.trim() || ''; const selected = expectedSessionId ? memberSessions.find((session) => session.id === expectedSessionId) : memberSessions.length === 1 ? memberSessions[0] : null; if (!selected) { return { kind: expectedSessionId ? 'conflict' : 'ambiguous', diagnostics: [ expectedSessionId ? 'opencode_overlay_session_conflict' : 'opencode_overlay_ambiguous_sessions', ], }; } if ( params.activeRunId && selected.runId && params.activeRunId.trim() !== selected.runId.trim() ) { return { kind: 'conflict', diagnostics: ['opencode_overlay_session_run_mismatch'], }; } if (selected.runId) { const tombstoneStore = createRuntimeRunTombstoneStore({ filePath: getOpenCodeRuntimeRunTombstonesPath( getTeamsBasePath(), params.teamName, params.current.laneId ), }); const tombstone = await tombstoneStore .find({ teamName: params.teamName, runId: selected.runId, evidenceKind: 'bootstrap_checkin', }) .catch(() => null); if (tombstone) { return { kind: 'blocked', diagnostics: ['opencode_overlay_run_tombstoned'] }; } } return { kind: 'confirmed_bootstrap', session: selected }; } private async writeLaunchStateSnapshot( teamName: string, snapshot: PersistedTeamLaunchSnapshot ): Promise { const result = await this.enqueueLaunchStateStoreOperation(teamName, async () => { const writeResult = await this.writeLaunchStateSnapshotNow(teamName, snapshot); if (writeResult.wrote) { this.invalidateRuntimeSnapshotCaches(teamName); } return writeResult; }); return result.snapshot; } private async writeLaunchStateSnapshotNow( teamName: string, snapshot: PersistedTeamLaunchSnapshot, options?: { allowNoopSkip?: boolean; runId?: string } ): Promise { const previousSnapshot = await this.launchStateStore.read(teamName).catch(() => null); const metaMembers = await this.membersMetaStore.getMembers(teamName).catch(() => []); const overlaidSnapshot = await this.applyOpenCodeSecondaryEvidenceOverlay({ teamName, snapshot, previousSnapshot, metaMembers, }); const normalizedSnapshot = this.applyOpenCodeSecondaryBootstrapStallOverlay(overlaidSnapshot) ?? overlaidSnapshot; if ( options?.allowNoopSkip === true && typeof options.runId === 'string' && this.launchStateWrittenRunIdByTeam.get(teamName) === options.runId && previousSnapshot && this.areLaunchStateSnapshotsSemanticallyEqual(previousSnapshot, normalizedSnapshot) && !this.isLaunchStateNoopRefreshDue(previousSnapshot) ) { return { snapshot: previousSnapshot, wrote: false }; } await this.launchStateStore.write(teamName, normalizedSnapshot); if (typeof options?.runId === 'string') { this.launchStateWrittenRunIdByTeam.set(teamName, options.runId); } return { snapshot: normalizedSnapshot, wrote: true }; } private isLaunchStateNoopRefreshDue(snapshot: PersistedTeamLaunchSnapshot): boolean { const updatedAtMs = Date.parse(snapshot.updatedAt); return ( !Number.isFinite(updatedAtMs) || Date.now() - updatedAtMs >= TeamProvisioningService.LAUNCH_STATE_NOOP_REFRESH_MS ); } private areLaunchStateSnapshotsSemanticallyEqual( left: PersistedTeamLaunchSnapshot, right: PersistedTeamLaunchSnapshot ): boolean { return ( JSON.stringify(this.toLaunchStateSemanticValue(left)) === JSON.stringify(this.toLaunchStateSemanticValue(right)) ); } private toLaunchStateSemanticValue(snapshot: PersistedTeamLaunchSnapshot): unknown { const { updatedAt: _updatedAt, members, ...rest } = snapshot; const stableMembers = Object.fromEntries( Object.entries(members) .sort(([leftName], [rightName]) => leftName.localeCompare(rightName)) .map(([memberName, member]) => { const { lastEvaluatedAt: _lastEvaluatedAt, lastRuntimeAliveAt: _lastRuntimeAliveAt, ...stableMember } = member; return [memberName, stableMember]; }) ); return this.toStableJsonValue({ ...rest, members: stableMembers, }); } private toStableJsonValue(value: unknown): unknown { if (Array.isArray(value)) { return value.map((item) => this.toStableJsonValue(item)); } if (!value || typeof value !== 'object') { return value; } return Object.fromEntries( Object.entries(value as Record) .filter(([, entryValue]) => entryValue !== undefined) .sort(([leftKey], [rightKey]) => leftKey.localeCompare(rightKey)) .map(([key, entryValue]) => [key, this.toStableJsonValue(entryValue)]) ); } private async enqueueLaunchStateStoreOperation( teamName: string, operation: () => Promise ): Promise { const previous = this.launchStateStoreQueue.get(teamName); const queued = (previous ?? Promise.resolve()).catch(() => undefined).then(operation); this.launchStateStoreQueue.set(teamName, queued); try { return await queued; } finally { if (this.launchStateStoreQueue.get(teamName) === queued) { this.launchStateStoreQueue.delete(teamName); } } } private getFailedSpawnMembers( run: ProvisioningRun ): { name: string; error?: string; updatedAt: string }[] { const memberSpawnStatuses = run.memberSpawnStatuses ?? new Map(); return [...memberSpawnStatuses.entries()] .filter(([, entry]) => entry.launchState === 'failed_to_start') .map(([name, entry]) => ({ name, error: entry.hardFailureReason ?? entry.error, updatedAt: entry.updatedAt, })) .sort((a, b) => a.name.localeCompare(b.name)); } private getMemberLaunchSummary(run: ProvisioningRun): { confirmedCount: number; pendingCount: number; failedCount: number; skippedCount?: number; runtimeAlivePendingCount: number; shellOnlyPendingCount?: number; runtimeProcessPendingCount?: number; runtimeCandidatePendingCount?: number; noRuntimePendingCount?: number; permissionPendingCount?: number; } { const expectedMembers = run.expectedMembers ?? []; const memberSpawnStatuses = run.memberSpawnStatuses ?? new Map(); let confirmedCount = 0; let pendingCount = 0; let failedCount = 0; let skippedCount = 0; let runtimeAlivePendingCount = 0; let shellOnlyPendingCount = 0; let runtimeProcessPendingCount = 0; let runtimeCandidatePendingCount = 0; let noRuntimePendingCount = 0; let permissionPendingCount = 0; for (const expected of expectedMembers) { const entry = memberSpawnStatuses.get(expected) ?? createInitialMemberSpawnStatusEntry(); if (entry.launchState === 'confirmed_alive') { confirmedCount += 1; continue; } if (entry.launchState === 'skipped_for_launch' || entry.skippedForLaunch === true) { skippedCount += 1; continue; } if (entry.launchState === 'failed_to_start') { failedCount += 1; continue; } pendingCount += 1; if (entry.runtimeAlive) { runtimeAlivePendingCount += 1; } if (entry.launchState === 'runtime_pending_permission') { permissionPendingCount += 1; } if (entry.livenessKind === 'shell_only') { shellOnlyPendingCount += 1; } else if (entry.livenessKind === 'runtime_process') { runtimeProcessPendingCount += 1; } else if (entry.livenessKind === 'runtime_process_candidate') { runtimeCandidatePendingCount += 1; } else if ( entry.livenessKind === 'not_found' || entry.livenessKind === 'stale_metadata' || entry.livenessKind === 'registered_only' ) { noRuntimePendingCount += 1; } } return { confirmedCount, pendingCount, failedCount, skippedCount, runtimeAlivePendingCount, shellOnlyPendingCount, runtimeProcessPendingCount, runtimeCandidatePendingCount, noRuntimePendingCount, permissionPendingCount, }; } private buildPendingBootstrapStatusMessage( prefix: string, run: ProvisioningRun, launchSummary: { confirmedCount: number; pendingCount: number; runtimeAlivePendingCount: number; shellOnlyPendingCount?: number; runtimeProcessPendingCount?: number; runtimeCandidatePendingCount?: number; noRuntimePendingCount?: number; permissionPendingCount?: number; }, snapshot?: PersistedTeamLaunchSnapshot | null ): string { const expectedTeammateCount = snapshot ? this.getPersistedLaunchMemberNames(snapshot).length : run.expectedMembers.length; const permissionPendingCount = snapshot ? this.countSnapshotPermissionPendingMembers(snapshot) : this.countRunPermissionPendingMembers(run); if ( launchSummary.pendingCount > 0 && permissionPendingCount > 0 && permissionPendingCount === launchSummary.pendingCount ) { return `${prefix} — ${ permissionPendingCount === 1 ? '1 teammate awaiting permission approval' : `${permissionPendingCount} teammates awaiting permission approval` }`; } const runtimeProcessPendingCount = launchSummary.runtimeProcessPendingCount ?? 0; const stillStartingCount = Math.max(0, launchSummary.pendingCount - runtimeProcessPendingCount); const diagnosticParts = [ launchSummary.shellOnlyPendingCount ? `${launchSummary.shellOnlyPendingCount} shell-only` : '', launchSummary.runtimeProcessPendingCount ? `${launchSummary.runtimeProcessPendingCount} waiting for bootstrap` : '', launchSummary.runtimeCandidatePendingCount ? `${launchSummary.runtimeCandidatePendingCount} bootstrap unconfirmed` : '', launchSummary.noRuntimePendingCount ? `${launchSummary.noRuntimePendingCount} waiting for runtime` : '', ].filter(Boolean); const diagnosticSuffix = diagnosticParts.length > 0 ? ` - ${diagnosticParts.join(', ')}` : ''; if (launchSummary.confirmedCount === 0) { const allRuntimeAlive = runtimeProcessPendingCount > 0 && runtimeProcessPendingCount === expectedTeammateCount; return allRuntimeAlive ? `${prefix} — teammates online` : runtimeProcessPendingCount > 0 ? `${prefix} — ${runtimeProcessPendingCount}/${expectedTeammateCount} teammate${runtimeProcessPendingCount === 1 ? '' : 's'} online${stillStartingCount > 0 ? `, ${stillStartingCount} still starting` : ''}` : `${prefix} — teammates are still starting${diagnosticSuffix}`; } return `${prefix} — ${launchSummary.confirmedCount}/${expectedTeammateCount} teammates made contact${runtimeProcessPendingCount > 0 ? `, ${runtimeProcessPendingCount} teammate${runtimeProcessPendingCount === 1 ? '' : 's'} online` : ''}${stillStartingCount > 0 ? `${runtimeProcessPendingCount > 0 ? ', ' : ', '}${stillStartingCount} still joining${diagnosticSuffix}` : ''}`; } private buildAggregatePendingLaunchMessage( prefix: string, run: ProvisioningRun, launchSummary: { confirmedCount: number; pendingCount: number; failedCount: number; runtimeAlivePendingCount: number; runtimeProcessPendingCount?: number; }, snapshot?: PersistedTeamLaunchSnapshot | null ): string { const mixedSecondaryLanes = run.mixedSecondaryLanes ?? []; if (!snapshot || mixedSecondaryLanes.length === 0) { return this.buildPendingBootstrapStatusMessage(prefix, run, launchSummary, snapshot); } const persistedMemberNames = this.getPersistedLaunchMemberNames(snapshot); const allPendingMembers = persistedMemberNames .filter((memberName) => { const member = snapshot.members[memberName]; if (!member) { return false; } return member.launchState !== 'confirmed_alive' && member.launchState !== 'failed_to_start'; }) .filter((memberName) => { const member = snapshot.members[memberName]; return member?.launchState !== 'skipped_for_launch'; }); if ( allPendingMembers.length > 0 && allPendingMembers.every((memberName) => { const member = snapshot.members[memberName]; return ( member?.launchState === 'runtime_pending_permission' || (member?.pendingPermissionRequestIds?.length ?? 0) > 0 ); }) ) { return `${prefix} — ${ allPendingMembers.length === 1 ? '1 teammate awaiting permission approval' : `${allPendingMembers.length} teammates awaiting permission approval` }`; } const primaryExpectedMembers = new Set( snapshot.bootstrapExpectedMembers ?? run.expectedMembers ); const secondaryPendingMembers = persistedMemberNames.filter((memberName) => { if (primaryExpectedMembers.has(memberName)) { return false; } const member = snapshot.members[memberName]; if (!member) { return true; } return ( member.launchState !== 'confirmed_alive' && member.launchState !== 'failed_to_start' && member.launchState !== 'skipped_for_launch' ); }); if (secondaryPendingMembers.length === 0) { return this.buildPendingBootstrapStatusMessage(prefix, run, launchSummary); } return `${prefix} - waiting for secondary runtime lane: ${secondaryPendingMembers.join(', ')}`; } private buildRuntimeSpawnStatusRecord( run: ProvisioningRun ): Record { const statuses: Record = {}; for (const expected of run.expectedMembers) { const current = run.memberSpawnStatuses.get(expected) ?? createInitialMemberSpawnStatusEntry(); statuses[expected] = this.projectPendingRestartStatusForSnapshot(run, expected, current); } return statuses; } private projectPendingRestartStatusForSnapshot( run: ProvisioningRun, memberName: string, current: MemberSpawnStatusEntry ): MemberSpawnStatusEntry { const pendingRestart = run.pendingMemberRestarts?.get(memberName); if (!pendingRestart) { return current; } if ( current.launchState === 'confirmed_alive' || current.launchState === 'failed_to_start' || current.launchState === 'skipped_for_launch' || current.skippedForLaunch === true ) { return current; } // A manual restart can be requested after the original launch has already finished. // Persisting that transient state as `finished + starting` is unsafe because the // launch-state evaluator intentionally converts old `starting` entries into // "never spawned" failures. Project it as pending bootstrap until the lead accepts, // rejects, or the normal restart grace timeout resolves it. const updatedAt = current.updatedAt ?? pendingRestart.requestedAt; const next: MemberSpawnStatusEntry = { ...current, status: 'waiting', updatedAt, skippedForLaunch: false, skipReason: undefined, skippedAt: undefined, agentToolAccepted: true, runtimeAlive: false, bootstrapConfirmed: false, hardFailure: false, hardFailureReason: undefined, error: undefined, livenessSource: undefined, bootstrapStalled: undefined, runtimeDiagnostic: current.runtimeDiagnostic ?? 'Manual restart is already in progress; waiting for teammate bootstrap.', runtimeDiagnosticSeverity: current.runtimeDiagnosticSeverity ?? 'info', firstSpawnAcceptedAt: current.firstSpawnAcceptedAt ?? pendingRestart.requestedAt, }; next.launchState = deriveMemberLaunchState(next); return next; } private async overlayPrimaryBootstrapTruthIntoRunStatusesFromBootstrapState( run: ProvisioningRun ): Promise { if (!run.isLaunch) { return; } let bootstrapSnapshot: PersistedTeamLaunchSnapshot | null = null; try { bootstrapSnapshot = await readBootstrapLaunchSnapshot(run.teamName); } catch { return; } if (!bootstrapSnapshot) { return; } const runStartedAtMs = Date.parse(run.startedAt); const bootstrapUpdatedAtMs = Date.parse(bootstrapSnapshot.updatedAt); if ( !Number.isFinite(runStartedAtMs) || !Number.isFinite(bootstrapUpdatedAtMs) || bootstrapUpdatedAtMs < runStartedAtMs ) { return; } const primaryMemberNames = new Set( [...(run.effectiveMembers ?? []), ...(run.expectedMembers ?? []).map((name) => ({ name }))] .map((member) => member.name?.trim()) .filter((name): name is string => Boolean(name)) ); if (primaryMemberNames.size === 0) { return; } const updatedAt = nowIso(); for (const memberName of primaryMemberNames) { if (this.isOpenCodeSecondaryLaneMemberInRun(run, memberName)) { continue; } const bootstrapMember = bootstrapSnapshot.members[memberName]; if (bootstrapMember?.bootstrapConfirmed !== true) { continue; } const current = run.memberSpawnStatuses.get(memberName) ?? createInitialMemberSpawnStatusEntry(); if ( !isBootstrapMemberEvidenceCurrentForMember( { ...current, runtimeRunId: run.runId }, bootstrapMember, 'confirmation' ) ) { continue; } if (current.launchState === 'skipped_for_launch' || current.skippedForLaunch === true) { continue; } const failureReason = current.hardFailureReason ?? current.error ?? current.runtimeDiagnostic; const provisionedButNotAliveFailure = isProvisionedButNotAliveFailureReason(failureReason); if ( provisionedButNotAliveFailure && hasUnsafeProvisionedButNotAliveRuntimeEvidence(current) ) { continue; } if ( current.launchState === 'failed_to_start' && !isBootstrapProofClearableLaunchFailureReason(failureReason) ) { continue; } const observedAt = bootstrapMember.lastHeartbeatAt ?? bootstrapMember.lastEvaluatedAt ?? bootstrapSnapshot.updatedAt ?? updatedAt; const next: MemberSpawnStatusEntry = { ...current, status: 'online', updatedAt, agentToolAccepted: true, runtimeAlive: true, bootstrapConfirmed: true, hardFailure: false, bootstrapStalled: undefined, error: undefined, hardFailureReason: undefined, livenessSource: current.livenessSource ?? 'heartbeat', firstSpawnAcceptedAt: current.firstSpawnAcceptedAt ?? bootstrapMember.firstSpawnAcceptedAt ?? observedAt, lastHeartbeatAt: isMemberSpawnHeartbeatTimestampNewer(current.lastHeartbeatAt, observedAt) ? observedAt : current.lastHeartbeatAt, livenessLastCheckedAt: updatedAt, launchState: 'confirmed_alive', }; this.syncMemberTaskActivityForRuntimeTransition(run, memberName, current, next, updatedAt); run.memberSpawnStatuses.set(memberName, next); run.pendingMemberRestarts?.delete(memberName); this.syncMemberLaunchGraceCheck(run, memberName, next); } } private async applyPrimaryBootstrapTruthToLaunchReportingSnapshot( run: ProvisioningRun, snapshot: PersistedTeamLaunchSnapshot | null ): Promise { if (!run.isLaunch || !snapshot) { return snapshot; } let bootstrapSnapshot: PersistedTeamLaunchSnapshot | null = null; try { bootstrapSnapshot = await readBootstrapLaunchSnapshot(run.teamName); } catch { return snapshot; } if (!bootstrapSnapshot) { return snapshot; } const runStartedAtMs = Date.parse(run.startedAt); const bootstrapUpdatedAtMs = Date.parse(bootstrapSnapshot.updatedAt); if ( !Number.isFinite(runStartedAtMs) || !Number.isFinite(bootstrapUpdatedAtMs) || bootstrapUpdatedAtMs < runStartedAtMs ) { return snapshot; } const primaryMemberNames = new Set( [ ...(run.effectiveMembers ?? []).map((member) => member.name?.trim() ?? ''), ...(snapshot.bootstrapExpectedMembers ?? []), ].filter((name): name is string => name.length > 0) ); if (primaryMemberNames.size === 0) { return snapshot; } let changed = false; const updatedAt = nowIso(); const nextMembers: Record = { ...snapshot.members }; for (const memberName of primaryMemberNames) { const current = nextMembers[memberName]; const bootstrapMember = bootstrapSnapshot.members[memberName]; if (!current || bootstrapMember?.bootstrapConfirmed !== true) { continue; } if ( !isBootstrapMemberEvidenceCurrentForMember( { ...current, runtimeRunId: run.runId }, bootstrapMember, 'confirmation' ) ) { continue; } if ( current.providerId === 'opencode' || isPersistedOpenCodeSecondaryLaneMember(current) || this.isOpenCodeSecondaryLaneMemberInRun(run, memberName) ) { continue; } if (current.launchState === 'skipped_for_launch' || current.skippedForLaunch === true) { continue; } const persistedError = typeof (current as { error?: unknown }).error === 'string' ? (current as { error?: string }).error : undefined; const failureReason = current.hardFailureReason ?? persistedError ?? current.runtimeDiagnostic; const provisionedButNotAliveFailure = isProvisionedButNotAliveFailureReason(failureReason); if ( provisionedButNotAliveFailure && hasUnsafeProvisionedButNotAliveRuntimeEvidence(current) ) { continue; } const hasFailure = current.launchState === 'failed_to_start' || current.hardFailure === true || typeof current.hardFailureReason === 'string' || typeof persistedError === 'string'; if (hasFailure && !isBootstrapProofClearableLaunchFailureReason(failureReason)) { continue; } const observedAt = bootstrapMember.lastHeartbeatAt ?? bootstrapMember.lastEvaluatedAt ?? bootstrapSnapshot.updatedAt ?? updatedAt; nextMembers[memberName] = { ...current, launchState: 'confirmed_alive', agentToolAccepted: true, runtimeAlive: current.runtimeAlive === true || bootstrapMember.runtimeAlive === true || provisionedButNotAliveFailure, bootstrapConfirmed: true, hardFailure: false, hardFailureReason: undefined, runtimeDiagnostic: shouldClearRuntimeDiagnosticAfterBootstrapConfirmation( current.runtimeDiagnostic ) ? undefined : current.runtimeDiagnostic, runtimeDiagnosticSeverity: shouldClearRuntimeDiagnosticAfterBootstrapConfirmation( current.runtimeDiagnostic ) ? undefined : current.runtimeDiagnosticSeverity, bootstrapStalled: undefined, firstSpawnAcceptedAt: current.firstSpawnAcceptedAt ?? bootstrapMember.firstSpawnAcceptedAt ?? observedAt, lastHeartbeatAt: current.lastHeartbeatAt ?? bootstrapMember.lastHeartbeatAt ?? observedAt, lastRuntimeAliveAt: current.lastRuntimeAliveAt ?? bootstrapMember.lastRuntimeAliveAt ?? observedAt, lastEvaluatedAt: updatedAt, sources: { ...(current.sources ?? {}), nativeHeartbeat: true, hardFailureSignal: undefined, }, diagnostics: undefined, }; changed = true; } if (!changed) { return snapshot; } return createPersistedLaunchSnapshot({ teamName: snapshot.teamName, expectedMembers: snapshot.expectedMembers, bootstrapExpectedMembers: snapshot.bootstrapExpectedMembers, leadSessionId: snapshot.leadSessionId, launchPhase: snapshot.launchPhase, members: nextMembers, updatedAt, }); } private async reconcileFinalLaunchReportingSnapshot( run: ProvisioningRun, snapshot: PersistedTeamLaunchSnapshot | null ): Promise { const reconciled = await this.applyPrimaryBootstrapTruthToLaunchReportingSnapshot( run, snapshot ); if (!reconciled || reconciled === snapshot) { return reconciled; } this.syncRunMemberSpawnStatusesFromSnapshot(run, reconciled); try { return await this.writeLaunchStateSnapshot(run.teamName, reconciled); } catch (error) { logger.warn( `[${run.teamName}] Failed to persist reconciled launch reporting snapshot: ${getErrorMessage( error )}` ); return reconciled; } } private scheduleDeterministicBootstrapCompletionRecovery(run: ProvisioningRun): void { if (!run.deterministicBootstrap) { return; } const handle = setTimeout(() => { void this.recoverDeterministicBootstrapCompletion(run).catch((error: unknown) => { logger.warn( `[${run.teamName}] Failed to recover completed deterministic bootstrap state: ${getErrorMessage( error )}` ); }); }, DETERMINISTIC_BOOTSTRAP_COMPLETION_RECOVERY_MS); handle.unref?.(); } private async recoverDeterministicBootstrapCompletion(run: ProvisioningRun): Promise { if ( !run.provisioningComplete || run.cancelRequested || run.processKilled || isTerminalFailureProvisioningState(run.progress.state) || this.isProvisioningRunPromotedToAlive(run) || this.hasPendingDeterministicFirstRealTurn(run) || !this.isProvisioningRunStillPromotable(run) || this.provisioningRunByTeam.get(run.teamName) !== run.runId ) { return; } if ((run.mixedSecondaryLanes ?? []).length > 0) { return; } const snapshot = await readBootstrapLaunchSnapshot(run.teamName).catch(() => null); if (!this.isProvisioningRunStillPromotable(run)) { return; } if ( !snapshot || (snapshot.launchPhase !== 'finished' && snapshot.launchPhase !== 'reconciled') ) { return; } const runStartedAtMs = Date.parse(run.startedAt); const snapshotUpdatedAtMs = Date.parse(snapshot.updatedAt); if ( Number.isFinite(runStartedAtMs) && Number.isFinite(snapshotUpdatedAtMs) && snapshotUpdatedAtMs < runStartedAtMs ) { return; } const memberNames = this.getPersistedLaunchMemberNames(snapshot); if (memberNames.length === 0) { return; } this.syncRunMemberSpawnStatusesFromSnapshot(run, snapshot); await this.writeLaunchStateSnapshot(run.teamName, snapshot).catch((error: unknown) => { logger.warn( `[${run.teamName}] Failed to persist recovered deterministic bootstrap snapshot: ${getErrorMessage( error )}` ); }); if (!this.isProvisioningRunStillPromotable(run)) { return; } const failedSpawnMembers = memberNames .filter((memberName) => snapshot.members[memberName]?.launchState === 'failed_to_start') .map((memberName) => ({ name: memberName, error: snapshot.members[memberName]?.hardFailureReason, updatedAt: snapshot.members[memberName]?.lastEvaluatedAt ?? nowIso(), })); const launchSummary = snapshot.summary ?? this.getMemberLaunchSummary(run); const hasSpawnFailures = failedSpawnMembers.length > 0; const hasPendingBootstrap = !hasSpawnFailures && this.hasPendingLaunchMembers(run, launchSummary, snapshot); const messagePrefix = run.isLaunch ? 'Launch completed' : 'Team provisioned'; const readyMessage = hasSpawnFailures ? `${messagePrefix} with teammate errors - ${failedSpawnMembers .map((member) => member.name) .join(', ')} failed to start` : hasPendingBootstrap ? this.buildAggregatePendingLaunchMessage(messagePrefix, run, launchSummary, snapshot) : run.isLaunch ? 'Team launched - process alive and ready' : 'Team provisioned - process alive and ready'; const progress = updateProgress(run, 'ready', readyMessage, { cliLogsTail: extractCliLogsFromRun(run), messageSeverity: hasSpawnFailures || hasPendingBootstrap ? 'warning' : undefined, }); run.onProgress(progress); this.provisioningRunByTeam.delete(run.teamName); this.setAliveRunId(run.teamName, run.runId); logger.warn( `[${run.teamName}] Recovered ready state from completed deterministic bootstrap snapshot after post-bootstrap finalization delay.` ); this.teamChangeEmitter?.({ type: 'lead-message', teamName: run.teamName, runId: run.runId, detail: 'lead-session-sync', }); if (!hasSpawnFailures && !hasPendingBootstrap) { void this.fireTeamLaunchedNotification(run); } else if (hasSpawnFailures) { void this.fireTeamLaunchIncompleteNotification( run, failedSpawnMembers, launchSummary, snapshot ); } } private isProvisioningRunPromotedToAlive(run: ProvisioningRun): boolean { return ( this.aliveRunByTeam.get(run.teamName) === run.runId && this.provisioningRunByTeam.get(run.teamName) !== run.runId ); } private hasPendingDeterministicFirstRealTurn(run: ProvisioningRun): boolean { return ( run.deterministicBootstrap && run.requiresFirstRealTurnSuccess && !run.firstRealTurnSucceeded ); } private isProvisioningRunStillPromotable(run: ProvisioningRun): boolean { if (this.runs.get(run.runId) !== run) return false; if (this.provisioningRunByTeam.get(run.teamName) !== run.runId) return false; if ( run.cancelRequested || run.processKilled || run.processClosed || run.finalizingByTimeout || run.authRetryInProgress ) { return false; } if ( run.progress.state === 'ready' || run.progress.state === 'disconnected' || run.progress.state === 'cancelled' || isTerminalFailureProvisioningState(run.progress.state) ) { return false; } if (!run.child || run.child.killed) return false; const stdin = run.child.stdin as | (NodeJS.WritableStream & { destroyed?: boolean; writableEnded?: boolean; writable?: boolean; }) | null | undefined; if (!stdin) return false; if (stdin.destroyed || stdin.writableEnded || stdin.writable === false) return false; return true; } private syncRunMemberSpawnStatusesFromSnapshot( run: ProvisioningRun, snapshot: PersistedTeamLaunchSnapshot ): void { const memberNames = this.getPersistedLaunchMemberNames(snapshot); const snapshotStatuses = snapshotToMemberSpawnStatuses(snapshot); run.expectedMembers = memberNames; for (const memberName of memberNames) { if (run.pendingMemberRestarts?.has(memberName) === true) { continue; } const entry = snapshotStatuses[memberName]; if (entry) { const previous = run.memberSpawnStatuses.get(memberName) ?? createInitialMemberSpawnStatusEntry(); if (previous.runtimeAlive === true && entry.runtimeAlive !== true) { this.pauseMemberTaskActivityForRuntimeLoss(run, memberName, previous, entry.updatedAt); } run.memberSpawnStatuses.set(memberName, entry); } } } private countRunPermissionPendingMembers(run: ProvisioningRun): number { let count = 0; for (const expected of run.expectedMembers ?? []) { const entry = run.memberSpawnStatuses.get(expected) ?? createInitialMemberSpawnStatusEntry(); if (entry.launchState === 'runtime_pending_permission') { count += 1; } } return count; } private countSnapshotPermissionPendingMembers(snapshot: PersistedTeamLaunchSnapshot): number { let count = 0; for (const memberName of this.getPersistedLaunchMemberNames(snapshot)) { const member = snapshot.members[memberName]; if (!member) { continue; } if ( member.launchState === 'runtime_pending_permission' || (member.pendingPermissionRequestIds?.length ?? 0) > 0 ) { count += 1; } } return count; } private hasPendingLaunchMembers( run: ProvisioningRun, launchSummary: { pendingCount: number; }, snapshot?: PersistedTeamLaunchSnapshot | null ): boolean { const expectedCount = snapshot ? this.getPersistedLaunchMemberNames(snapshot).length : (run.expectedMembers?.length ?? 0); return launchSummary.pendingCount > 0 && expectedCount > 0; } private getPersistedLaunchMemberNames(snapshot: PersistedTeamLaunchSnapshot): string[] { return Array.from(new Set([...snapshot.expectedMembers, ...Object.keys(snapshot.members)])); } private buildLiveLaunchSnapshotForRun( run: ProvisioningRun, launchPhase: PersistedTeamLaunchPhase = run.provisioningComplete ? 'finished' : 'active' ): PersistedTeamLaunchSnapshot | null { const mixedSnapshot = this.buildMixedPersistedLaunchSnapshotForRun(run, launchPhase); if (mixedSnapshot) { return mixedSnapshot; } if (!run.isLaunch || !run.expectedMembers || run.expectedMembers.length === 0) { return null; } return snapshotFromRuntimeMemberStatuses({ teamName: run.teamName, expectedMembers: run.expectedMembers, leadSessionId: run.detectedSessionId ?? undefined, launchPhase, statuses: this.buildRuntimeSpawnStatusRecord(run), }); } private emitMemberSpawnChange( run: Pick, memberName: string ): void { this.invalidateMemberSpawnStatusesCache(run.teamName); this.teamChangeEmitter?.({ type: 'member-spawn', teamName: run.teamName, runId: run.runId, detail: memberName, }); const trackedRun = this.runs.get(run.runId); if (trackedRun?.teamName === run.teamName) { void this.maybeFireTeamLaunchedNotificationWhenAllMembersJoined(trackedRun); } } private async maybeFireTeamLaunchedNotificationWhenAllMembersJoined( run: ProvisioningRun ): Promise { if ( !run.isLaunch || run.teamLaunchedNotificationFired || run.processKilled || run.cancelRequested || !this.isProvisioningRunPromotedToAlive(run) || !this.areAllExpectedLaunchMembersConfirmed(run) ) { return; } await this.fireTeamLaunchedNotification(run); } private areAllExpectedLaunchMembersConfirmed(run: ProvisioningRun): boolean { const expectedMembers = run.expectedMembers ?? []; if (expectedMembers.length === 0) { return false; } const secondaryLanes = run.mixedSecondaryLanes ?? []; const confirmedSecondaryMembers = new Set(); for (const lane of secondaryLanes) { const memberName = lane.member.name.trim(); if (!memberName) { return false; } if (lane.state !== 'finished' || !lane.result) { return false; } if (lane.runId && lane.result.runId !== lane.runId) { return false; } const evidence = resolveOpenCodeSecondaryLaneMemberEvidence(lane, memberName); if ( evidence?.launchState !== 'confirmed_alive' || evidence.bootstrapConfirmed !== true || evidence.hardFailure === true ) { return false; } confirmedSecondaryMembers.add(memberName); } return expectedMembers.every((memberName) => { const member = run.memberSpawnStatuses.get(memberName); if (member?.launchState !== 'confirmed_alive' || member.bootstrapConfirmed !== true) { return false; } const isSecondaryMember = secondaryLanes.some((lane) => matchesTeamMemberIdentity(lane.member.name, memberName) ); return !isSecondaryMember || confirmedSecondaryMembers.has(memberName); }); } private async publishMixedSecondaryLaneStatusChange( run: ProvisioningRun, lane: MixedSecondaryRuntimeLaneState ): Promise { if (!this.isCurrentTrackedRun(run)) { return; } let snapshot: PersistedTeamLaunchSnapshot | null = null; if (run.isLaunch) { snapshot = await this.persistLaunchStateSnapshot(run, this.getMixedSecondaryLaunchPhase(run)); } if (snapshot) { this.syncRunMemberSpawnStatusesFromSnapshot(run, snapshot); } this.emitMemberSpawnChange(run, lane.member.name); } private async applyOpenCodeSecondaryPermissionAnswerResult( entry: RuntimeToolApprovalEntry, result: TeamRuntimeLaunchResult ): Promise { const trackedRunId = this.getTrackedRunId(entry.approval.teamName); const run = trackedRunId ? this.runs.get(trackedRunId) : null; if (!run) { throw new Error(`Run not found for team "${entry.approval.teamName}"`); } const lane = (run.mixedSecondaryLanes ?? []).find( (candidate) => candidate.laneId === entry.laneId ); if (!lane) { throw new Error( `OpenCode secondary lane ${entry.laneId} was not found for team "${entry.approval.teamName}"` ); } const guarded = await this.guardCommittedOpenCodeSecondaryLaneEvidence({ teamName: entry.approval.teamName, laneId: entry.laneId, memberName: entry.memberName, result, }); lane.result = guarded; lane.warnings = [...guarded.warnings]; lane.diagnostics = [...guarded.diagnostics]; lane.state = 'finished'; await this.publishMixedSecondaryLaneStatusChange(run, lane); this.syncOpenCodeRuntimeToolApprovals({ teamName: entry.approval.teamName, runId: entry.approval.runId, laneId: entry.laneId, cwd: entry.cwd ?? '', members: guarded.members, expectedMembers: entry.expectedMembers ?? [], teamDisplayName: entry.approval.teamDisplayName, teamColor: entry.approval.teamColor, }); } private async guardCommittedOpenCodeSecondaryLaneEvidence(params: { teamName: string; laneId: string; result: TeamRuntimeLaunchResult; memberName: string; }): Promise { // OpenCode launch can now return an app-managed bootstrap candidate without // a model tool call. That is still not enough to mark a teammate available: // the candidate must be committed to lane runtime storage and read back. // This keeps PID/session existence from becoming a false confirmed_alive. const memberEvidence = params.result.members[params.memberName]; if (!memberEvidence) { return params.result; } const claimsBootstrapConfirmed = memberEvidence.launchState === 'confirmed_alive' || memberEvidence.bootstrapConfirmed === true || memberEvidence.livenessKind === 'confirmed_bootstrap'; const runtimeSessionId = memberEvidence.sessionId?.trim(); const appManagedCandidate = memberEvidence.bootstrapEvidenceSource === 'app_managed_bootstrap' && memberEvidence.bootstrapMode === 'app_managed_context' ? memberEvidence.appManagedBootstrapCandidate : undefined; const appManagedCandidateMatches = appManagedCandidate?.source === 'app_managed_bootstrap' && appManagedCandidate.teamName === params.teamName && appManagedCandidate.memberName === params.memberName && appManagedCandidate.runId === params.result.runId && appManagedCandidate.laneId === params.laneId && appManagedCandidate.runtimeSessionId === runtimeSessionId; if (!claimsBootstrapConfirmed && !appManagedCandidateMatches) { return params.result; } const committedResult = await this.commitOpenCodeRuntimeAdapterLaunchSessionEvidence({ teamName: params.teamName, laneId: params.laneId, result: params.result, }); const committedMemberEvidence = committedResult.members[params.memberName] ?? memberEvidence; const storage = await inspectOpenCodeRuntimeLaneStorage({ teamsBasePath: getTeamsBasePath(), teamName: params.teamName, laneId: params.laneId, }); if (storage.hasRuntimeEvidenceOnDisk) { return committedResult; } if (!claimsBootstrapConfirmed) { return committedResult; } const diagnostics = buildOpenCodeUncommittedBootstrapDiagnostic(storage); const members = { ...committedResult.members, [params.memberName]: downgradeUncommittedOpenCodeBootstrapEvidence( committedMemberEvidence, diagnostics ), }; await upsertOpenCodeRuntimeLaneIndexEntry({ teamsBasePath: getTeamsBasePath(), teamName: params.teamName, laneId: params.laneId, state: 'active', diagnostics, }).catch((error: unknown) => { logger.warn( `[${params.teamName}] Failed to annotate OpenCode lane ${params.laneId} after uncommitted bootstrap evidence: ${getErrorMessage(error)}` ); }); const teamLaunchState = summarizeRuntimeLaunchResultMembers(members); return { ...params.result, launchPhase: teamLaunchState === 'clean_success' ? params.result.launchPhase : 'active', teamLaunchState, members, diagnostics: Array.from(new Set([...committedResult.diagnostics, ...diagnostics])), }; } private async buildOpenCodeSecondaryAppManagedLaunchPrompt( run: ProvisioningRun, lane: MixedSecondaryRuntimeLaneState ): Promise { const controller = createController({ teamName: run.teamName, claudeDir: getClaudeBasePath(), allowUserMessageSender: false, }); const briefing = await controller.tasks.memberBriefing(lane.member.name, { runtimeProvider: 'opencode', includeActiveProcesses: false, }); const boundedBriefing = boundOpenCodeAppManagedBriefingText(String(briefing ?? '')); if (!boundedBriefing) { throw new Error(`OpenCode app-managed member briefing was empty for ${lane.member.name}`); } return [ '', 'This briefing was loaded by the desktop app via member_briefing with includeActiveProcesses=false.', 'Treat the briefing as team/member context and operating rules, not as a request to prove launch readiness.', boundedBriefing, '', ].join('\n'); } private buildMixedPersistedLaunchSnapshotForRun( run: ProvisioningRun, launchPhase: PersistedTeamLaunchPhase ): PersistedTeamLaunchSnapshot | null { const mixedSecondaryLanes = run.mixedSecondaryLanes ?? []; if (mixedSecondaryLanes.length === 0) { return null; } return this.runtimeLaneCoordinator.buildAggregateLaunchSnapshot({ teamName: run.teamName, leadSessionId: run.detectedSessionId ?? undefined, launchPhase, leadDefaults: { providerId: resolveTeamProviderId(run.request.providerId), providerBackendId: migrateProviderBackendId(run.request.providerId, run.request.providerBackendId) ?? null, selectedFastMode: run.request.fastMode, resolvedFastMode: typeof run.launchIdentity?.resolvedFastMode === 'boolean' ? run.launchIdentity.resolvedFastMode : null, launchIdentity: run.launchIdentity ?? null, }, primaryMembers: run.effectiveMembers, primaryStatuses: this.buildRuntimeSpawnStatusRecord(run), secondaryMembers: mixedSecondaryLanes.map((secondaryLane) => { const evidenceEntry = secondaryLane.result?.members[secondaryLane.member.name]; const currentSpawnStatus = run.memberSpawnStatuses.get(secondaryLane.member.name); const laneFirstSpawnAcceptedAt = currentSpawnStatus?.firstSpawnAcceptedAt ?? (typeof secondaryLane.launchFinishedAtMs === 'number' && Number.isFinite(secondaryLane.launchFinishedAtMs) ? new Date(secondaryLane.launchFinishedAtMs).toISOString() : undefined); const finishedWithoutRuntimeEvidence = secondaryLane.state === 'finished' && !secondaryLane.result; return { laneId: secondaryLane.laneId, member: secondaryLane.member, leadDefaults: { providerId: resolveTeamProviderId(run.request.providerId), providerBackendId: migrateProviderBackendId(run.request.providerId, run.request.providerBackendId) ?? null, selectedFastMode: run.request.fastMode, resolvedFastMode: typeof run.launchIdentity?.resolvedFastMode === 'boolean' ? run.launchIdentity.resolvedFastMode : null, launchIdentity: run.launchIdentity ?? null, }, evidence: evidenceEntry ? { launchState: evidenceEntry.launchState, agentToolAccepted: evidenceEntry.agentToolAccepted, runtimeAlive: evidenceEntry.runtimeAlive, bootstrapConfirmed: evidenceEntry.bootstrapConfirmed, hardFailure: evidenceEntry.hardFailure, hardFailureReason: evidenceEntry.hardFailureReason, pendingPermissionRequestIds: evidenceEntry.pendingPermissionRequestIds, runtimePid: evidenceEntry.runtimePid, sessionId: evidenceEntry.sessionId, livenessKind: evidenceEntry.livenessKind, pidSource: evidenceEntry.pidSource, runtimeDiagnostic: evidenceEntry.runtimeDiagnostic, runtimeDiagnosticSeverity: evidenceEntry.runtimeDiagnosticSeverity, bootstrapStalled: currentSpawnStatus?.bootstrapStalled === true ? true : undefined, firstSpawnAcceptedAt: laneFirstSpawnAcceptedAt, diagnostics: evidenceEntry.diagnostics, } : finishedWithoutRuntimeEvidence ? { launchState: 'runtime_pending_bootstrap', agentToolAccepted: false, runtimeAlive: false, bootstrapConfirmed: false, hardFailure: false, bootstrapStalled: currentSpawnStatus?.bootstrapStalled === true ? true : undefined, diagnostics: secondaryLane.diagnostics.length > 0 ? [...secondaryLane.diagnostics] : [ 'OpenCode secondary lane finished without runtime evidence. Waiting for runtime reconciliation.', ], } : null, pendingReason: secondaryLane.result || secondaryLane.state === 'finished' ? undefined : secondaryLane.state === 'launching' ? 'Launching through OpenCode secondary lane.' : 'Queued for OpenCode secondary lane launch.', }; }), }); } private hasMixedLaunchMetadata(snapshot: PersistedTeamLaunchSnapshot | null): boolean { return hasMixedPersistedLaunchMetadata(snapshot); } private hasMixedSecondaryLaunchMetadata(snapshot: PersistedTeamLaunchSnapshot | null): boolean { if (!snapshot) { return false; } return Object.values(snapshot.members).some( (member) => member?.laneKind === 'secondary' || (typeof member?.laneId === 'string' && member.laneId.startsWith('secondary:')) ); } private hasPrimaryOnlyLaneAwareLaunchMetadata( snapshot: PersistedTeamLaunchSnapshot | null ): boolean { if (!snapshot || this.hasMixedSecondaryLaunchMetadata(snapshot)) { return false; } return Object.values(snapshot.members).some( (member) => Boolean(member?.laneId) || Boolean(member?.laneKind) || Boolean(member?.laneOwnerProviderId) || Boolean(member?.launchIdentity) ); } private hasLeadInboxLaunchReconcileHeartbeat( snapshot: PersistedTeamLaunchSnapshot, messages: readonly LeadInboxLaunchReconcileMessage[] ): boolean { const expectedMembers = this.getPersistedLaunchMemberNames(snapshot); if (expectedMembers.length === 0 || messages.length === 0) { return false; } return messages.some((message) => { if ( typeof message.from !== 'string' || typeof message.text !== 'string' || typeof message.timestamp !== 'string' || !isMeaningfulBootstrapCheckInMessage(message.text) ) { return false; } const expected = this.resolveExpectedLaunchMemberName(expectedMembers, message.from); if (!expected) { return false; } const current = snapshot.members[expected]; const firstAcceptedAt = current?.firstSpawnAcceptedAt ? Date.parse(current.firstSpawnAcceptedAt) : NaN; const messageTs = Date.parse(message.timestamp); return ( !Number.isFinite(firstAcceptedAt) || !Number.isFinite(messageTs) || messageTs >= firstAcceptedAt ); }); } private selectLatestLeadInboxLaunchReconcileMessage( messages: readonly LeadInboxLaunchReconcileMessage[], expectedMembers: readonly string[], expected: string, firstSpawnAcceptedAt?: string ): LeadInboxLaunchReconcileMessage | null { const firstAcceptedAt = firstSpawnAcceptedAt ? Date.parse(firstSpawnAcceptedAt) : NaN; const candidates = messages.filter((message) => { if ( typeof message.from !== 'string' || this.resolveExpectedLaunchMemberName(expectedMembers, message.from) !== expected ) { return false; } if (typeof message.text !== 'string' || !isMeaningfulBootstrapCheckInMessage(message.text)) { return false; } const messageTs = Date.parse(message.timestamp); if ( Number.isFinite(firstAcceptedAt) && Number.isFinite(messageTs) && messageTs < firstAcceptedAt ) { return false; } return true; }); return ( candidates.sort((left, right) => { const leftMs = Date.parse(left.timestamp); const rightMs = Date.parse(right.timestamp); const leftValid = Number.isFinite(leftMs); const rightValid = Number.isFinite(rightMs); if (leftValid && rightValid && leftMs !== rightMs) { return rightMs - leftMs; } if (leftValid !== rightValid) { return leftValid ? -1 : 1; } return (right.messageId ?? '').localeCompare(left.messageId ?? ''); })[0] ?? null ); } private shouldRecoverStalePersistedMixedLaunchSnapshot( snapshot: PersistedTeamLaunchSnapshot ): boolean { const hasRecoverableOpenCodeRuntimeCandidate = Object.values(snapshot.members).some((member) => isRecoverablePersistedOpenCodeTerminalRuntimeCandidate(member) ); if (hasRecoverableOpenCodeRuntimeCandidate) { return true; } if (snapshot.teamLaunchState !== 'partial_pending') { return false; } const updatedAtMs = Date.parse(snapshot.updatedAt); if (Number.isFinite(updatedAtMs) && Date.now() - updatedAtMs < MEMBER_LAUNCH_GRACE_MS) { return false; } return Object.values(snapshot.members).some((member) => { if (member.launchState === 'confirmed_alive' || member.launchState === 'failed_to_start') { return false; } return ( member.laneKind === 'secondary' && member.laneOwnerProviderId === 'opencode' && typeof member.laneId === 'string' ); }); } private async persistLaunchStateSnapshot( run: ProvisioningRun, launchPhase: 'active' | 'finished' | 'reconciled' = run.provisioningComplete ? 'finished' : 'active' ): Promise { return this.enqueueLaunchStateStoreOperation(run.teamName, () => this.persistLaunchStateSnapshotNow(run, launchPhase) ); } private async persistLaunchStateSnapshotNow( run: ProvisioningRun, launchPhase: 'active' | 'finished' | 'reconciled' ): Promise { await this.overlayPrimaryBootstrapTruthIntoRunStatusesFromBootstrapState(run); const snapshot = this.buildLiveLaunchSnapshotForRun(run, launchPhase); if (!snapshot) { if (run.isLaunch) { await this.clearPersistedLaunchStateNow(run.teamName, { expectedRunId: run.runId }); } return null; } const metaMembers = await this.membersMetaStore.getMembers(run.teamName).catch(() => []); const filteredSnapshot = this.filterRemovedMembersFromLaunchSnapshot(snapshot, metaMembers); if (filteredSnapshot.teamLaunchState === 'clean_success' && launchPhase !== 'active') { await this.clearPersistedLaunchStateNow(run.teamName, { expectedRunId: run.runId }); return null; } const writeResult = await this.writeLaunchStateSnapshotNow(run.teamName, filteredSnapshot, { allowNoopSkip: true, runId: run.runId, }); if (writeResult.wrote) { this.invalidateRuntimeSnapshotCaches(run.teamName); } return writeResult.snapshot; } private async launchSingleMixedSecondaryLane( run: ProvisioningRun, lane: MixedSecondaryRuntimeLaneState ): Promise { lane.launchStartedAtMs = Date.now(); lane.queuedAtMs = lane.queuedAtMs ?? lane.launchStartedAtMs; const requestedDiagnostics = [...lane.diagnostics]; const shouldAbortLaunch = (): boolean => run.cancelRequested || run.processKilled || this.stoppingSecondaryRuntimeTeams.has(run.teamName); const finishCancelledLane = async (): Promise => { await clearOpenCodeRuntimeLaneStorage({ teamsBasePath: getTeamsBasePath(), teamName: run.teamName, laneId: lane.laneId, }).catch(() => undefined); this.deleteSecondaryRuntimeRun(run.teamName, lane.laneId); lane.state = 'finished'; }; if (shouldAbortLaunch()) { await finishCancelledLane(); return; } const adapter = this.getOpenCodeRuntimeAdapter(); if (!adapter) { const message = 'OpenCode runtime adapter is not registered for mixed team launch.'; lane.launchFinishedAtMs = Date.now(); const timingDiagnostic = buildOpenCodeSecondaryLaneTimingDiagnostic(lane); lane.state = 'finished'; lane.result = { runId: lane.runId ?? randomUUID(), teamName: run.teamName, launchPhase: 'finished', teamLaunchState: 'partial_failure', members: { [lane.member.name]: { memberName: lane.member.name, providerId: 'opencode', launchState: 'failed_to_start', agentToolAccepted: false, runtimeAlive: false, bootstrapConfirmed: false, hardFailure: true, hardFailureReason: 'opencode_runtime_adapter_missing', diagnostics: appendDiagnosticOnce([message], timingDiagnostic), }, }, warnings: [], diagnostics: appendDiagnosticOnce([...requestedDiagnostics, message], timingDiagnostic), }; lane.warnings = []; lane.diagnostics = appendDiagnosticOnce([...requestedDiagnostics, message], timingDiagnostic); await this.publishMixedSecondaryLaneStatusChange(run, lane); lane.state = 'finished'; return; } const migration = await migrateLegacyOpenCodeRuntimeState({ teamsBasePath: getTeamsBasePath(), teamName: run.teamName, laneId: lane.laneId, }); if (shouldAbortLaunch()) { await finishCancelledLane(); return; } await upsertOpenCodeRuntimeLaneIndexEntry({ teamsBasePath: getTeamsBasePath(), teamName: run.teamName, laneId: lane.laneId, state: migration.degraded ? 'degraded' : 'active', diagnostics: migration.diagnostics, }); if (shouldAbortLaunch()) { await finishCancelledLane(); return; } lane.state = 'launching'; lane.runId = lane.runId ?? randomUUID(); const laneRunId = lane.runId; lane.warnings = []; lane.diagnostics = [...requestedDiagnostics, ...migration.diagnostics]; const laneCwd = lane.member.cwd?.trim() || run.request.cwd; this.setSecondaryRuntimeRun({ teamName: run.teamName, runId: laneRunId, providerId: 'opencode', laneId: lane.laneId, memberName: lane.member.name, cwd: laneCwd, }); await this.publishMixedSecondaryLaneStatusChange(run, lane); const previousLaunchState = await this.launchStateStore.read(run.teamName); try { if (shouldAbortLaunch()) { await finishCancelledLane(); return; } await prepareOpenCodeRuntimeLaneForLaunchGeneration({ teamsBasePath: getTeamsBasePath(), teamName: run.teamName, laneId: lane.laneId, runId: laneRunId, reason: 'mixed_secondary_launch', }); if (shouldAbortLaunch()) { await finishCancelledLane(); return; } const appManagedLaunchPrompt = await this.buildOpenCodeSecondaryAppManagedLaunchPrompt( run, lane ); if (shouldAbortLaunch()) { await finishCancelledLane(); return; } const laneExpectedMembers: TeamRuntimeMemberSpec[] = [ { name: lane.member.name, role: lane.member.role, workflow: lane.member.workflow, isolation: lane.member.isolation === 'worktree' ? ('worktree' as const) : undefined, providerId: 'opencode', model: lane.member.model, effort: lane.member.effort, cwd: laneCwd, }, ]; const launchOpenCodeLane = () => adapter.launch({ runId: laneRunId, laneId: lane.laneId, teamName: run.teamName, cwd: laneCwd, prompt: appManagedLaunchPrompt, providerId: 'opencode', model: lane.member.model, effort: lane.member.effort, runtimeOnly: true, skipPermissions: run.request.skipPermissions !== false, expectedMembers: laneExpectedMembers, previousLaunchState, }); let rawResult: TeamRuntimeLaunchResult; try { rawResult = await launchOpenCodeLane(); } catch (error) { const message = error instanceof Error ? error.message : String(error); const staleManifestMessage = 'Bridge server runtime manifest high watermark is stale'; if ( message !== staleManifestMessage && message !== `OpenCode bridge failed: ${staleManifestMessage}` ) { throw error; } if (shouldAbortLaunch()) { await finishCancelledLane(); return; } const recovery = await prepareOpenCodeRuntimeLaneForLaunchGeneration({ teamsBasePath: getTeamsBasePath(), teamName: run.teamName, laneId: lane.laneId, runId: laneRunId, reason: 'mixed_secondary_launch_stale_manifest_recovery', forceReset: true, }); lane.diagnostics = appendDiagnosticOnce( [...lane.diagnostics, ...recovery.diagnostics], 'Retried OpenCode secondary launch after resetting stale runtime manifest.' ); if (shouldAbortLaunch()) { await finishCancelledLane(); return; } rawResult = await launchOpenCodeLane(); } if (shouldAbortLaunch()) { await finishCancelledLane(); return; } // Treat the bridge result as provisional. The guard below is the single // promotion gate that turns app-managed OpenCode bootstrap into // confirmed_alive only after durable lane evidence exists on disk. const result = await this.guardCommittedOpenCodeSecondaryLaneEvidence({ teamName: run.teamName, laneId: lane.laneId, memberName: lane.member.name, result: rawResult, }); if (shouldAbortLaunch()) { await finishCancelledLane(); return; } lane.launchFinishedAtMs = Date.now(); const timingDiagnostic = buildOpenCodeSecondaryLaneTimingDiagnostic(lane); const memberEvidence = result.members[lane.member.name]; const resultWithTiming: TeamRuntimeLaunchResult = timingDiagnostic ? { ...result, diagnostics: appendDiagnosticOnce(result.diagnostics, timingDiagnostic), members: { ...result.members, ...(memberEvidence ? { [lane.member.name]: { ...memberEvidence, diagnostics: appendDiagnosticOnce( memberEvidence.diagnostics ?? [], timingDiagnostic ), }, } : {}), }, } : result; const baseFailureDiagnostics = appendDiagnosticOnce( [...requestedDiagnostics, ...migration.diagnostics], timingDiagnostic ); const recoverableBootstrapPending = isRecoverableOpenCodeBootstrapPendingLaunchResult( resultWithTiming, lane.member.name ); const normalizedResult = recoverableBootstrapPending ? normalizeRecoverableOpenCodeBootstrapPendingLaunchResult( resultWithTiming, lane.member.name, baseFailureDiagnostics ) : resultWithTiming; lane.result = normalizedResult; this.syncOpenCodeRuntimeToolApprovals({ teamName: run.teamName, runId: laneRunId, laneId: lane.laneId, cwd: laneCwd, members: normalizedResult.members, expectedMembers: laneExpectedMembers, teamColor: run.request.color, teamDisplayName: run.request.displayName, }); lane.warnings = [...normalizedResult.warnings]; const launchDiagnostics = appendDiagnosticOnce( [...requestedDiagnostics, ...migration.diagnostics, ...normalizedResult.diagnostics], timingDiagnostic ); lane.diagnostics = launchDiagnostics; if (recoverableBootstrapPending) { await upsertOpenCodeRuntimeLaneIndexEntry({ teamsBasePath: getTeamsBasePath(), teamName: run.teamName, laneId: lane.laneId, state: 'active', diagnostics: collectOpenCodeSecondaryLaneFailureDiagnostics( normalizedResult, lane.member.name, baseFailureDiagnostics ), }).catch(() => undefined); } else if ( isDefinitiveOpenCodePreLaunchFailure(normalizedResult, lane.member.name) || normalizedResult.teamLaunchState === 'partial_failure' ) { const diagnostics = collectOpenCodeSecondaryLaneFailureDiagnostics( normalizedResult, lane.member.name, baseFailureDiagnostics ); await upsertOpenCodeRuntimeLaneIndexEntry({ teamsBasePath: getTeamsBasePath(), teamName: run.teamName, laneId: lane.laneId, state: 'degraded', diagnostics, }).catch(() => undefined); this.deleteSecondaryRuntimeRun(run.teamName, lane.laneId); } } catch (error) { if (shouldAbortLaunch()) { await finishCancelledLane(); return; } const message = error instanceof Error ? error.message : String(error); lane.launchFinishedAtMs = Date.now(); const timingDiagnostic = buildOpenCodeSecondaryLaneTimingDiagnostic(lane); lane.result = { runId: laneRunId, teamName: run.teamName, launchPhase: 'finished', teamLaunchState: 'partial_failure', members: { [lane.member.name]: { memberName: lane.member.name, providerId: 'opencode', launchState: 'failed_to_start', agentToolAccepted: false, runtimeAlive: false, bootstrapConfirmed: false, hardFailure: true, hardFailureReason: message, diagnostics: appendDiagnosticOnce([message], timingDiagnostic), }, }, warnings: [], diagnostics: appendDiagnosticOnce([message], timingDiagnostic), }; lane.warnings = []; lane.diagnostics = appendDiagnosticOnce( [...requestedDiagnostics, ...migration.diagnostics, message], timingDiagnostic ); await upsertOpenCodeRuntimeLaneIndexEntry({ teamsBasePath: getTeamsBasePath(), teamName: run.teamName, laneId: lane.laneId, state: 'degraded', diagnostics: appendDiagnosticOnce([message], timingDiagnostic), }).catch(() => undefined); this.deleteSecondaryRuntimeRun(run.teamName, lane.laneId); } await this.publishMixedSecondaryLaneStatusChange(run, lane); lane.state = 'finished'; } private async stopSingleMixedSecondaryRuntimeLane( run: ProvisioningRun, lane: MixedSecondaryRuntimeLaneState, reason: TeamRuntimeStopInput['reason'] ): Promise { const adapter = this.getOpenCodeRuntimeAdapter(); const previousLaunchState = await this.launchStateStore.read(run.teamName); await upsertOpenCodeRuntimeLaneIndexEntry({ teamsBasePath: getTeamsBasePath(), teamName: run.teamName, laneId: lane.laneId, state: 'stopped', diagnostics: [`OpenCode lane stop requested: ${reason}`], }).catch(() => undefined); try { if (adapter && lane.runId) { await adapter.stop({ runId: lane.runId, laneId: lane.laneId, teamName: run.teamName, cwd: lane.member.cwd?.trim() || run.request.cwd, providerId: 'opencode', reason, previousLaunchState, force: true, }); } } catch (error) { logger.warn( `[${run.teamName}] Failed to stop mixed OpenCode lane ${lane.laneId}: ${ error instanceof Error ? error.message : String(error) }` ); } finally { await clearOpenCodeRuntimeLaneStorage({ teamsBasePath: getTeamsBasePath(), teamName: run.teamName, laneId: lane.laneId, }).catch(() => undefined); this.deleteSecondaryRuntimeRun(run.teamName, lane.laneId); lane.runId = null; lane.state = 'finished'; lane.result = null; lane.warnings = []; lane.diagnostics = []; } } private launchQueuedMixedSecondaryLaneInBackground( run: ProvisioningRun, lane: MixedSecondaryRuntimeLaneState ): void { if (lane.state !== 'queued' || lane.launchScheduled) { return; } lane.queuedAtMs = lane.queuedAtMs ?? Date.now(); lane.launchScheduled = true; lane.runId = lane.runId ?? randomUUID(); const launch = async () => { try { if (run.cancelRequested || run.processKilled) { await clearOpenCodeRuntimeLaneStorage({ teamsBasePath: getTeamsBasePath(), teamName: run.teamName, laneId: lane.laneId, }).catch(() => undefined); this.deleteSecondaryRuntimeRun(run.teamName, lane.laneId); lane.state = 'finished'; return; } lane.state = 'launching'; await this.launchSingleMixedSecondaryLane(run, lane); } catch (error) { if (run.cancelRequested || run.processKilled) { await clearOpenCodeRuntimeLaneStorage({ teamsBasePath: getTeamsBasePath(), teamName: run.teamName, laneId: lane.laneId, }).catch(() => undefined); this.deleteSecondaryRuntimeRun(run.teamName, lane.laneId); return; } const message = error instanceof Error ? error.message : String(error); logger.warn( `[${run.teamName}] OpenCode secondary lane ${lane.laneId} crashed during launch orchestration: ${message}` ); lane.result = createUnexpectedMixedSecondaryLaneFailureResult({ runId: lane.runId ?? randomUUID(), teamName: run.teamName, memberName: lane.member.name, message, }); lane.warnings = []; lane.diagnostics = [...lane.diagnostics, message]; await upsertOpenCodeRuntimeLaneIndexEntry({ teamsBasePath: getTeamsBasePath(), teamName: run.teamName, laneId: lane.laneId, state: 'degraded', diagnostics: [message], }).catch(() => undefined); this.deleteSecondaryRuntimeRun(run.teamName, lane.laneId); await this.publishMixedSecondaryLaneStatusChange(run, lane).catch(() => undefined); lane.state = 'finished'; } }; const previousLaunch = run.mixedSecondaryLaneLaunchQueue ?? Promise.resolve(); const nextLaunch = previousLaunch.catch(() => undefined).then(launch); run.mixedSecondaryLaneLaunchQueue = nextLaunch.catch((error) => { logger.warn( `[${run.teamName}] OpenCode secondary lane launch queue failed: ${ error instanceof Error ? error.message : String(error) }` ); }); void run.mixedSecondaryLaneLaunchQueue; } private async launchMixedSecondaryLaneIfNeeded( run: ProvisioningRun ): Promise { if (run.cancelRequested || run.processKilled) { return this.launchStateStore.read(run.teamName).catch(() => null); } const mixedSecondaryLanes = run.mixedSecondaryLanes ?? []; if (mixedSecondaryLanes.length === 0) { return this.persistLaunchStateSnapshot(run, 'finished'); } const adapter = this.getOpenCodeRuntimeAdapter(); if (!adapter) { for (const lane of mixedSecondaryLanes) { lane.state = 'finished'; lane.result = { runId: lane.runId ?? randomUUID(), teamName: run.teamName, launchPhase: 'finished', teamLaunchState: 'partial_failure', members: { [lane.member.name]: { memberName: lane.member.name, providerId: 'opencode', launchState: 'failed_to_start', agentToolAccepted: false, runtimeAlive: false, bootstrapConfirmed: false, hardFailure: true, hardFailureReason: 'opencode_runtime_adapter_missing', diagnostics: ['OpenCode runtime adapter is not registered for mixed team launch.'], }, }, warnings: [], diagnostics: ['OpenCode runtime adapter is not registered for mixed team launch.'], }; lane.diagnostics = lane.result.diagnostics; await this.publishMixedSecondaryLaneStatusChange(run, lane); } return this.persistLaunchStateSnapshot(run, 'finished'); } for (const lane of mixedSecondaryLanes) { this.launchQueuedMixedSecondaryLaneInBackground(run, lane); } return this.persistLaunchStateSnapshot(run, this.getMixedSecondaryLaunchPhase(run)); } private async recoverStaleMixedSecondaryLaunchSnapshot( teamName: string, bootstrapSnapshot: PersistedTeamLaunchSnapshot | null, persistedSnapshot: PersistedTeamLaunchSnapshot | null ): Promise { if ( persistedSnapshot && this.hasMixedSecondaryLaunchMetadata(persistedSnapshot) && !this.shouldRecoverStalePersistedMixedLaunchSnapshot(persistedSnapshot) ) { return persistedSnapshot; } const teamMeta = await this.teamMetaStore.getMeta(teamName).catch(() => null); const leadLaunchIdentity = teamMeta?.launchIdentity; const leadProviderId = normalizeOptionalTeamProviderId(leadLaunchIdentity?.providerId) ?? normalizeOptionalTeamProviderId(teamMeta?.providerId); if (!leadProviderId || leadProviderId === 'opencode') { return null; } const membersMeta = await this.membersMetaStore.getMeta(teamName).catch(() => null); const activeMembers = (membersMeta?.members ?? []).filter( (member) => !member.removedAt && !isLeadMember({ name: member.name }) ); if (activeMembers.length === 0) { return null; } const projectPath = this.readPersistedTeamProjectPath(teamName); const laneIndex = await readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName).catch( () => ({ version: 1 as const, updatedAt: nowIso(), lanes: {} as Record< string, { laneId: string; state: 'active' | 'stopped' | 'degraded'; updatedAt: string; diagnostics?: string[]; } >, }) ); const bootstrapStatuses = snapshotToMemberSpawnStatuses(bootstrapSnapshot); const leadDefaults = { providerId: leadProviderId, providerBackendId: migrateProviderBackendId( leadProviderId, leadLaunchIdentity ? (leadLaunchIdentity.providerBackendId ?? teamMeta?.providerBackendId ?? membersMeta?.providerBackendId) : (teamMeta?.providerBackendId ?? membersMeta?.providerBackendId) ) ?? null, selectedFastMode: leadLaunchIdentity?.selectedFastMode ?? teamMeta?.fastMode, resolvedFastMode: typeof teamMeta?.launchIdentity?.resolvedFastMode === 'boolean' ? teamMeta.launchIdentity.resolvedFastMode : null, launchIdentity: teamMeta?.launchIdentity ?? null, }; const primaryMembers: TeamMember[] = []; const secondaryMembers: { laneId: string; member: TeamMember; leadDefaults: typeof leadDefaults; evidence?: { launchState?: MemberLaunchState; agentToolAccepted?: boolean; runtimeAlive?: boolean; bootstrapConfirmed?: boolean; hardFailure?: boolean; hardFailureReason?: string; pendingPermissionRequestIds?: string[]; runtimePid?: number; sessionId?: string; runtimeSessionId?: string; bootstrapEvidenceSource?: OpenCodeBootstrapEvidenceSource; bootstrapMode?: 'model_tool_checkin' | 'app_managed_context'; appManagedBootstrapCandidate?: OpenCodeAppManagedBootstrapCandidate; livenessKind?: TeamAgentRuntimeLivenessKind; pidSource?: TeamAgentRuntimePidSource; runtimeDiagnostic?: string; runtimeDiagnosticSeverity?: TeamAgentRuntimeDiagnosticSeverity; firstSpawnAcceptedAt?: string; diagnostics?: string[]; }; pendingReason?: string; }[] = []; let recoveredAny = false; for (const member of activeMembers) { const laneIdentity = buildPlannedMemberLaneIdentity({ leadProviderId, member: { name: member.name, providerId: normalizeOptionalTeamProviderId(member.providerId), }, }); if ( laneIdentity.laneKind !== 'secondary' || laneIdentity.laneOwnerProviderId !== 'opencode' ) { primaryMembers.push(member); continue; } let laneEntry = laneIndex.lanes[laneIdentity.laneId]; const persistedMember = persistedSnapshot?.members?.[member.name] ?? bootstrapSnapshot?.members?.[member.name]; if ( !laneEntry && persistedMember && isRecoverablePersistedOpenCodeRuntimeCandidate(persistedMember) && persistedMember.laneId === laneIdentity.laneId ) { const runtimeEvidence = await this.tryRecoverMissingOpenCodeSecondaryLaneFromRuntime({ teamName, laneId: laneIdentity.laneId, member, projectPath, previousLaunchState: persistedSnapshot ?? bootstrapSnapshot, persistedMember, }); if (runtimeEvidence) { recoveredAny = true; secondaryMembers.push({ laneId: laneIdentity.laneId, member, leadDefaults, evidence: { launchState: runtimeEvidence.launchState, agentToolAccepted: runtimeEvidence.agentToolAccepted, runtimeAlive: runtimeEvidence.runtimeAlive, bootstrapConfirmed: runtimeEvidence.bootstrapConfirmed, hardFailure: runtimeEvidence.hardFailure, hardFailureReason: runtimeEvidence.hardFailureReason, pendingPermissionRequestIds: runtimeEvidence.pendingPermissionRequestIds, runtimePid: runtimeEvidence.runtimePid, sessionId: runtimeEvidence.sessionId, runtimeSessionId: runtimeEvidence.sessionId, bootstrapEvidenceSource: runtimeEvidence.bootstrapEvidenceSource, bootstrapMode: runtimeEvidence.bootstrapMode, appManagedBootstrapCandidate: runtimeEvidence.appManagedBootstrapCandidate, livenessKind: runtimeEvidence.livenessKind, pidSource: runtimeEvidence.pidSource, runtimeDiagnostic: runtimeEvidence.runtimeDiagnostic, runtimeDiagnosticSeverity: runtimeEvidence.runtimeDiagnosticSeverity, firstSpawnAcceptedAt: persistedMember.firstSpawnAcceptedAt, diagnostics: runtimeEvidence.diagnostics, }, }); continue; } } if (laneEntry?.state === 'active') { const runtimeEvidence = await this.tryRecoverActiveOpenCodeSecondaryLaneFromRuntime({ teamName, laneId: laneIdentity.laneId, member, projectPath, previousLaunchState: persistedSnapshot ?? bootstrapSnapshot, }); if (isRecoverableOpenCodeRuntimeEvidence(runtimeEvidence)) { recoveredAny = true; secondaryMembers.push({ laneId: laneIdentity.laneId, member, leadDefaults, evidence: { launchState: runtimeEvidence.launchState, agentToolAccepted: runtimeEvidence.agentToolAccepted, runtimeAlive: runtimeEvidence.runtimeAlive, bootstrapConfirmed: runtimeEvidence.bootstrapConfirmed, hardFailure: runtimeEvidence.hardFailure, hardFailureReason: runtimeEvidence.hardFailureReason, pendingPermissionRequestIds: runtimeEvidence.pendingPermissionRequestIds, runtimePid: runtimeEvidence.runtimePid, sessionId: runtimeEvidence.sessionId, bootstrapEvidenceSource: runtimeEvidence.bootstrapEvidenceSource, bootstrapMode: runtimeEvidence.bootstrapMode, appManagedBootstrapCandidate: runtimeEvidence.appManagedBootstrapCandidate, livenessKind: runtimeEvidence.livenessKind, pidSource: runtimeEvidence.pidSource, runtimeDiagnostic: runtimeEvidence.runtimeDiagnostic, runtimeDiagnosticSeverity: runtimeEvidence.runtimeDiagnosticSeverity, firstSpawnAcceptedAt: persistedMember?.firstSpawnAcceptedAt, diagnostics: runtimeEvidence.diagnostics, }, }); continue; } const recovery = await recoverStaleOpenCodeRuntimeLaneIndexEntry({ teamsBasePath: getTeamsBasePath(), teamName, laneId: laneIdentity.laneId, }); if (recovery.stale) { recoveredAny = true; laneEntry = { laneId: laneIdentity.laneId, state: 'degraded', updatedAt: nowIso(), diagnostics: recovery.diagnostics, }; } } if (laneEntry?.state === 'degraded') { recoveredAny = true; const diagnostics = laneEntry.diagnostics?.length ? [...laneEntry.diagnostics] : [`OpenCode lane ${laneIdentity.laneId} is degraded and requires stop + relaunch.`]; secondaryMembers.push({ laneId: laneIdentity.laneId, member, leadDefaults, evidence: { launchState: 'failed_to_start', agentToolAccepted: false, runtimeAlive: false, bootstrapConfirmed: false, hardFailure: true, hardFailureReason: diagnostics[0], diagnostics, }, }); continue; } secondaryMembers.push({ laneId: laneIdentity.laneId, member, leadDefaults, pendingReason: 'Waiting for OpenCode secondary lane recovery.', }); } if (!recoveredAny) { return null; } const primaryStatuses = Object.fromEntries( primaryMembers.map((member) => [ member.name, bootstrapStatuses[member.name] ?? createInitialMemberSpawnStatusEntry(), ]) ); const recoveredSnapshot = this.runtimeLaneCoordinator.buildAggregateLaunchSnapshot({ teamName, leadSessionId: persistedSnapshot?.leadSessionId ?? bootstrapSnapshot?.leadSessionId, launchPhase: persistedSnapshot?.launchPhase === 'active' ? 'active' : bootstrapSnapshot?.launchPhase === 'active' ? 'active' : 'reconciled', leadDefaults, primaryMembers, primaryStatuses, secondaryMembers, }); return this.writeLaunchStateSnapshot(teamName, recoveredSnapshot); } private async tryRecoverActiveOpenCodeSecondaryLaneFromRuntime(params: { teamName: string; laneId: string; member: TeamMember; projectPath: string | null; previousLaunchState: PersistedTeamLaunchSnapshot | null; }): Promise { const adapter = this.getOpenCodeRuntimeAdapter(); const runtimeProjectPath = params.member.cwd?.trim() || params.projectPath; if (!adapter || !runtimeProjectPath) { return null; } try { const reconcileResult = await adapter.reconcile({ runId: randomUUID(), laneId: params.laneId, teamName: params.teamName, providerId: 'opencode', expectedMembers: [ { name: params.member.name, role: params.member.role, workflow: params.member.workflow, isolation: params.member.isolation === 'worktree' ? ('worktree' as const) : undefined, providerId: 'opencode', model: params.member.model, effort: params.member.effort, cwd: runtimeProjectPath, }, ], previousLaunchState: params.previousLaunchState, reason: 'startup_recovery', }); return reconcileResult.members[params.member.name] ?? null; } catch (error) { logger.warn( `[${params.teamName}] Failed to recover stale OpenCode lane ${params.laneId} from runtime bridge: ${ error instanceof Error ? error.message : String(error) }` ); return null; } } private async tryRecoverMissingOpenCodeSecondaryLaneFromRuntime(params: { teamName: string; laneId: string; member: TeamMember; projectPath: string | null; previousLaunchState: PersistedTeamLaunchSnapshot | null; persistedMember: PersistedTeamLaunchMemberState; }): Promise { const currentLaneIndex = await readOpenCodeRuntimeLaneIndex( getTeamsBasePath(), params.teamName ).catch(() => null); const currentEntry = currentLaneIndex?.lanes[params.laneId]; if (currentEntry?.state === 'degraded' || currentEntry?.state === 'stopped') { return null; } if (!isRecoverablePersistedOpenCodeRuntimeCandidate(params.persistedMember)) { return null; } const runtimeEvidence = await this.tryRecoverActiveOpenCodeSecondaryLaneFromRuntime({ teamName: params.teamName, laneId: params.laneId, member: params.member, projectPath: params.projectPath, previousLaunchState: params.previousLaunchState, }); if (!isRecoverableOpenCodeRuntimeEvidence(runtimeEvidence)) { return null; } const diagnostics = Array.from( new Set([ 'Recovered missing OpenCode runtime lane index from persisted runtime evidence.', ...(runtimeEvidence.diagnostics ?? []), ]) ); await upsertOpenCodeRuntimeLaneIndexEntry({ teamsBasePath: getTeamsBasePath(), teamName: params.teamName, laneId: params.laneId, state: 'active', diagnostics, }).catch((error: unknown) => { logger.warn( `[${params.teamName}] Failed to recover missing OpenCode lane index ${params.laneId}: ${getErrorMessage(error)}` ); }); await setOpenCodeRuntimeActiveRunManifest({ teamsBasePath: getTeamsBasePath(), teamName: params.teamName, laneId: params.laneId, runId: params.persistedMember.runtimeRunId ?? null, }).catch((error: unknown) => { logger.warn( `[${params.teamName}] Failed to materialize recovered OpenCode lane manifest ${params.laneId}: ${getErrorMessage(error)}` ); }); return { ...runtimeEvidence, diagnostics, }; } private async readLeadInboxMessagesForLaunchReconcile( teamName: string, leadName: string ): Promise { const inboxPath = path.join(getTeamsBasePath(), teamName, 'inboxes', `${leadName}.json`); try { const raw = await tryReadRegularFileUtf8(inboxPath, { timeoutMs: TEAM_JSON_READ_TIMEOUT_MS, maxBytes: TEAM_INBOX_MAX_BYTES, }); if (!raw) { return []; } const parsed = JSON.parse(raw) as unknown; if (!Array.isArray(parsed)) { return []; } return parsed.flatMap((item): LeadInboxLaunchReconcileMessage[] => { if (!item || typeof item !== 'object') { return []; } const row = item as Partial; return typeof row.from === 'string' && typeof row.text === 'string' && typeof row.timestamp === 'string' ? [ { from: row.from, text: row.text, timestamp: row.timestamp, messageId: row.messageId, }, ] : []; }); } catch { return []; } } private async hasBootstrapTranscriptLaunchReconcileOutcome( snapshot: PersistedTeamLaunchSnapshot ): Promise { const expectedMembers = this.getPersistedLaunchMemberNames(snapshot); for (const expected of expectedMembers) { const current = snapshot.members[expected]; if (!current || current.bootstrapConfirmed) { continue; } const acceptedAtMs = current.firstSpawnAcceptedAt != null ? Date.parse(current.firstSpawnAcceptedAt) : NaN; if ( current.launchState !== 'failed_to_start' || isBootstrapProofClearableLaunchFailureReason( current.hardFailureReason ?? current.runtimeDiagnostic ) ) { const runtimeProofObservedAt = await this.findBootstrapRuntimeProofObservedAt( snapshot.teamName, expected, current ); if (runtimeProofObservedAt) { return true; } } const transcriptOutcome = await this.findBootstrapTranscriptOutcome( snapshot.teamName, expected, Number.isFinite(acceptedAtMs) ? acceptedAtMs : null ); if ( transcriptOutcome && (transcriptOutcome.kind !== 'success' || !isPersistedOpenCodeSecondaryLaneMember(current)) ) { return true; } } return false; } private resolveBootstrapRuntimeMember( teamName: string, memberName: string ): PersistedRuntimeMemberLike | undefined { return this.readPersistedRuntimeMembers(teamName).find((member) => { const candidateName = typeof member.name === 'string' ? member.name.trim() : ''; return candidateName.length > 0 && matchesMemberNameOrBase(candidateName, memberName); }); } private getBootstrapRuntimeEventsPath( teamName: string, memberName: string, runtimeMember: PersistedRuntimeMemberLike | undefined ): string { const configuredPath = runtimeMember?.bootstrapRuntimeEventsPath?.trim(); if (configuredPath && isContainedTeamRuntimeEventsPath(teamName, configuredPath)) { return configuredPath; } const filePrefix = sanitizeProcessRuntimeEventFilePrefix(runtimeMember?.name ?? memberName); return path.join(getTeamRuntimeEventsDir(teamName), `${filePrefix}.runtime.jsonl`); } private async readRuntimeBootstrapProofEvents( eventsPath: string ): Promise[]> { let handle: fs.promises.FileHandle | null = null; try { const pathStat = await fs.promises.lstat(eventsPath); if (!pathStat.isFile()) { return []; } handle = await fs.promises.open(eventsPath, 'r'); const stat = await handle.stat(); if (!stat.isFile() || stat.size <= 0) { return []; } const start = Math.max(0, stat.size - BOOTSTRAP_RUNTIME_PROOF_TAIL_BYTES); const buffer = Buffer.alloc(stat.size - start); if (buffer.length === 0) { return []; } await handle.read(buffer, 0, buffer.length, start); const lines = buffer.toString('utf8').split('\n'); if (start > 0) { lines.shift(); } if (lines.length > BOOTSTRAP_RUNTIME_EVENT_MAX_LINES) { lines.splice(0, lines.length - BOOTSTRAP_RUNTIME_EVENT_MAX_LINES); } const events: Record[] = []; for (const rawLine of lines) { const line = rawLine.trim(); if (!line) continue; if (Buffer.byteLength(line, 'utf8') > BOOTSTRAP_RUNTIME_EVENT_MAX_LINE_BYTES) { continue; } try { const parsed = JSON.parse(line) as unknown; if ( parsed && typeof parsed === 'object' && (parsed as { version?: unknown }).version === 1 && typeof (parsed as { type?: unknown }).type === 'string' && typeof (parsed as { timestamp?: unknown }).timestamp === 'string' ) { events.push(parsed as Record); } } catch { // Ignore partial lines from concurrently written runtime event files. } } return events; } catch { return []; } finally { await handle?.close().catch(() => undefined); } } private isRuntimeBootstrapProofEventValid(input: { event: Record; detail: Record; teamName: string; memberName: string; runtimeMember?: PersistedRuntimeMemberLike; boundaryMs: number; }): boolean { const { event, detail, teamName, memberName, runtimeMember, boundaryMs } = input; if ( !validateBootstrapRuntimeProofEnvelope({ event, detail, expected: { teamName, boundaryMs, proofToken: runtimeMember?.bootstrapProofToken?.trim(), proofMode: runtimeMember?.bootstrapProofMode?.trim(), contextHash: runtimeMember?.bootstrapContextHash?.trim(), briefingHash: runtimeMember?.bootstrapBriefingHash?.trim(), runId: runtimeMember?.bootstrapRunId?.trim(), }, }) ) { return false; } const eventAgentName = typeof event.agentName === 'string' ? event.agentName.trim() : ''; const eventAgentId = typeof event.agentId === 'string' ? event.agentId.trim() : ''; const runtimeName = runtimeMember?.name?.trim() ?? ''; const runtimeAgentId = runtimeMember?.agentId?.trim() ?? ''; return ( (eventAgentName.length > 0 && (matchesMemberNameOrBase(eventAgentName, memberName) || (runtimeName.length > 0 && matchesTeamMemberIdentity(eventAgentName, runtimeName)))) || (eventAgentId.length > 0 && runtimeAgentId.length > 0 && eventAgentId === runtimeAgentId) ); } private async findBootstrapRuntimeProofObservedAt( teamName: string, memberName: string, member: Pick< PersistedTeamLaunchMemberState, 'firstSpawnAcceptedAt' | 'launchState' | 'hardFailureReason' > ): Promise { const runtimeMember = this.resolveBootstrapRuntimeMember(teamName, memberName); const boundaryText = member.firstSpawnAcceptedAt ?? runtimeMember?.bootstrapExpectedAfter; const boundaryMs = boundaryText ? Date.parse(boundaryText) : Number.NaN; if (!runtimeMember?.bootstrapProofToken && !Number.isFinite(boundaryMs)) { return null; } const eventsPath = this.getBootstrapRuntimeEventsPath(teamName, memberName, runtimeMember); const events = await this.readRuntimeBootstrapProofEvents(eventsPath); let latest: string | null = null; let latestMs = Number.NEGATIVE_INFINITY; for (const event of events) { const detail = parseBootstrapRuntimeProofDetail(event.detail); if ( !this.isRuntimeBootstrapProofEventValid({ event, detail, teamName, memberName, runtimeMember, boundaryMs, }) ) { continue; } const timestamp = typeof event.timestamp === 'string' ? event.timestamp : ''; const timestampMs = Date.parse(timestamp); if (Number.isFinite(timestampMs) && timestampMs >= latestMs) { latest = timestamp; latestMs = timestampMs; } } return latest; } private isRuntimeBootstrapTransportEventCurrent(input: { event: Record; teamName: string; memberName: string; runtimeMember?: PersistedRuntimeMemberLike; expectedPid?: number; expectedBootstrapRunId?: string; boundaryMs: number; }): boolean { const { event, teamName, memberName, runtimeMember, expectedPid, expectedBootstrapRunId } = input; const eventTeamName = typeof event.teamName === 'string' ? event.teamName.trim() : ''; if (eventTeamName && eventTeamName !== teamName) { return false; } const eventAgentId = typeof event.agentId === 'string' ? event.agentId.trim() : ''; const expectedAgentId = runtimeMember?.agentId?.trim() ?? ''; if (eventAgentId && expectedAgentId && eventAgentId !== expectedAgentId) { return false; } const eventAgentName = typeof event.agentName === 'string' ? event.agentName.trim() : ''; const runtimeName = runtimeMember?.name?.trim() ?? ''; if ( eventAgentName && !matchesMemberNameOrBase(eventAgentName, memberName) && !(runtimeName && matchesTeamMemberIdentity(eventAgentName, runtimeName)) ) { return false; } const eventBootstrapRunId = typeof event.bootstrapRunId === 'string' ? event.bootstrapRunId.trim() : ''; if ( expectedBootstrapRunId && eventBootstrapRunId && eventBootstrapRunId !== expectedBootstrapRunId ) { return false; } const eventPid = typeof event.pid === 'number' && Number.isFinite(event.pid) ? event.pid : NaN; if (typeof expectedPid === 'number' && expectedPid > 0 && eventPid !== expectedPid) { return false; } if (Number.isFinite(input.boundaryMs)) { const timestamp = typeof event.timestamp === 'string' ? event.timestamp : ''; const timestampMs = Date.parse(timestamp); if (!Number.isFinite(timestampMs) || timestampMs < input.boundaryMs) { return false; } } return true; } private async readProcessBootstrapTransportSummary(input: { teamName: string; memberName: string; member: PersistedTeamLaunchMemberState; }): Promise { const { teamName, memberName, member } = input; const runtimeMember = this.resolveBootstrapRuntimeMember(teamName, memberName); const memberRecord = member as unknown as Record; const runtimeBackendType = runtimeMember?.backendType?.trim() || (typeof memberRecord.backendType === 'string' ? memberRecord.backendType.trim() : ''); const processPaneId = runtimeMember?.tmuxPaneId?.trim() || (typeof memberRecord.tmuxPaneId === 'string' ? memberRecord.tmuxPaneId.trim() : ''); if (runtimeBackendType !== 'process' && !processPaneId?.startsWith('process:')) { return null; } const boundaryText = member.firstSpawnAcceptedAt ?? runtimeMember?.bootstrapExpectedAfter; const boundaryMs = boundaryText ? Date.parse(boundaryText) : Number.NaN; const expectedPid = typeof member.runtimePid === 'number' && member.runtimePid > 0 ? member.runtimePid : typeof runtimeMember?.runtimePid === 'number' && runtimeMember.runtimePid > 0 ? runtimeMember.runtimePid : undefined; const expectedBootstrapRunId = runtimeMember?.bootstrapRunId?.trim() || (typeof member.runtimeRunId === 'string' ? member.runtimeRunId.trim() : '') || (typeof memberRecord.bootstrapRunId === 'string' ? memberRecord.bootstrapRunId.trim() : ''); if (!expectedBootstrapRunId && !Number.isFinite(boundaryMs) && !expectedPid) { return null; } const eventsPath = this.getBootstrapRuntimeEventsPath(teamName, memberName, runtimeMember); // Runtime event paths are persisted by process teammates. Keep both path // containment and payload identity checks so stale or foreign JSONL cannot // affect another team/member launch. if (!isContainedTeamRuntimeEventsPath(teamName, eventsPath)) { return null; } const events = await this.readRuntimeBootstrapProofEvents(eventsPath); const currentEvents = events.filter((event) => this.isRuntimeBootstrapTransportEventCurrent({ event, teamName, memberName, runtimeMember, expectedPid, expectedBootstrapRunId, boundaryMs, }) ); return summarizeProcessBootstrapTransportEvents( currentEvents as ProcessBootstrapTransportEvent[] ); } private applyProcessBootstrapTransportOverlay(input: { member: PersistedTeamLaunchMemberState; summary: ProcessBootstrapTransportSummary | null; launchPhase: PersistedTeamLaunchPhase; finalTimeoutReached?: boolean; }): PersistedTeamLaunchMemberState { const { member, summary } = input; if ( !summary || member.bootstrapConfirmed || member.launchState === 'confirmed_alive' || member.launchState === 'skipped_for_launch' || member.launchState === 'runtime_pending_permission' || member.skippedForLaunch === true ) { return member; } const existingFailure = member.hardFailureReason ?? member.runtimeDiagnostic; if ( member.launchState === 'failed_to_start' && member.hardFailure === true && !isAutoClearableLaunchFailureReason(existingFailure) ) { return member; } const projectionPhase = deriveProcessTransportProjectionPhase({ launchPhase: input.launchPhase, finalTimeoutReached: input.finalTimeoutReached, }); const base: PersistedTeamLaunchMemberState = { ...member, agentToolAccepted: true, lastEvaluatedAt: nowIso(), }; if (summary.terminalFailure) { // Terminal transport events are failures for bootstrap only. They are // surfaced as hard failure text, but still do not create readiness proof. const reason = summary.terminalFailure.reason; return { ...base, launchState: 'failed_to_start', bootstrapConfirmed: false, hardFailure: true, hardFailureReason: reason, runtimeDiagnostic: reason, runtimeDiagnosticSeverity: 'error', diagnostics: mergeRuntimeDiagnostics(base.diagnostics, [reason, summary.lastStage]), sources: { ...(base.sources ?? {}), hardFailureSignal: true, }, }; } if (!summary.hasProgress) { return member; } if (projectionPhase === 'final') { const reason = buildProcessBootstrapTimeoutDiagnostic(summary); return { ...base, launchState: 'failed_to_start', bootstrapConfirmed: false, hardFailure: true, hardFailureReason: reason, runtimeDiagnostic: reason, runtimeDiagnosticSeverity: 'error', diagnostics: mergeRuntimeDiagnostics(base.diagnostics, [reason, summary.lastStage]), sources: { ...(base.sources ?? {}), hardFailureSignal: true, }, }; } const runtimeDiagnostic = buildProcessBootstrapPendingDiagnostic(summary); // Active launch progress remains pending. A submitted bootstrap prompt is // not enough for confirmed_alive; durable bootstrap proof is handled by // the runtime/transcript evidence path above. return { ...base, launchState: 'runtime_pending_bootstrap', bootstrapConfirmed: false, hardFailure: false, hardFailureReason: undefined, runtimeDiagnostic, runtimeDiagnosticSeverity: summary.submitted ? 'info' : 'warning', diagnostics: mergeRuntimeDiagnostics(base.diagnostics, [ runtimeDiagnostic, summary.lastStage, ]), sources: { ...(base.sources ?? {}), hardFailureSignal: undefined, }, }; } private async applyBootstrapTranscriptEvidenceOverlay( snapshot: PersistedTeamLaunchSnapshot | null ): Promise { if (!snapshot) { return null; } let changed = false; const nextMembers: Record = { ...snapshot.members }; for (const expected of this.getPersistedLaunchMemberNames(snapshot)) { const current = nextMembers[expected]; if ( !current || current.bootstrapConfirmed || isPersistedOpenCodeSecondaryLaneMember(current) ) { continue; } const failureReason = current.hardFailureReason ?? current.runtimeDiagnostic; const provisionedButNotAliveFailure = isProvisionedButNotAliveFailureReason(failureReason); if ( provisionedButNotAliveFailure && hasUnsafeProvisionedButNotAliveRuntimeEvidence(current) ) { continue; } const canClearFailedBootstrap = current.launchState !== 'failed_to_start' || isBootstrapProofClearableLaunchFailureReason(failureReason); if (!canClearFailedBootstrap) { continue; } const acceptedAtMs = current.firstSpawnAcceptedAt != null ? Date.parse(current.firstSpawnAcceptedAt) : NaN; const runtimeProofObservedAt = await this.findBootstrapRuntimeProofObservedAt( snapshot.teamName, expected, current ); const transcriptOutcome = await this.findBootstrapTranscriptOutcome( snapshot.teamName, expected, Number.isFinite(acceptedAtMs) ? acceptedAtMs : null ); const observedAt = runtimeProofObservedAt ?? (transcriptOutcome?.kind === 'success' ? transcriptOutcome.observedAt : null); if (!observedAt) { continue; } const nextMember: PersistedTeamLaunchMemberState = { ...current, agentToolAccepted: true, bootstrapConfirmed: true, runtimeAlive: runtimeProofObservedAt ? true : current.runtimeAlive === true || provisionedButNotAliveFailure, hardFailure: false, hardFailureReason: undefined, lastHeartbeatAt: current.lastHeartbeatAt ?? observedAt, lastRuntimeAliveAt: runtimeProofObservedAt ? (current.lastRuntimeAliveAt ?? observedAt) : current.lastRuntimeAliveAt, lastEvaluatedAt: nowIso(), sources: { ...(current.sources ?? {}), hardFailureSignal: undefined, }, diagnostics: undefined, }; nextMember.launchState = deriveMemberLaunchState(nextMember); nextMembers[expected] = nextMember; changed = true; } if (!changed) { return snapshot; } return createPersistedLaunchSnapshot({ teamName: snapshot.teamName, expectedMembers: snapshot.expectedMembers, bootstrapExpectedMembers: snapshot.bootstrapExpectedMembers, leadSessionId: snapshot.leadSessionId, launchPhase: snapshot.launchPhase, members: nextMembers, updatedAt: nowIso(), }); } private needsBootstrapAcceptanceReconcile( snapshot: PersistedTeamLaunchSnapshot | null, bootstrapSnapshot: PersistedTeamLaunchSnapshot | null ): boolean { if (!snapshot || !bootstrapSnapshot) { return false; } for (const expected of this.getPersistedLaunchMemberNames(snapshot)) { const current = snapshot.members[expected]; const bootstrapMember = bootstrapSnapshot.members[expected]; if (!current || !bootstrapMember) { continue; } if ( bootstrapMember.bootstrapConfirmed === true && !isPersistedOpenCodeSecondaryLaneMember(current) && isBootstrapMemberEvidenceCurrentForMember(current, bootstrapMember, 'confirmation') ) { const currentConfirmed = current.bootstrapConfirmed === true || current.launchState === 'confirmed_alive'; const failureReason = current.hardFailureReason ?? current.runtimeDiagnostic; const hasAutoClearableFailure = (current.launchState === 'failed_to_start' || current.hardFailure === true) && isBootstrapProofClearableLaunchFailureReason(failureReason); if (!currentConfirmed || hasAutoClearableFailure) { return true; } } const bootstrapProvesSpawnAcceptance = (bootstrapMember.agentToolAccepted === true || typeof bootstrapMember.firstSpawnAcceptedAt === 'string') && isBootstrapMemberEvidenceCurrentForMember(current, bootstrapMember, 'acceptance'); if (!bootstrapProvesSpawnAcceptance) { continue; } const currentProvesSpawnAcceptance = current.agentToolAccepted === true || typeof current.firstSpawnAcceptedAt === 'string'; if (!currentProvesSpawnAcceptance) { return true; } if (isNeverSpawnedDuringLaunchReason(current.hardFailureReason)) { return true; } } return false; } private needsConfirmedBootstrapDiagnosticReconcile( snapshot: PersistedTeamLaunchSnapshot | null ): boolean { if (!snapshot) { return false; } for (const member of Object.values(snapshot.members)) { if ( member?.bootstrapConfirmed !== true || member.hardFailure === true || isPersistedOpenCodeSecondaryLaneMember(member) ) { continue; } if ( member.livenessKind === 'stale_metadata' || member.livenessKind === 'registered_only' || member.pidSource === 'persisted_metadata' || shouldClearRuntimeDiagnosticAfterBootstrapConfirmation(member.runtimeDiagnostic) ) { return true; } } return false; } private cleanConfirmedBootstrapRuntimeDiagnostics( snapshot: PersistedTeamLaunchSnapshot | null ): PersistedTeamLaunchSnapshot | null { if (!snapshot) { return null; } let changed = false; const updatedAt = nowIso(); const members: Record = { ...snapshot.members }; for (const memberName of this.getPersistedLaunchMemberNames(snapshot)) { const current = members[memberName]; if ( !current || current.bootstrapConfirmed !== true || current.hardFailure === true || isPersistedOpenCodeSecondaryLaneMember(current) ) { continue; } const hasConfirmedBootstrapStaleRuntimeState = current.livenessKind === 'stale_metadata' || current.livenessKind === 'registered_only' || current.pidSource === 'persisted_metadata' || shouldClearRuntimeDiagnosticAfterBootstrapConfirmation(current.runtimeDiagnostic) || current.bootstrapStalled === true; if (!hasConfirmedBootstrapStaleRuntimeState) { continue; } const next: PersistedTeamLaunchMemberState = { ...current, livenessKind: current.livenessKind === 'stale_metadata' || current.livenessKind === 'registered_only' || current.livenessKind == null ? 'confirmed_bootstrap' : current.livenessKind, pidSource: current.pidSource === 'persisted_metadata' || current.pidSource == null ? 'runtime_bootstrap' : current.pidSource, bootstrapStalled: undefined, diagnostics: undefined, lastEvaluatedAt: updatedAt, }; if (shouldClearRuntimeDiagnosticAfterBootstrapConfirmation(next.runtimeDiagnostic)) { next.runtimeDiagnostic = undefined; next.runtimeDiagnosticSeverity = undefined; } else if (!next.runtimeDiagnostic) { next.runtimeDiagnosticSeverity = undefined; } next.launchState = deriveMemberLaunchState(next); members[memberName] = next; changed = true; } if (!changed) { return snapshot; } return createPersistedLaunchSnapshot({ teamName: snapshot.teamName, expectedMembers: snapshot.expectedMembers, bootstrapExpectedMembers: snapshot.bootstrapExpectedMembers, leadSessionId: snapshot.leadSessionId, launchPhase: snapshot.launchPhase, members, updatedAt, }); } private async reconcilePersistedLaunchState(teamName: string): Promise<{ snapshot: ReturnType | null; statuses: Record; }> { const bootstrapSnapshot = await readBootstrapLaunchSnapshot(teamName); const persisted = await this.launchStateStore.read(teamName); const metaMembers = await this.membersMetaStore.getMembers(teamName).catch(() => []); const recoveredMixedSnapshot = await this.recoverStaleMixedSecondaryLaunchSnapshot( teamName, bootstrapSnapshot, persisted ); const filteredRecoveredMixedSnapshot = recoveredMixedSnapshot ? this.filterRemovedMembersFromLaunchSnapshot(recoveredMixedSnapshot, metaMembers) : null; const overlaidRecoveredMixedSnapshot = filteredRecoveredMixedSnapshot ? await this.applyOpenCodeSecondaryEvidenceOverlay({ teamName, snapshot: filteredRecoveredMixedSnapshot, previousSnapshot: persisted, metaMembers, }) : null; const recoveredMixedSnapshotWithBootstrapStall = this.applyOpenCodeSecondaryBootstrapStallOverlay(overlaidRecoveredMixedSnapshot); const stableRecoveredMixedSnapshotWithCommittedEvidence = recoveredMixedSnapshotWithBootstrapStall && (this.hasCommittedOpenCodeSecondaryEvidenceOverlayDelta( recoveredMixedSnapshotWithBootstrapStall, persisted ) || recoveredMixedSnapshotWithBootstrapStall !== overlaidRecoveredMixedSnapshot) ? await this.writeLaunchStateSnapshot(teamName, recoveredMixedSnapshotWithBootstrapStall) : recoveredMixedSnapshotWithBootstrapStall; const promotedRecoveredMixedSnapshot = promoteOpenCodePersistedFailureReasonsFromDiagnostics( stableRecoveredMixedSnapshotWithCommittedEvidence ); const cleanedRecoveredMixedSnapshot = this.cleanConfirmedBootstrapRuntimeDiagnostics( promotedRecoveredMixedSnapshot ); const stableRecoveredMixedSnapshot = cleanedRecoveredMixedSnapshot && (promotedRecoveredMixedSnapshot !== stableRecoveredMixedSnapshotWithCommittedEvidence || cleanedRecoveredMixedSnapshot !== promotedRecoveredMixedSnapshot) ? await this.writeLaunchStateSnapshot(teamName, cleanedRecoveredMixedSnapshot) : cleanedRecoveredMixedSnapshot; const filteredBootstrapSnapshot = bootstrapSnapshot ? this.filterRemovedMembersFromLaunchSnapshot(bootstrapSnapshot, metaMembers) : null; const overlaidBootstrapSnapshot = await this.applyBootstrapTranscriptEvidenceOverlay(filteredBootstrapSnapshot); if ( stableRecoveredMixedSnapshot && !this.needsBootstrapAcceptanceReconcile( stableRecoveredMixedSnapshot, overlaidBootstrapSnapshot ) && !this.needsConfirmedBootstrapDiagnosticReconcile(stableRecoveredMixedSnapshot) && !(await this.hasBootstrapTranscriptLaunchReconcileOutcome(stableRecoveredMixedSnapshot)) ) { return { snapshot: stableRecoveredMixedSnapshot, statuses: snapshotToMemberSpawnStatuses(stableRecoveredMixedSnapshot), }; } const filteredPersistedBase = stableRecoveredMixedSnapshot ?? (persisted ? this.filterRemovedMembersFromLaunchSnapshot(persisted, metaMembers) : null); const filteredPersisted = filteredPersistedBase ? await this.applyOpenCodeSecondaryEvidenceOverlay({ teamName, snapshot: filteredPersistedBase, previousSnapshot: persisted, metaMembers, }) : null; const filteredPersistedWithBootstrapStall = this.applyOpenCodeSecondaryBootstrapStallOverlay(filteredPersisted); const shouldPersistCommittedEvidenceOverlay = this.hasCommittedOpenCodeSecondaryEvidenceOverlayDelta( filteredPersistedWithBootstrapStall, persisted ); const promotedPersisted = promoteOpenCodePersistedFailureReasonsFromDiagnostics( filteredPersistedWithBootstrapStall ); const shouldPersistFailureReasonPromotion = promotedPersisted !== filteredPersistedWithBootstrapStall; const cleanedPersisted = this.cleanConfirmedBootstrapRuntimeDiagnostics(promotedPersisted); const shouldPersistConfirmedBootstrapDiagnosticCleanup = cleanedPersisted !== promotedPersisted; const shouldPersistBootstrapStallOverlay = filteredPersistedWithBootstrapStall !== filteredPersisted; const persistedWithCommittedEvidence = cleanedPersisted && (shouldPersistCommittedEvidenceOverlay || shouldPersistFailureReasonPromotion || shouldPersistConfirmedBootstrapDiagnosticCleanup || shouldPersistBootstrapStallOverlay) ? await this.writeLaunchStateSnapshot(teamName, cleanedPersisted) : cleanedPersisted; const preferredSnapshot = choosePreferredLaunchSnapshot( overlaidBootstrapSnapshot, persistedWithCommittedEvidence ); const bootstrapSelectionWouldCollapseMixedLaunch = preferredSnapshot && preferredSnapshot === overlaidBootstrapSnapshot && preferredSnapshot.teamLaunchState === 'clean_success' && !this.hasMixedLaunchMetadata(preferredSnapshot) && this.hasMixedLaunchMetadata(persistedWithCommittedEvidence); if ( preferredSnapshot && preferredSnapshot === overlaidBootstrapSnapshot && !bootstrapSelectionWouldCollapseMixedLaunch ) { if (persistedWithCommittedEvidence) { if ( preferredSnapshot.teamLaunchState === 'clean_success' && !this.hasMixedLaunchMetadata(preferredSnapshot) ) { await this.clearPersistedLaunchState(teamName); return { snapshot: preferredSnapshot, statuses: snapshotToMemberSpawnStatuses(preferredSnapshot), }; } const writtenSnapshot = await this.writeLaunchStateSnapshot(teamName, preferredSnapshot); return { snapshot: writtenSnapshot, statuses: snapshotToMemberSpawnStatuses(writtenSnapshot), }; } return { snapshot: preferredSnapshot, statuses: snapshotToMemberSpawnStatuses(preferredSnapshot), }; } if (!persistedWithCommittedEvidence) { return { snapshot: null, statuses: {} }; } const configPath = path.join(getTeamsBasePath(), teamName, 'config.json'); let configMembers = new Set(); let configBootstrapRunIds = new Map(); let leadName = 'team-lead'; try { const raw = await tryReadRegularFileUtf8(configPath, { timeoutMs: TEAM_JSON_READ_TIMEOUT_MS, maxBytes: TEAM_CONFIG_MAX_BYTES, }); if (raw) { const config = JSON.parse(raw) as { members?: { name?: string; agentType?: string; bootstrapRunId?: string }[]; }; const configuredMembers = config.members ?? []; leadName = configuredMembers.find((member) => isLeadMember(member))?.name?.trim() || leadName; configMembers = new Set( configuredMembers .map((member) => (typeof member?.name === 'string' ? member.name.trim() : '')) .filter((name) => name.length > 0 && !isLeadMember({ name })) ); configBootstrapRunIds = new Map( configuredMembers.flatMap((member) => { const name = typeof member?.name === 'string' ? member.name.trim() : ''; const runId = typeof member?.bootstrapRunId === 'string' ? member.bootstrapRunId.trim() : ''; return name.length > 0 && runId.length > 0 && !isLeadMember({ name }) ? [[name, runId] as const] : []; }) ); } } catch { // best-effort } const leadInboxMessages = await this.readLeadInboxMessagesForLaunchReconcile( teamName, leadName ); if ( this.hasMixedLaunchMetadata(persistedWithCommittedEvidence) && !this.hasLeadInboxLaunchReconcileHeartbeat( persistedWithCommittedEvidence, leadInboxMessages ) && !this.needsBootstrapAcceptanceReconcile( persistedWithCommittedEvidence, overlaidBootstrapSnapshot ) && !this.needsConfirmedBootstrapDiagnosticReconcile(persistedWithCommittedEvidence) && !(await this.hasBootstrapTranscriptLaunchReconcileOutcome(persistedWithCommittedEvidence)) ) { return { snapshot: persistedWithCommittedEvidence, statuses: snapshotToMemberSpawnStatuses(persistedWithCommittedEvidence), }; } const liveRuntimeByMember = await this.getLiveTeamAgentRuntimeMetadata(teamName); const nextMembers = { ...persistedWithCommittedEvidence.members }; const persistedMemberNames = this.getPersistedLaunchMemberNames(persistedWithCommittedEvidence); const now = nowIso(); for (const expected of persistedMemberNames) { const bootstrapMember = bootstrapSnapshot?.members[expected]; let current = nextMembers[expected] ?? { name: expected, launchState: 'starting', agentToolAccepted: false, runtimeAlive: false, bootstrapConfirmed: false, hardFailure: false, lastEvaluatedAt: now, }; const isOpenCodeSecondaryLaneMember = isPersistedOpenCodeSecondaryLaneMember(current); const matchedConfigNames = [...configMembers].filter((name) => matchesObservedMemberNameForExpected(name, expected) ); const configBootstrapRunId = matchedConfigNames .map((name) => configBootstrapRunIds.get(name)) .find((runId): runId is string => typeof runId === 'string' && runId.length > 0); const currentBootstrapEvidenceBoundary = configBootstrapRunId ? { ...current, runtimeRunId: configBootstrapRunId } : current; if ( bootstrapMember?.agentToolAccepted && !current.agentToolAccepted && isBootstrapMemberEvidenceCurrentForMember( currentBootstrapEvidenceBoundary, bootstrapMember, 'acceptance' ) ) { current.agentToolAccepted = true; current.firstSpawnAcceptedAt = current.firstSpawnAcceptedAt ?? bootstrapMember.firstSpawnAcceptedAt; } if ( bootstrapMember?.bootstrapConfirmed && !current.bootstrapConfirmed && !isOpenCodeSecondaryLaneMember && isBootstrapMemberEvidenceCurrentForMember( currentBootstrapEvidenceBoundary, bootstrapMember, 'confirmation' ) ) { current.bootstrapConfirmed = true; current.lastHeartbeatAt = current.lastHeartbeatAt ?? bootstrapMember.lastHeartbeatAt; } const runtimeMetadataCandidates = [...liveRuntimeByMember.entries()].filter(([name]) => matchesObservedMemberNameForExpected(name, expected) ); const runtimeMetadata = runtimeMetadataCandidates.find(([, metadata]) => metadata.alive) ?? runtimeMetadataCandidates[0]; const observedRuntimeAlive = runtimeMetadata?.[1].alive === true; const heartbeatMessage = this.selectLatestLeadInboxLaunchReconcileMessage( leadInboxMessages, persistedMemberNames, expected, current.firstSpawnAcceptedAt ); const heartbeatReason = heartbeatMessage ? extractBootstrapFailureReason(heartbeatMessage.text) : null; const acceptedAtMs = current.firstSpawnAcceptedAt != null ? Date.parse(current.firstSpawnAcceptedAt) : NaN; const initialFailureReason = current.hardFailureReason ?? current.runtimeDiagnostic; const hadAutoClearableFailure = isAutoClearableLaunchFailureReason(initialFailureReason); const requiresConfirmedBootstrapToClearFailure = isCliProvisionedButNotAliveFailureReason(initialFailureReason); const metadataRuntimeDiagnostic = runtimeMetadata?.[1].runtimeDiagnostic; const metadataRuntimeDiagnosticSeverity = runtimeMetadata?.[1].runtimeDiagnosticSeverity; const metadataLivenessKind = runtimeMetadata?.[1].livenessKind; const refreshedRuntimeDiagnosticEvidence = metadataRuntimeDiagnostic && current.runtimeDiagnostic && metadataRuntimeDiagnostic !== current.runtimeDiagnostic ? `${metadataRuntimeDiagnostic}; ${current.runtimeDiagnostic}` : (metadataRuntimeDiagnostic ?? current.runtimeDiagnostic); const hasUnsafeProvisionedButNotAliveFailure = requiresConfirmedBootstrapToClearFailure && hasUnsafeProvisionedButNotAliveRuntimeEvidence({ ...current, runtimeDiagnostic: refreshedRuntimeDiagnosticEvidence, runtimeDiagnosticSeverity: metadataRuntimeDiagnosticSeverity ?? current.runtimeDiagnosticSeverity, livenessKind: metadataLivenessKind ?? current.livenessKind, }); const shouldPreserveUnsafeMetadataLivenessKind = hasUnsafeProvisionedButNotAliveFailure && (metadataLivenessKind === 'not_found' || metadataLivenessKind === 'shell_only' || metadataLivenessKind === 'runtime_process_candidate' || ((metadataLivenessKind === 'registered_only' || metadataLivenessKind === 'stale_metadata') && (metadataRuntimeDiagnosticSeverity ?? current.runtimeDiagnosticSeverity) !== 'error' && !mentionsProcessTableUnavailable(refreshedRuntimeDiagnosticEvidence) && !mentionsProcessTableUnavailable(initialFailureReason))); const nextLivenessKind = current.bootstrapConfirmed ? metadataLivenessKind === 'runtime_process' || metadataLivenessKind === 'confirmed_bootstrap' || shouldPreserveUnsafeMetadataLivenessKind ? metadataLivenessKind : current.livenessKind === 'stale_metadata' || current.livenessKind === 'registered_only' ? 'confirmed_bootstrap' : (current.livenessKind ?? 'confirmed_bootstrap') : (metadataLivenessKind ?? current.livenessKind); current.runtimeAlive = observedRuntimeAlive; current.lastRuntimeAliveAt = observedRuntimeAlive ? now : current.lastRuntimeAliveAt; current.livenessKind = nextLivenessKind; current.pidSource = runtimeMetadata?.[1].pidSource; const shouldKeepUnsafeRuntimeDiagnostic = hasUnsafeProvisionedButNotAliveFailure && (metadataRuntimeDiagnostic == null || (current.runtimeDiagnosticSeverity === 'error' && metadataRuntimeDiagnosticSeverity !== 'error')); current.runtimeDiagnostic = shouldKeepUnsafeRuntimeDiagnostic ? current.runtimeDiagnostic : metadataRuntimeDiagnostic; current.runtimeDiagnosticSeverity = shouldKeepUnsafeRuntimeDiagnostic ? current.runtimeDiagnosticSeverity : metadataRuntimeDiagnosticSeverity; current.sources = { ...(current.sources ?? {}), processAlive: observedRuntimeAlive || undefined, configRegistered: matchedConfigNames.length > 0 || undefined, configDrift: heartbeatMessage != null && matchedConfigNames.length === 0 ? true : current.sources?.configDrift, inboxHeartbeat: heartbeatMessage != null ? true : current.sources?.inboxHeartbeat, }; const bootstrapProvesSpawnAcceptance = bootstrapMember?.agentToolAccepted === true || typeof bootstrapMember?.firstSpawnAcceptedAt === 'string'; const currentProvesSpawnAcceptance = current.agentToolAccepted === true || typeof current.firstSpawnAcceptedAt === 'string'; if ( hadAutoClearableFailure && !requiresConfirmedBootstrapToClearFailure && (bootstrapProvesSpawnAcceptance || currentProvesSpawnAcceptance) ) { current.hardFailure = false; current.hardFailureReason = undefined; if (current.sources) { current.sources.hardFailureSignal = undefined; } } if ( current.bootstrapConfirmed && !isOpenCodeSecondaryLaneMember && !hasUnsafeProvisionedButNotAliveFailure && isBootstrapProofClearableLaunchFailureReason(current.hardFailureReason) ) { if (isProvisionedButNotAliveFailureReason(current.hardFailureReason)) { current.runtimeAlive = true; } current.hardFailure = false; current.hardFailureReason = undefined; if (current.sources) { current.sources.hardFailureSignal = undefined; } } if (heartbeatReason) { current.hardFailure = true; current.hardFailureReason = heartbeatReason; current.sources.hardFailureSignal = true; } else if (heartbeatMessage && !isOpenCodeSecondaryLaneMember) { current.bootstrapConfirmed = true; current.lastHeartbeatAt = heartbeatMessage.timestamp; current.hardFailure = false; current.hardFailureReason = undefined; } const canApplyBootstrapSuccess = !heartbeatReason && !hasUnsafeProvisionedButNotAliveFailure && (current.launchState !== 'failed_to_start' || hadAutoClearableFailure || isBootstrapProofClearableLaunchFailureReason( current.hardFailureReason ?? current.runtimeDiagnostic )); if (!current.bootstrapConfirmed && canApplyBootstrapSuccess) { const runtimeProofObservedAt = !isOpenCodeSecondaryLaneMember ? await this.findBootstrapRuntimeProofObservedAt(teamName, expected, current) : null; const transcriptOutcome = runtimeProofObservedAt ? null : await this.findBootstrapTranscriptOutcome( teamName, expected, Number.isFinite(acceptedAtMs) ? acceptedAtMs : null ); const bootstrapObservedAt = runtimeProofObservedAt ?? (transcriptOutcome?.kind === 'success' ? transcriptOutcome.observedAt : null); if (bootstrapObservedAt && !isOpenCodeSecondaryLaneMember) { current.bootstrapConfirmed = true; current.lastHeartbeatAt = current.lastHeartbeatAt ?? bootstrapObservedAt; current.runtimeAlive = runtimeProofObservedAt ? true : current.runtimeAlive === true || requiresConfirmedBootstrapToClearFailure; current.lastRuntimeAliveAt = runtimeProofObservedAt ? (current.lastRuntimeAliveAt ?? bootstrapObservedAt) : current.lastRuntimeAliveAt; current.hardFailure = false; current.hardFailureReason = undefined; if (current.sources) { current.sources.hardFailureSignal = undefined; } } else if (transcriptOutcome?.kind === 'failure' && !current.hardFailure) { current.hardFailure = true; current.hardFailureReason = transcriptOutcome.reason; current.sources.hardFailureSignal = true; } } const graceExpired = Number.isFinite(acceptedAtMs) && Date.now() - acceptedAtMs >= MEMBER_LAUNCH_GRACE_MS; if (!isOpenCodeSecondaryLaneMember) { current = this.applyProcessBootstrapTransportOverlay({ member: current, summary: await this.readProcessBootstrapTransportSummary({ teamName, memberName: expected, member: current, }), launchPhase: persistedWithCommittedEvidence.launchPhase, finalTimeoutReached: graceExpired, }); } if (current.bootstrapConfirmed && !current.hardFailure && !isOpenCodeSecondaryLaneMember) { current.livenessKind = current.livenessKind === 'stale_metadata' || current.livenessKind === 'registered_only' || current.livenessKind == null ? 'confirmed_bootstrap' : current.livenessKind; current.pidSource = current.pidSource === 'persisted_metadata' || current.pidSource == null ? 'runtime_bootstrap' : current.pidSource; if (shouldClearRuntimeDiagnosticAfterBootstrapConfirmation(current.runtimeDiagnostic)) { current.runtimeDiagnostic = undefined; current.runtimeDiagnosticSeverity = undefined; } else if (!current.runtimeDiagnostic) { current.runtimeDiagnosticSeverity = undefined; } current.bootstrapStalled = undefined; } if ( isOpenCodeSecondaryLaneMember && shouldMarkPersistedOpenCodeBootstrapStalled(current, Date.now()) ) { const runtimeDiagnostic = getOpenCodeSecondaryBootstrapStallDiagnosticFromPersisted(current); current.launchState = 'runtime_pending_bootstrap'; current.agentToolAccepted = true; current.runtimeAlive = current.runtimeAlive === true && current.livenessKind === 'runtime_process'; current.bootstrapConfirmed = false; current.hardFailure = false; current.hardFailureReason = undefined; current.livenessKind = current.livenessKind ?? 'registered_only'; current.runtimeDiagnostic = runtimeDiagnostic; current.runtimeDiagnosticSeverity = 'warning'; current.bootstrapStalled = true; current.diagnostics = mergeRuntimeDiagnostics(current.diagnostics, [ runtimeDiagnostic, 'opencode_bootstrap_stalled', ]); } if ( current.agentToolAccepted === true && !current.bootstrapConfirmed && !current.runtimeAlive && !current.hardFailure && current.bootstrapStalled !== true && graceExpired ) { current.hardFailure = true; current.hardFailureReason = current.hardFailureReason ?? 'Teammate did not join within the launch grace window.'; } current.launchState = deriveMemberLaunchState(current); current.lastEvaluatedAt = now; nextMembers[expected] = { ...current, diagnostics: undefined, }; } const reconciled = createPersistedLaunchSnapshot({ teamName, expectedMembers: persistedMemberNames, leadSessionId: persistedWithCommittedEvidence.leadSessionId, launchPhase: persistedWithCommittedEvidence.launchPhase, members: nextMembers, updatedAt: now, }); if ( reconciled.teamLaunchState === 'clean_success' && !this.hasMixedLaunchMetadata(reconciled) ) { await this.clearPersistedLaunchState(teamName); return { snapshot: null, statuses: {} }; } const writtenSnapshot = await this.writeLaunchStateSnapshot(teamName, reconciled); return { snapshot: writtenSnapshot, statuses: snapshotToMemberSpawnStatuses(writtenSnapshot), }; } private async findBootstrapTranscriptFailureReason( teamName: string, memberName: string, sinceMs: number | null ): Promise { const outcome = await this.findBootstrapTranscriptOutcome(teamName, memberName, sinceMs); return outcome?.kind === 'failure' ? outcome.reason : null; } private async findBootstrapTranscriptOutcome( teamName: string, memberName: string, sinceMs: number | null ): Promise { const lookupCacheKey = this.buildBootstrapTranscriptOutcomeLookupCacheKey( teamName, memberName, sinceMs ); const cachedLookup = this.getPersistedBootstrapTranscriptOutcomeLookupCacheEntry( teamName, lookupCacheKey ); if (cachedLookup !== undefined) { return this.cloneBootstrapTranscriptOutcome(cachedLookup); } let summaries: Awaited>; try { summaries = await this.memberLogsFinder.findMemberLogs(teamName, memberName, sinceMs); } catch { summaries = []; } const outcomes: BootstrapTranscriptOutcome[] = []; for (const summary of summaries) { if (!summary.filePath) continue; const outcome = await this.readRecentBootstrapTranscriptOutcome( summary.filePath, sinceMs, memberName, teamName, { allowAnonymousFailure: true } ); if (outcome) { outcomes.push(outcome); } } outcomes.push( ...(await this.readBootstrapTranscriptOutcomesInProjectRoot(teamName, memberName, sinceMs)) ); const outcome = this.selectLatestBootstrapTranscriptOutcome(outcomes); this.setPersistedBootstrapTranscriptOutcomeLookupCacheEntry(teamName, lookupCacheKey, outcome); return outcome; } private cloneBootstrapTranscriptOutcome( outcome: BootstrapTranscriptOutcome | null ): BootstrapTranscriptOutcome | null { return outcome ? { ...outcome } : null; } private buildBootstrapTranscriptOutcomeLookupCacheKey( teamName: string, memberName: string, sinceMs: number | null ): string { return [teamName.trim().toLowerCase(), memberName.trim().toLowerCase(), sinceMs ?? ''].join( '\0' ); } private getPersistedBootstrapTranscriptOutcomeLookupCacheEntry( teamName: string, cacheKey: string ): BootstrapTranscriptOutcome | null | undefined { if (this.getTrackedRunId(teamName) || this.runtimeAdapterRunByTeam.has(teamName)) { return undefined; } const cached = this.bootstrapTranscriptOutcomeLookupCache.get(cacheKey); if (!cached) { return undefined; } if (cached.expiresAtMs <= Date.now()) { this.bootstrapTranscriptOutcomeLookupCache.delete(cacheKey); return undefined; } return cached.outcome; } private setPersistedBootstrapTranscriptOutcomeLookupCacheEntry( teamName: string, cacheKey: string, outcome: BootstrapTranscriptOutcome | null ): void { if (this.getTrackedRunId(teamName) || this.runtimeAdapterRunByTeam.has(teamName)) { return; } if ( !this.bootstrapTranscriptOutcomeLookupCache.has(cacheKey) && this.bootstrapTranscriptOutcomeLookupCache.size >= TeamProvisioningService.BOOTSTRAP_TRANSCRIPT_OUTCOME_CACHE_MAX_ENTRIES ) { const oldestKey = this.bootstrapTranscriptOutcomeLookupCache.keys().next().value; if (oldestKey) { this.bootstrapTranscriptOutcomeLookupCache.delete(oldestKey); } } this.bootstrapTranscriptOutcomeLookupCache.set(cacheKey, { expiresAtMs: Date.now() + TeamProvisioningService.PERSISTED_BOOTSTRAP_TRANSCRIPT_OUTCOME_LOOKUP_CACHE_TTL_MS, outcome: this.cloneBootstrapTranscriptOutcome(outcome), }); } private async readRecentBootstrapTranscriptOutcome( filePath: string, sinceMs: number | null, memberName: string, teamName: string, options: { allowAnonymousFailure?: boolean; contextMemberNames?: readonly string[]; } = {} ): Promise { const normalizedMemberName = memberName.trim().toLowerCase(); const normalizedTeamName = teamName.trim().toLowerCase(); const contextMemberNames = Array.from( new Set( [memberName, ...(options.contextMemberNames ?? [])] .map((name) => name.trim()) .filter(Boolean) ) ); const normalizedContextMemberNames = contextMemberNames.map((name) => name.trim().toLowerCase() ); const cacheKey = this.buildBootstrapTranscriptOutcomeCacheKey({ filePath, sinceMs, memberName: normalizedMemberName, teamName, allowAnonymousFailure: options.allowAnonymousFailure === true, contextMemberNames, }); try { // Stat without opening: on a cache hit we must NOT open the file. During a // tracked launch the per-member lookup cache is bypassed, so this scan runs // for every recent session file x every member x every poll; opening before // the cache check turned every one of those into a wasted open() syscall. const stat = await fs.promises.stat(filePath); if (!stat.isFile() || stat.size <= 0) { return null; } const cached = this.bootstrapTranscriptOutcomeCache.get(cacheKey); if (cached && cached.mtimeMs === stat.mtimeMs && cached.size === stat.size) { return cached.outcome; } // Parse the transcript tail once per (filePath, mtime, size) and share it // across members. getParsedBootstrapTranscriptTail opens the file ITSELF only // when its parse cache misses, so a shared-cache hit also avoids the open. const parsedLines = await this.getParsedBootstrapTranscriptTail(filePath, stat); const shouldCollectBootstrapContext = options.allowAnonymousFailure !== true; const bootstrapContextMembers = new Set(); const candidates: BootstrapTranscriptOutcomeCandidate[] = []; for (const parsedLine of parsedLines) { const { timestampMs, parsedAgentName, text, rawTimestamp, normalizedText } = parsedLine; if (sinceMs != null && (!Number.isFinite(timestampMs) || timestampMs < sinceMs)) { continue; } if ( parsedAgentName && !matchesObservedMemberNameForExpected(parsedAgentName, normalizedMemberName) ) { continue; } if (!text) { continue; } const lineNormalizedText = normalizedText ?? ''; if (shouldCollectBootstrapContext) { const isBootstrapContextLine = getCachedBootstrapContextCandidateForLine( parsedLine, lineNormalizedText, normalizedTeamName ); if (isBootstrapContextLine) { for (const contextMemberName of normalizedContextMemberNames) { if ( getCachedBootstrapContextMemberMatchForLine( parsedLine, lineNormalizedText, contextMemberName ) ) { bootstrapContextMembers.add(contextMemberName); } } } } candidates.push({ text, normalizedText: lineNormalizedText, observedAt: rawTimestamp && rawTimestamp.length > 0 ? rawTimestamp : new Date().toISOString(), parsedAgentName, parsedLine, }); } const hasUnambiguousMatchingBootstrapContext = shouldCollectBootstrapContext && bootstrapContextMembers.size === 1 && bootstrapContextMembers.has(normalizedMemberName); let outcome: BootstrapTranscriptOutcome | null = null; for (let index = candidates.length - 1; index >= 0; index -= 1) { const candidate = candidates[index]; if (!candidate) continue; // Lazy + memoized on the shared parsed line: computed at most once per line // across all members and re-scans, and only for lines this newest-first loop // actually reaches (lines past the first match are never extracted). const cachedLine = candidate.parsedLine; if (cachedLine.bootstrapFailureReason === undefined) { cachedLine.bootstrapFailureReason = extractBootstrapFailureReason(candidate.text); } const reason = cachedLine.bootstrapFailureReason; if (reason) { if ( !candidate.parsedAgentName && options.allowAnonymousFailure !== true && !hasUnambiguousMatchingBootstrapContext ) { continue; } outcome = { kind: 'failure', observedAt: candidate.observedAt, reason }; break; } const successSource = getCachedBootstrapSuccessSourceForLine( cachedLine, candidate.normalizedText, normalizedTeamName, normalizedMemberName ); if (successSource) { outcome = { kind: 'success', observedAt: candidate.observedAt, source: successSource }; break; } } this.setBootstrapTranscriptOutcomeCacheEntry(cacheKey, { mtimeMs: stat.mtimeMs, size: stat.size, outcome, }); return outcome; } catch { return null; } } private buildBootstrapTranscriptOutcomeCacheKey(input: { filePath: string; sinceMs: number | null; memberName: string; teamName: string; allowAnonymousFailure: boolean; contextMemberNames: readonly string[]; }): string { const normalizedContextMembers = Array.from( new Set(input.contextMemberNames.map((name) => name.trim().toLowerCase()).filter(Boolean)) ) .sort() .join('\0'); return [ input.filePath, input.sinceMs ?? '', input.memberName, input.teamName.trim().toLowerCase(), input.allowAnonymousFailure ? '1' : '0', normalizedContextMembers, ].join('\0'); } private async getParsedBootstrapTranscriptTail( filePath: string, stat: { mtimeMs: number; size: number } ): Promise { const cached = this.parsedBootstrapTranscriptTailCache.get(filePath); if (cached && cached.mtimeMs === stat.mtimeMs && cached.size === stat.size) { return cached.lines; } const lines: ParsedBootstrapTranscriptTailLine[] = []; const start = Math.max(0, stat.size - TeamProvisioningService.BOOTSTRAP_FAILURE_TAIL_BYTES); const length = stat.size - start; if (length > 0) { // Open lazily: only a genuine parse-cache miss (file changed since last // parse) reaches here, so we never open a file whose tail is already cached. const handle = await fs.promises.open(filePath, 'r'); let rawLines: string[]; try { const buffer = Buffer.alloc(length); await handle.read(buffer, 0, length, start); rawLines = buffer.toString('utf8').split('\n'); } finally { await handle.close().catch(() => undefined); } if (start > 0) { rawLines.shift(); } for (const rawLine of rawLines) { const line = rawLine?.trim(); if (!line) continue; let parsed: { timestamp?: unknown; agentName?: unknown } | null = null; try { parsed = JSON.parse(line) as { timestamp?: unknown; agentName?: unknown }; } catch { continue; } const rawTimestamp = typeof parsed.timestamp === 'string' && parsed.timestamp.trim().length > 0 ? parsed.timestamp.trim() : null; const timestampMs = typeof parsed.timestamp === 'string' ? Date.parse(parsed.timestamp) : Number.NaN; const parsedAgentName = typeof parsed.agentName === 'string' ? parsed.agentName.trim().toLowerCase() || null : null; const text = extractTranscriptMessageText(parsed); const normalizedText = text ? text.replace(/\s+/g, ' ').trim().toLowerCase() : null; lines.push({ rawTimestamp, timestampMs, text, normalizedText, parsedAgentName }); } } this.setParsedBootstrapTranscriptTailCacheEntry(filePath, { mtimeMs: stat.mtimeMs, size: stat.size, lines, }); return lines; } private setParsedBootstrapTranscriptTailCacheEntry( filePath: string, entry: ParsedBootstrapTranscriptTailCacheEntry ): void { if ( !this.parsedBootstrapTranscriptTailCache.has(filePath) && this.parsedBootstrapTranscriptTailCache.size >= TeamProvisioningService.BOOTSTRAP_TRANSCRIPT_OUTCOME_CACHE_MAX_ENTRIES ) { const oldestKey = this.parsedBootstrapTranscriptTailCache.keys().next().value; if (oldestKey) { this.parsedBootstrapTranscriptTailCache.delete(oldestKey); } } this.parsedBootstrapTranscriptTailCache.set(filePath, entry); } private setBootstrapTranscriptOutcomeCacheEntry( cacheKey: string, entry: BootstrapTranscriptOutcomeCacheEntry ): void { if ( !this.bootstrapTranscriptOutcomeCache.has(cacheKey) && this.bootstrapTranscriptOutcomeCache.size >= TeamProvisioningService.BOOTSTRAP_TRANSCRIPT_OUTCOME_CACHE_MAX_ENTRIES ) { const oldestKey = this.bootstrapTranscriptOutcomeCache.keys().next().value; if (oldestKey) { this.bootstrapTranscriptOutcomeCache.delete(oldestKey); } } this.bootstrapTranscriptOutcomeCache.set(cacheKey, entry); } private async readBootstrapTranscriptOutcomesInProjectRoot( teamName: string, memberName: string, sinceMs: number | null ): Promise { let config: TeamConfig | null; try { config = await this.readConfigSnapshot(teamName); } catch { return []; } const outcomes: BootstrapTranscriptOutcome[] = []; const projectDirs = await this.collectBootstrapTranscriptProjectDirs( teamName, memberName, config ); const contextMemberNames = [ memberName, ...((config?.members ?? []) .map((member) => member.name?.trim()) .filter((name): name is string => Boolean(name)) ?? []), ]; for (const projectDir of projectDirs) { let entries: fs.Dirent[]; try { entries = await fs.promises.readdir(projectDir, { withFileTypes: true }); } catch { continue; } const jsonlFiles = entries .filter((entry) => entry.isFile() && entry.name.endsWith('.jsonl')) .sort((left, right) => right.name.localeCompare(left.name)); for (const entry of jsonlFiles) { if (config?.leadSessionId && entry.name === `${config.leadSessionId}.jsonl`) { continue; } const candidatePath = path.join(projectDir, entry.name); // Project dirs can hold hundreds of old session transcripts. A file last // modified before the lookup window cannot contain a bootstrap line at/after // sinceMs (append-only: line timestamp <= write time <= mtime), so // readRecentBootstrapTranscriptOutcome would return null. Skip it with a // cheap stat instead of opening + tail-reading every file each poll. if (sinceMs != null) { try { const candidateStat = await fs.promises.stat(candidatePath); if ( candidateStat.mtimeMs < sinceMs - TeamProvisioningService.BOOTSTRAP_TRANSCRIPT_MTIME_SLACK_MS ) { continue; } } catch { continue; } } const outcome = await this.readRecentBootstrapTranscriptOutcome( candidatePath, sinceMs, memberName, teamName, { contextMemberNames } ); if (outcome) { outcomes.push(outcome); } } } return outcomes; } private async collectBootstrapTranscriptProjectDirs( teamName: string, memberName: string, config: TeamConfig | null ): Promise { const pathCandidates: string[] = []; const pathSeen = new Set(); const pushPath = (value: unknown): void => { if (typeof value !== 'string') { return; } let trimmed = value.trim(); while (trimmed.endsWith('/') || trimmed.endsWith('\\')) { trimmed = trimmed.slice(0, -1); } if (!trimmed || pathSeen.has(trimmed)) { return; } pathSeen.add(trimmed); pathCandidates.push(trimmed); }; pushPath(config?.projectPath); if (Array.isArray(config?.projectPathHistory)) { for (let index = config.projectPathHistory.length - 1; index >= 0; index -= 1) { pushPath(config.projectPathHistory[index]); } } const normalizedMemberName = memberName.trim().toLowerCase(); const pushMatchingMemberCwd = (member: { name?: unknown; cwd?: unknown }): void => { const candidateName = typeof member.name === 'string' ? member.name.trim().toLowerCase() : ''; if (candidateName && matchesTeamMemberIdentity(candidateName, normalizedMemberName)) { pushPath(member.cwd); } }; for (const member of config?.members ?? []) { pushMatchingMemberCwd(member); } const metaMembers = await this.membersMetaStore.getMembers(teamName).catch(() => []); for (const member of metaMembers) { pushMatchingMemberCwd(member); } const dirs: string[] = []; const dirSeen = new Set(); const pushDir = (dir: string): void => { if (!dir || dirSeen.has(dir)) { return; } dirSeen.add(dir); dirs.push(dir); }; for (const projectPath of pathCandidates) { const projectId = extractBaseDir(encodePath(projectPath)); pushDir(path.join(getProjectsBasePath(), projectId)); if (projectId.includes('_')) { pushDir(path.join(getProjectsBasePath(), projectId.replace(/_/g, '-'))); } } return dirs; } private selectLatestBootstrapTranscriptOutcome( outcomes: readonly BootstrapTranscriptOutcome[] ): BootstrapTranscriptOutcome | null { return ( [...outcomes].sort((left, right) => { const leftMs = Date.parse(left.observedAt); const rightMs = Date.parse(right.observedAt); const leftValid = Number.isFinite(leftMs); const rightValid = Number.isFinite(rightMs); if (leftValid && rightValid && leftMs !== rightMs) { return rightMs - leftMs; } if (leftValid !== rightValid) { return leftValid ? -1 : 1; } return 0; })[0] ?? null ); } private captureSendMessages(run: ProvisioningRun, content: Record[]): void { for (const part of content) { if (part.type !== 'tool_use' || typeof part.name !== 'string') continue; const isNativeSendMessage = part.name === 'SendMessage'; const input = part.input; if (!input || typeof input !== 'object') continue; const inp = input as Record; const isTeamMessageSendTool = isAgentTeamsToolUse({ rawName: part.name, canonicalName: 'message_send', toolInput: inp, currentTeamName: run.teamName, }); const isDirectCrossTeamSendTool = isAgentTeamsToolUse({ rawName: part.name, canonicalName: 'cross_team_send', toolInput: inp, currentTeamName: run.teamName, }); if (!isNativeSendMessage && !isTeamMessageSendTool && !isDirectCrossTeamSendTool) continue; if (isDirectCrossTeamSendTool) { const toTeam = typeof inp.toTeam === 'string' ? inp.toTeam.trim() : ''; const text = typeof inp.text === 'string' ? stripAgentBlocks(inp.text).trim() : ''; if (toTeam && text) { run.pendingDirectCrossTeamSendRefresh = true; } continue; } const rawRecipient = isNativeSendMessage ? typeof inp.recipient === 'string' ? inp.recipient : '' : typeof inp.to === 'string' ? inp.to : ''; const trimmedRecipient = rawRecipient.trim(); if (!trimmedRecipient) continue; const recipient = trimmedRecipient.toLowerCase() === 'user' ? 'user' : trimmedRecipient; const msgContent = isNativeSendMessage ? typeof inp.content === 'string' ? inp.content : '' : typeof inp.text === 'string' ? inp.text : ''; if (msgContent.trim().length === 0) continue; const summary = typeof inp.summary === 'string' ? inp.summary : ''; const leadName = run.request.members.find((m) => m.role?.toLowerCase().includes('lead'))?.name || 'team-lead'; const cleanContent = stripAgentBlocks(msgContent); if (cleanContent.trim().length === 0) continue; const strippedCrossTeamContent = stripCrossTeamPrefix(cleanContent).trim(); if (strippedCrossTeamContent.length === 0) continue; const localRecipientNames = new Set( (run.request.members ?? []) .map((member) => (typeof member.name === 'string' ? member.name.trim() : '')) .filter((name) => name.length > 0) ); localRecipientNames.add('user'); localRecipientNames.add('team-lead'); const mistakenToolHint = this.isCrossTeamToolRecipientName(recipient) ? this.resolveSingleActiveCrossTeamReplyHint(run) : null; const crossTeamRecipient = this.parseCrossTeamRecipient(run.teamName, recipient, localRecipientNames) ?? (mistakenToolHint ? { teamName: mistakenToolHint.toTeam, memberName: 'team-lead' } : null); if (crossTeamRecipient && this.crossTeamSender) { const inferredReplyMeta = mistakenToolHint?.toTeam === crossTeamRecipient.teamName ? { conversationId: mistakenToolHint.conversationId, replyToConversationId: mistakenToolHint.conversationId, } : this.resolveCrossTeamReplyMetadata(run.teamName, crossTeamRecipient.teamName); const crossTeamMeta = parseCrossTeamPrefix(cleanContent); const replyMeta = inferredReplyMeta; const timestamp = nowIso(); const messageId = `lead-sendmsg-${run.runId}-${Date.now()}`; const taskRefs = teamToolTaskRefs(run.teamName, inp.taskRefs); void this.crossTeamSender({ fromTeam: run.teamName, fromMember: leadName, toTeam: crossTeamRecipient.teamName, text: strippedCrossTeamContent, summary, ...(taskRefs ? { taskRefs } : {}), messageId, timestamp, conversationId: crossTeamMeta?.conversationId ?? replyMeta?.conversationId, replyToConversationId: replyMeta?.replyToConversationId ?? crossTeamMeta?.conversationId ?? replyMeta?.conversationId, }) .then((result) => { if (result.deduplicated) { return; } if (this.getTrackedRunId(run.teamName) !== run.runId) { logger.debug( `[${run.teamName}] Skipping stale cross-team send result for old run ${run.runId}` ); return; } const msg: InboxMessage = { from: leadName, to: recipient.startsWith('cross-team:') ? recipient : this.isCrossTeamToolRecipientName(recipient) ? `${crossTeamRecipient.teamName}.${crossTeamRecipient.memberName}` : `${crossTeamRecipient.teamName}.${crossTeamRecipient.memberName}`, text: strippedCrossTeamContent, timestamp, read: true, summary: (summary || strippedCrossTeamContent).length > 60 ? (summary || strippedCrossTeamContent).slice(0, 57) + '...' : summary || strippedCrossTeamContent, messageId: result.messageId, source: 'cross_team_sent', conversationId: crossTeamMeta?.conversationId ?? replyMeta?.conversationId, replyToConversationId: replyMeta?.replyToConversationId ?? crossTeamMeta?.conversationId ?? replyMeta?.conversationId, ...(taskRefs ? { taskRefs } : {}), }; this.pushLiveLeadProcessMessage(run.teamName, msg); this.teamChangeEmitter?.({ type: 'lead-message', teamName: run.teamName, runId: run.runId, detail: 'cross-team-send', }); }) .catch((error: unknown) => { logger.warn( `[${run.teamName}] qualified SendMessage→${recipient} cross-team fallback failed: ${ error instanceof Error ? error.message : String(error) }` ); }); continue; } if (this.isCrossTeamToolRecipientName(recipient)) { continue; } if (!isNativeSendMessage) { continue; } // Suppress SendMessage(to="user") during member_inbox_relay. // Context: when relaying inbox messages, the lead sometimes ignores the relay // instruction and responds to the user directly instead of forwarding to the // target teammate. This filter prevents that wrong response from appearing // in the UI and being persisted to sentMessages.json. // Note: teammate DM relay is currently disabled (see teams.ts handleSendMessage // and index.ts FileWatcher). This guard is kept as safety net in case relay // is re-enabled in the future. if (recipient === 'user' && run.silentUserDmForward?.mode === 'member_inbox_relay') { logger.debug( `[${run.teamName}] Suppressed SendMessage→user during member_inbox_relay to "${run.silentUserDmForward.target}"` ); continue; } const relayOfMessageId = recipient !== 'user' ? this.consumePendingInboxRelayCandidate( run, recipient, strippedCrossTeamContent, summary ) : undefined; const msg: InboxMessage = { from: leadName, to: recipient, text: strippedCrossTeamContent, timestamp: nowIso(), read: recipient !== 'user', summary: (summary || strippedCrossTeamContent).length > 60 ? (summary || strippedCrossTeamContent).slice(0, 57) + '...' : summary || strippedCrossTeamContent, messageId: `lead-sendmsg-${run.runId}-${Date.now()}`, ...(relayOfMessageId ? { relayOfMessageId } : {}), source: 'lead_process', }; this.pushLiveLeadProcessMessage(run.teamName, msg); if (recipient === 'user') { // User-directed messages go to sentMessages.json (canonical outbound store) this.persistSentMessage(run.teamName, msg); this.teamChangeEmitter?.({ type: 'inbox', teamName: run.teamName, detail: 'sentMessages.json', }); } else { // Non-user messages go to canonical recipient inbox for relay delivery this.persistInboxMessage(run.teamName, recipient, msg); this.teamChangeEmitter?.({ type: 'inbox', teamName: run.teamName, detail: `inboxes/${recipient}.json`, }); } logger.debug( `[${run.teamName}] Captured SendMessage→${recipient} from stdout: ${cleanContent.slice(0, 100)}` ); } } pushLiveLeadProcessMessage(teamName: string, message: InboxMessage): void { // Enrich with leadSessionId if missing — needed for session boundary separators if (!message.leadSessionId) { const runId = this.getTrackedRunId(teamName); if (runId) { const run = this.runs.get(runId); if (run?.detectedSessionId) { message.leadSessionId = run.detectedSessionId; } } } const MAX = 100; const list = this.liveLeadProcessMessages.get(teamName) ?? []; const id = typeof message.messageId === 'string' ? message.messageId.trim() : ''; if (id) { const existingIdx = list.findIndex((m) => (m.messageId ?? '').trim() === id); if (existingIdx >= 0) { list[existingIdx] = message; } else { list.push(message); } } else { list.push(message); } if (list.length > MAX) { list.splice(0, list.length - MAX); } this.liveLeadProcessMessages.set(teamName, list); } resolveCrossTeamReplyMetadata( teamName: string, toTeam: string ): { conversationId: string; replyToConversationId: string } | null { const runId = this.getAliveRunId(teamName); if (!runId) return null; const run = this.runs.get(runId); const hints = run?.activeCrossTeamReplyHints ?? []; if (hints.length === 0) return null; const matches = hints.filter((hint) => hint.toTeam === toTeam); if (matches.length !== 1) return null; return { conversationId: matches[0].conversationId, replyToConversationId: matches[0].conversationId, }; } /** * Create an InboxMessage from assistant text and push it into the live cache. * Used for both pre-ready (provisioning) and post-ready assistant text. * Emits a coalesced `lead-message` event for renderer refresh. */ private getStableLeadThoughtMessageId(msg: Record): string | null { const entryUuid = typeof msg.uuid === 'string' ? msg.uuid.trim() : ''; if (entryUuid) { return `lead-thought-${entryUuid}`; } const message = (msg.message ?? msg) as Record; const assistantMessageId = typeof message.id === 'string' ? message.id.trim() : ''; if (assistantMessageId) { return `lead-thought-msg-${assistantMessageId}`; } return null; } private isSyntheticLeadTextChunk(msg: Record): boolean { const message = (msg.message ?? msg) as Record; return message.model === '' && message.type === 'message'; } private joinLeadRelayCaptureText( capture: NonNullable ): string { return capture.textParts.join(capture.textJoinMode === 'stream' ? '' : '\n').trim(); } private resetLiveLeadTextBuffer(run: ProvisioningRun): void { run.liveLeadTextBuffer = null; } private appendProvisioningAssistantText( run: ProvisioningRun, msg: Record, text: string ): void { const normalized = text.trim(); if (normalized.length === 0) { return; } const stableMessageId = this.getStableLeadThoughtMessageId(msg); if (stableMessageId) { const existingIndex = run.provisioningOutputIndexByMessageId.get(stableMessageId); if (existingIndex != null) { run.provisioningOutputParts[existingIndex] = text; return; } } const lastIndex = run.provisioningOutputParts.length - 1; if (lastIndex >= 0 && run.provisioningOutputParts[lastIndex]?.trim() === normalized) { return; } const newIndex = run.provisioningOutputParts.push(text) - 1; if (stableMessageId) { run.provisioningOutputIndexByMessageId.set(stableMessageId, newIndex); } } private shiftProvisioningOutputIndexesAfterRemoval( run: ProvisioningRun, removedIndex: number ): void { for (const [messageId, index] of run.provisioningOutputIndexByMessageId.entries()) { if (index > removedIndex) { run.provisioningOutputIndexByMessageId.set(messageId, index - 1); } } } private pushLiveLeadTextMessage( run: ProvisioningRun, cleanText: string, stableMessageId?: string, messageTimestamp?: string, options?: { coalesceStreamChunk?: boolean } ): void { const leadName = this.getRunLeadName(run); const timestamp = typeof messageTimestamp === 'string' && messageTimestamp.trim().length > 0 && Number.isFinite(Date.parse(messageTimestamp)) ? messageTimestamp : nowIso(); const coalesceStreamChunk = options?.coalesceStreamChunk === true; let messageId = stableMessageId; let text = cleanText; let timestampForMessage = timestamp; let toolCalls: ToolCallMeta[] | undefined; let toolSummary: string | undefined; if (coalesceStreamChunk) { if (!run.liveLeadTextBuffer) { run.leadMsgSeq += 1; toolCalls = run.pendingToolCalls.length > 0 ? [...run.pendingToolCalls] : undefined; toolSummary = toolCalls ? formatToolSummaryFromCalls(toolCalls) : undefined; run.liveLeadTextBuffer = { messageId: `lead-turn-${run.runId}-${run.leadMsgSeq}`, text: cleanText, timestamp, toolCalls, toolSummary, }; run.pendingToolCalls = []; } else { run.liveLeadTextBuffer.text += cleanText; } messageId = run.liveLeadTextBuffer.messageId; text = stripAgentBlocks(run.liveLeadTextBuffer.text).trim(); timestampForMessage = run.liveLeadTextBuffer.timestamp; toolCalls = run.liveLeadTextBuffer.toolCalls; toolSummary = run.liveLeadTextBuffer.toolSummary; } else { this.resetLiveLeadTextBuffer(run); run.leadMsgSeq += 1; messageId = messageId || `lead-turn-${run.runId}-${run.leadMsgSeq}`; // Attach accumulated tool call details from preceding tool_use messages, then reset. toolCalls = run.pendingToolCalls.length > 0 ? [...run.pendingToolCalls] : undefined; toolSummary = toolCalls ? formatToolSummaryFromCalls(toolCalls) : undefined; run.pendingToolCalls = []; } const leadMsg: InboxMessage = { from: leadName, text, timestamp: timestampForMessage, read: true, summary: text.length > 60 ? text.slice(0, 57) + '...' : text, messageId, source: 'lead_process', toolSummary, toolCalls, }; this.pushLiveLeadProcessMessage(run.teamName, leadMsg); // Coalesced refresh: at most one event per LEAD_TEXT_EMIT_THROTTLE_MS per team. const now = Date.now(); if (now - run.lastLeadTextEmitMs >= TeamProvisioningService.LEAD_TEXT_EMIT_THROTTLE_MS) { run.lastLeadTextEmitMs = now; this.teamChangeEmitter?.({ type: 'lead-message', teamName: run.teamName, runId: run.runId, detail: 'lead-text', }); } } /** * Stop the running process for a team. No-op if team is not running. * Always uses SIGKILL via killTeamProcess() to prevent CLI cleanup. */ async stopTeam(teamName: string): Promise { this.invalidateRuntimeSnapshotCaches(teamName); this.taskActivityIntervalService.pauseActiveIntervalsForTeam(teamName); this.stopPersistentTeamMembers(teamName); const runId = this.getTrackedRunId(teamName); if (!runId) { if (this.hasSecondaryRuntimeRuns(teamName)) { await this.stopMixedSecondaryRuntimeLanes(teamName); } await this.cleanupAnthropicApiKeyHelperMaterialForStoppedTeam(teamName); return; } const run = this.runs.get(runId); if (!run) { const runtimeProgress = this.runtimeAdapterProgressByRunId.get(runId); if (runtimeProgress && this.isCancellableRuntimeAdapterProgress(runtimeProgress)) { await this.cancelRuntimeAdapterProvisioning(runId, runtimeProgress); await this.cleanupAnthropicApiKeyHelperMaterialForStoppedTeam(teamName); return; } const runtimeRun = this.runtimeAdapterRunByTeam.get(teamName); if (runtimeRun?.runId === runId && runtimeRun.providerId === 'opencode') { await this.withTeamLock(teamName, async () => { const currentRuntimeRun = this.runtimeAdapterRunByTeam.get(teamName); if (currentRuntimeRun?.runId === runId && currentRuntimeRun.providerId === 'opencode') { await this.stopOpenCodeRuntimeAdapterTeam(teamName, runId); } }); await this.cleanupAnthropicApiKeyHelperMaterialForStoppedTeam(teamName); return; } if (this.hasSecondaryRuntimeRuns(teamName)) { await this.stopMixedSecondaryRuntimeLanes(teamName); } this.provisioningRunByTeam.delete(teamName); this.deleteAliveRunId(teamName); await this.cleanupAnthropicApiKeyHelperMaterialForStoppedTeam(teamName); return; } if (run.processKilled || run.cancelRequested) { if (this.hasSecondaryRuntimeRuns(teamName)) { await this.stopMixedSecondaryRuntimeLanes(teamName); } await this.cleanupAnthropicApiKeyHelperMaterialForStoppedTeam(teamName); return; } run.processKilled = true; run.cancelRequested = true; killTeamProcess(run.child); const stopSecondaryRuntimeLanes = this.hasSecondaryRuntimeRuns(teamName) ? this.stopMixedSecondaryRuntimeLanes(teamName) : null; const progress = updateProgress(run, 'disconnected', 'Team stopped by user'); run.onProgress(progress); this.cleanupRun(run); logger.info(`[${teamName}] Process stopped (SIGKILL)`); await stopSecondaryRuntimeLanes; await this.cleanupAnthropicApiKeyHelperMaterialForStoppedTeam(teamName); } private getShutdownTrackedTeamNames(): string[] { const teamNames = new Set(); for (const teamName of this.provisioningRunByTeam.keys()) teamNames.add(teamName); for (const teamName of this.aliveRunByTeam.keys()) teamNames.add(teamName); for (const teamName of this.runtimeAdapterRunByTeam.keys()) teamNames.add(teamName); for (const teamName of this.secondaryRuntimeRunByTeam.keys()) teamNames.add(teamName); for (const teamName of this.teamOpLocks.keys()) teamNames.add(teamName); for (const progress of this.getPendingRuntimeAdapterLaunchesForShutdown()) { teamNames.add(progress.teamName); } return Array.from(teamNames); } private async stopTrackedTeamsForShutdown(label: string): Promise { const teamNames = this.getShutdownTrackedTeamNames(); if (teamNames.length === 0) { return teamNames; } logger.info(`${label}: stopping tracked team processes: ${teamNames.join(', ')}`); await Promise.all( teamNames.map((teamName) => this.stopTeam(teamName).catch((error) => { logger.warn( `[${teamName}] Failed to stop team during shutdown: ${ error instanceof Error ? error.message : String(error) }` ); }) ) ); return teamNames; } private async cancelPendingRuntimeAdapterLaunchesForShutdown(): Promise { const pendingRuntimeLaunches = this.getPendingRuntimeAdapterLaunchesForShutdown(); if (pendingRuntimeLaunches.length === 0) { return; } logger.info( `Cancelling pending OpenCode runtime adapter launches on shutdown: ${pendingRuntimeLaunches .map((progress) => progress.teamName) .join(', ')}` ); await Promise.all( pendingRuntimeLaunches.map((progress) => this.cancelRuntimeAdapterProvisioning(progress.runId, progress).catch((error) => { logger.warn( `[${progress.teamName}] Failed to cancel pending OpenCode runtime adapter launch on shutdown: ${ error instanceof Error ? error.message : String(error) }` ); }) ) ); } private async waitForInFlightTeamOperationsForShutdown(timeoutMs = 2_000): Promise { const locks = Array.from(this.teamOpLocks.values()); if (locks.length === 0) { return; } let timedOut = false; let timeout: ReturnType | null = null; await Promise.race([ Promise.allSettled(locks).then(() => undefined), new Promise((resolve) => { timeout = setTimeout(() => { timedOut = true; resolve(); }, timeoutMs); timeout.unref?.(); }), ]); if (timeout) { clearTimeout(timeout); } if (timedOut) { logger.warn( `Timed out after ${timeoutMs}ms waiting for in-flight team operations during shutdown` ); } } private killTransientProbeProcessesForShutdown(): void { for (const child of Array.from(this.transientProbeProcesses)) { try { killProcessTree(child); } catch (error) { logger.debug( `Failed to kill transient probe process during shutdown: ${ error instanceof Error ? error.message : String(error) }` ); } } } private async stopMixedSecondaryRuntimeLanes(teamName: string): Promise { const secondaryRuns = this.getSecondaryRuntimeRuns(teamName); if (secondaryRuns.length === 0) { return; } this.stoppingSecondaryRuntimeTeams.add(teamName); try { const adapter = this.getOpenCodeRuntimeAdapter(); const previousLaunchState = await this.launchStateStore.read(teamName); if (!adapter) { await Promise.all( secondaryRuns.map((secondaryRun) => clearOpenCodeRuntimeLaneStorage({ teamsBasePath: getTeamsBasePath(), teamName, laneId: secondaryRun.laneId, }).catch(() => undefined) ) ); this.clearSecondaryRuntimeRuns(teamName); return; } try { for (const secondaryRun of secondaryRuns) { await clearOpenCodeRuntimeLaneStorage({ teamsBasePath: getTeamsBasePath(), teamName, laneId: secondaryRun.laneId, }).catch(() => undefined); try { await adapter.stop({ runId: secondaryRun.runId, laneId: secondaryRun.laneId, teamName, cwd: secondaryRun.cwd ?? this.readPersistedTeamProjectPath(teamName) ?? undefined, providerId: 'opencode', reason: 'user_requested', previousLaunchState, force: true, }); } catch (error) { logger.warn( `[${teamName}] Failed to stop mixed OpenCode secondary lane ${secondaryRun.laneId}: ${ error instanceof Error ? error.message : String(error) }` ); } finally { await clearOpenCodeRuntimeLaneStorage({ teamsBasePath: getTeamsBasePath(), teamName, laneId: secondaryRun.laneId, }).catch(() => undefined); this.deleteSecondaryRuntimeRun(teamName, secondaryRun.laneId); } } } finally { this.clearSecondaryRuntimeRuns(teamName); } } finally { this.stoppingSecondaryRuntimeTeams.delete(teamName); } } private async stopOpenCodeRuntimeAdapterTeam(teamName: string, runId: string): Promise { const adapter = this.getOpenCodeRuntimeAdapter(); const previousLaunchState = await this.launchStateStore.read(teamName); if (!adapter) { await clearOpenCodeRuntimeLaneStorage({ teamsBasePath: getTeamsBasePath(), teamName, laneId: 'primary', }).catch(() => undefined); this.runtimeAdapterRunByTeam.delete(teamName); this.deleteAliveRunId(teamName); this.provisioningRunByTeam.delete(teamName); this.invalidateRuntimeSnapshotCaches(teamName); return; } const startedAt = nowIso(); const previousProgress = this.runtimeAdapterProgressByRunId.get(runId); const runtimeRun = this.runtimeAdapterRunByTeam.get(teamName); this.setRuntimeAdapterProgress({ runId, teamName, state: 'disconnected', message: 'Stopping OpenCode team through runtime adapter', startedAt: previousProgress?.startedAt ?? startedAt, updatedAt: startedAt, }); this.clearOpenCodeRuntimeToolApprovals(teamName, { runId, laneId: 'primary', emitDismiss: true, }); this.runtimeAdapterRunByTeam.delete(teamName); this.deleteAliveRunId(teamName); if (this.provisioningRunByTeam.get(teamName) === runId) { this.provisioningRunByTeam.delete(teamName); } this.invalidateRuntimeSnapshotCaches(teamName); try { await clearOpenCodeRuntimeLaneStorage({ teamsBasePath: getTeamsBasePath(), teamName, laneId: 'primary', }).catch(() => undefined); const result = await adapter.stop({ runId, laneId: 'primary', teamName, cwd: runtimeRun?.cwd ?? this.readPersistedTeamProjectPath(teamName) ?? undefined, providerId: 'opencode', reason: 'user_requested', previousLaunchState, force: true, }); await this.writeLaunchStateSnapshot( teamName, createPersistedLaunchSnapshot({ teamName, expectedMembers: previousLaunchState?.expectedMembers ?? [], leadSessionId: previousLaunchState?.leadSessionId, launchPhase: 'reconciled', members: previousLaunchState?.members ?? {}, }) ); this.setRuntimeAdapterProgress({ runId, teamName, state: result.stopped ? 'disconnected' : 'failed', message: result.stopped ? 'OpenCode team stopped' : 'OpenCode team stop failed', messageSeverity: result.stopped ? undefined : 'error', startedAt: previousProgress?.startedAt ?? startedAt, updatedAt: nowIso(), cliLogsTail: result.diagnostics.join('\n') || undefined, warnings: result.warnings.length > 0 ? result.warnings : undefined, }); } catch (error) { const message = error instanceof Error ? error.message : String(error); this.setRuntimeAdapterProgress({ runId, teamName, state: 'failed', message: 'OpenCode team stop failed', messageSeverity: 'error', startedAt: previousProgress?.startedAt ?? startedAt, updatedAt: nowIso(), error: message, cliLogsTail: message, }); } finally { await clearOpenCodeRuntimeLaneStorage({ teamsBasePath: getTeamsBasePath(), teamName, laneId: 'primary', }).catch(() => undefined); this.runtimeAdapterRunByTeam.delete(teamName); this.deleteAliveRunId(teamName); this.provisioningRunByTeam.delete(teamName); this.teamChangeEmitter?.({ type: 'process', teamName, runId, detail: 'stopped', }); } } private stopPersistentTeamMembers(teamName: string): void { const members = this.readPersistedRuntimeMembers(teamName); if (members.length > 0) { this.killPersistedPaneMembers(teamName, members); } this.killOrphanedTeamAgentProcesses(teamName); } private async cleanupAnthropicApiKeyHelperMaterialForStoppedTeam( teamName: string ): Promise { try { await cleanupAnthropicTeamApiKeyHelperForTeam({ teamName, baseClaudeDir: getClaudeBasePath(), }); } catch (error) { logger.warn( `[${teamName}] Failed to cleanup Anthropic team API-key helper material: ${ error instanceof Error ? error.message : String(error) }` ); } } private clonePersistedRuntimeMember( member: PersistedRuntimeMemberLike ): PersistedRuntimeMemberLike { return { ...member }; } private isPersistedRuntimeMemberLike(member: unknown): member is PersistedRuntimeMemberLike { return !!member && typeof member === 'object'; } private readPersistedTeamConfig(teamName: string): PersistedTeamConfigCacheEntry | null { const configPath = path.join(getTeamsBasePath(), teamName, 'config.json'); let stat: fs.Stats; try { stat = fs.statSync(configPath); } catch { this.persistedTeamConfigCache.delete(teamName); return null; } const cached = this.persistedTeamConfigCache.get(teamName); if ( cached && cached.path === configPath && cached.size === stat.size && cached.mtimeMs === stat.mtimeMs && cached.ctimeMs === stat.ctimeMs ) { return cached; } try { const raw = fs.readFileSync(configPath, 'utf8'); const parsed = JSON.parse(raw) as { projectPath?: unknown; members?: unknown }; const projectPath = typeof parsed.projectPath === 'string' ? parsed.projectPath.trim() : ''; const members = Array.isArray(parsed.members) ? parsed.members .filter((member): member is PersistedRuntimeMemberLike => this.isPersistedRuntimeMemberLike(member) ) .map((member) => this.clonePersistedRuntimeMember(member)) : []; const entry: PersistedTeamConfigCacheEntry = { path: configPath, size: stat.size, mtimeMs: stat.mtimeMs, ctimeMs: stat.ctimeMs, projectPath: projectPath || null, members, }; this.persistedTeamConfigCache.set(teamName, entry); return entry; } catch { this.persistedTeamConfigCache.delete(teamName); return null; } } private readPersistedTeamProjectPath(teamName: string): string | null { return this.readPersistedTeamConfig(teamName)?.projectPath ?? null; } private readPersistedRuntimeMembers(teamName: string): PersistedRuntimeMemberLike[] { return ( this.readPersistedTeamConfig(teamName)?.members.map((member) => this.clonePersistedRuntimeMember(member) ) ?? [] ); } private listPersistedTeamNames(): string[] { try { return fs .readdirSync(getTeamsBasePath(), { withFileTypes: true }) .filter((entry) => entry.isDirectory()) .map((entry) => entry.name.trim()) .filter((name) => name.length > 0); } catch { return []; } } private killPersistedPaneMembers(teamName: string, members: PersistedRuntimeMemberLike[]): void { for (const member of members) { const name = typeof member.name === 'string' ? member.name.trim() : ''; const paneId = typeof member.tmuxPaneId === 'string' ? member.tmuxPaneId.trim() : ''; const backendType = typeof member.backendType === 'string' ? member.backendType.trim().toLowerCase() : ''; if (!name || name === 'team-lead' || !paneId || backendType !== 'tmux') { continue; } try { killTmuxPaneForCurrentPlatformSync(paneId); logger.info(`[${teamName}] Killed teammate pane ${name} (${paneId}) during stop`); } catch (error) { logger.debug( `[${teamName}] Failed to kill teammate pane ${name} (${paneId}) during stop: ${ error instanceof Error ? error.message : String(error) }` ); } } } private killOrphanedTeamAgentProcesses(teamName: string): void { const currentRunPid = this.getTrackedRunId(teamName) ? this.runs.get(this.getTrackedRunId(teamName)!)?.child?.pid : undefined; const pids = new Set(); const rows: { pid: number; command: string }[] = []; if (process.platform === 'win32') { try { rows.push( ...listWindowsProcessTableSync().map((row) => ({ pid: row.pid, command: row.command })) ); } catch { return; } } else { let output = ''; try { output = execFileSync('ps', ['-ax', '-o', 'pid=,command='], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], }); } catch { return; } for (const line of output.split('\n')) { const trimmed = line.trim(); const match = /^(\d+)\s+(.*)$/.exec(trimmed); if (!match) continue; const pid = Number.parseInt(match[1], 10); if (!Number.isFinite(pid) || pid <= 0) continue; rows.push({ pid, command: match[2] ?? '' }); } } for (const row of rows) { if ( !commandArgEquals(row.command, '--team-name', teamName) || !row.command.includes('--agent-id') ) { continue; } if (currentRunPid && row.pid === currentRunPid) continue; pids.add(row.pid); } for (const pid of pids) { try { killProcessByPid(pid); logger.info(`[${teamName}] Killed orphaned teammate process pid=${pid} during stop`); } catch (error) { logger.debug( `[${teamName}] Failed to kill orphaned teammate process pid=${pid}: ${ error instanceof Error ? error.message : String(error) }` ); } } } /** * Stop all running team processes. Called during app shutdown. * Uses killTeamProcess() (SIGKILL) to guarantee instant death * without CLI cleanup that would delete team files. */ async stopAllTeams(): Promise { this.stopAllTeamsGeneration += 1; for (const teamName of this.getShutdownTrackedTeamNames()) { this.taskActivityIntervalService.pauseActiveIntervalsForTeam(teamName); } killTrackedCliProcesses('SIGKILL'); this.killTransientProbeProcessesForShutdown(); const initialTracked = await this.stopTrackedTeamsForShutdown('Shutdown'); await this.cancelPendingRuntimeAdapterLaunchesForShutdown(); // A create/launch may have been inside a per-team lock before it exposed a // run in provisioningRunByTeam. Wait briefly, then rescan to catch anything // that became visible while shutdown was already in progress. await this.waitForInFlightTeamOperationsForShutdown(); await this.cancelPendingRuntimeAdapterLaunchesForShutdown(); await this.stopTrackedTeamsForShutdown('Shutdown follow-up'); const persistedTeamNames = this.listPersistedTeamNames(); const tracked = new Set([...initialTracked, ...this.getShutdownTrackedTeamNames()]); const orphanOnly = persistedTeamNames.filter((teamName) => !tracked.has(teamName)); if (orphanOnly.length > 0) { logger.info(`Cleaning up persisted teammate runtimes on shutdown: ${orphanOnly.join(', ')}`); for (const teamName of orphanOnly) { this.taskActivityIntervalService.pauseActiveIntervalsForTeam(teamName); this.stopPersistentTeamMembers(teamName); await this.cleanupAnthropicApiKeyHelperMaterialForStoppedTeam(teamName); } } } /** * Process a parsed stream-json message from stdout. * Extracts assistant text for progress reporting and detects turn completion. */ private handleDeterministicBootstrapEvent( run: ProvisioningRun, msg: Record ): boolean { if (msg.type !== 'system' || msg.subtype !== 'team_bootstrap') { return false; } const acceptance = shouldAcceptDeterministicBootstrapEvent({ runId: run.runId, teamName: run.teamName, lastSeq: run.lastDeterministicBootstrapSeq, msg, }); if (!acceptance.accept) { return true; } run.lastDeterministicBootstrapSeq = acceptance.nextSeq; const event = typeof msg.event === 'string' ? msg.event : undefined; if (!event) { return true; } this.recordDeterministicBootstrapTracking(run, event, msg); if (event === 'started') { const progress = updateProgress(run, 'configuring', 'Starting deterministic team bootstrap'); run.onProgress(progress); return true; } if (event === 'phase_changed') { const phase = typeof msg.phase === 'string' ? msg.phase : ''; if (phase === 'loading_existing_state') { const progress = updateProgress(run, 'configuring', 'Loading existing team state'); run.onProgress(progress); } else if (phase === 'acquiring_bootstrap_lock') { const progress = updateProgress( run, 'configuring', 'Acquiring deterministic bootstrap lock' ); run.onProgress(progress); } else if (phase === 'creating_team') { const progress = updateProgress(run, 'assembling', 'Creating team config'); run.onProgress(progress); } else if (phase === 'spawning_members') { const progress = updateProgress(run, 'assembling', 'Spawning teammate runtimes'); run.onProgress(progress); } else if (phase === 'auditing_truth') { const progress = updateProgress( run, 'finalizing', 'Auditing registered teammates and bootstrap truth', { configReady: true } ); run.onProgress(progress); } return true; } if (event === 'team_created') { const reused = msg.reused_existing_team === true; const progress = updateProgress( run, 'assembling', reused ? 'Attached to existing team, starting teammates' : 'Team config created, starting teammates', { configReady: true } ); run.onProgress(progress); return true; } if (event === 'member_spawn_started') { const memberName = typeof msg.member_name === 'string' ? msg.member_name.trim() : ''; if (memberName) { this.setMemberSpawnStatus(run, memberName, 'spawning'); } return true; } if (event === 'member_spawn_result') { const memberName = typeof msg.member_name === 'string' ? msg.member_name.trim() : ''; const outcome = typeof msg.outcome === 'string' ? msg.outcome : ''; const reason = typeof msg.reason === 'string' ? msg.reason.trim() : undefined; if (!memberName) { return true; } if (outcome === 'failed') { this.setMemberSpawnStatus( run, memberName, 'error', reason || 'Deterministic bootstrap failed to spawn teammate.' ); return true; } if (outcome === 'already_running') { if (run.pendingMemberRestarts.has(memberName)) { run.pendingMemberRestarts.delete(memberName); this.setMemberSpawnStatus( run, memberName, 'error', buildRestartStillRunningReason(memberName) ); return true; } this.invalidateRuntimeSnapshotCaches(run.teamName); this.setMemberSpawnStatus(run, memberName, 'waiting'); this.appendMemberBootstrapDiagnostic( run, memberName, 'already_running requires strong runtime verification' ); void this.reevaluateMemberLaunchStatus(run, memberName); return true; } this.setMemberSpawnStatus(run, memberName, 'waiting'); return true; } if (event === 'completed') { const failedMembers = Array.isArray(msg.failed_members) ? msg.failed_members : []; for (const failed of failedMembers) { const memberName = typeof failed?.name === 'string' ? failed.name.trim() : ''; const reason = typeof failed?.reason === 'string' ? failed.reason.trim() : undefined; if (memberName) { this.setMemberSpawnStatus( run, memberName, 'error', reason || 'Deterministic bootstrap failed to spawn teammate.' ); } } if (!run.requiresFirstRealTurnSuccess && !run.provisioningComplete && !run.cancelRequested) { void this.handleProvisioningTurnComplete(run).catch((error: unknown) => { logger.error( `[${run.teamName}] deterministic bootstrap completion handler failed: ${ error instanceof Error ? error.message : String(error) }` ); }); } return true; } if (event === 'failed') { if (run.progress.state === 'failed' || run.cancelRequested) { return true; } const reason = typeof msg.reason === 'string' && msg.reason.trim().length > 0 ? msg.reason.trim() : 'Deterministic bootstrap failed.'; const classification = classifyDeterministicBootstrapFailure(reason); const progress = updateProgress(run, 'failed', classification.title, { error: classification.normalizedReason, cliLogsTail: extractCliLogsFromRun(run), }); run.onProgress(progress); const hasConfirmedBootstrapMember = Array.from(run.memberSpawnStatuses.values()).some( (member) => member.bootstrapConfirmed === true ); const shouldCleanupUnconfirmedLaunchRuntimes = run.isLaunch && !hasConfirmedBootstrapMember; this.markUnconfirmedBootstrapMembersFailed(run, classification.normalizedReason, { cleanupRequested: shouldCleanupUnconfirmedLaunchRuntimes, }); if (shouldCleanupUnconfirmedLaunchRuntimes) { this.stopPersistentTeamMembers(run.teamName); if (run.anthropicApiKeyHelper) { void cleanupAnthropicTeamApiKeyHelperMaterial({ directory: run.anthropicApiKeyHelper.directory, skipIfLiveProcessReferences: true, }).catch((error: unknown) => { logger.warn( `[${run.teamName}] Failed to cleanup failed-run Anthropic API-key helper material: ${ error instanceof Error ? error.message : String(error) }` ); }); } } run.processKilled = true; killTeamProcess(run.child); void this.persistLaunchStateSnapshot(run, 'finished').catch((error: unknown) => { logger.warn( `[${run.teamName}] Failed to persist failed bootstrap launch snapshot: ${ error instanceof Error ? error.message : String(error) }` ); }); this.cleanupRun(run); return true; } return true; } private recordDeterministicBootstrapTracking( run: ProvisioningRun, event: string, msg: Record ): void { run.deterministicBootstrapStartedAt ??= nowIso(); run.lastDeterministicBootstrapEvent = event; if (event === 'phase_changed') { const phase = typeof msg.phase === 'string' ? msg.phase.trim() : ''; if (phase) { run.lastDeterministicBootstrapPhase = phase; } } if (event === 'member_spawn_started') { run.deterministicBootstrapMemberSpawnSeen = true; } else if (event === 'member_spawn_result') { run.deterministicBootstrapMemberSpawnSeen = true; run.deterministicBootstrapMemberResultSeen = true; } } private handleStreamJsonMessage(run: ProvisioningRun, msg: Record): void { // stream-json output has various message types: // {"type":"assistant","content":[{"type":"text","text":"..."},...]} // {"type":"result","subtype":"success",...} // Capture session_id as early as possible so live messages emitted during this // handler already carry the session identity used by merge/dedup paths. if (!run.detectedSessionId) { const sid = typeof msg.session_id === 'string' ? msg.session_id : undefined; if (sid && sid.trim().length > 0) { run.detectedSessionId = sid.trim(); logger.info( `[${run.teamName}] Detected session ID from stream-json: ${run.detectedSessionId}` ); } } if (msg.type === 'user') { this.resetLiveLeadTextBuffer(run); // Check for permission_request in raw user message text BEFORE teammate-message parsing. // The permission_request may arrive as plain JSON without wrapper, // and handleNativeTeammateUserMessage only processes blocks. const rawUserText = this.extractStreamUserText(msg); const content = this.extractStreamContentBlocks(msg); if (rawUserText) { const perm = parsePermissionRequest(rawUserText); if (perm) { logger.warn( `[${run.teamName}] [PERM-TRACE] Intercepted permission_request from stdout user message: agent=${perm.agentId} tool=${perm.toolName} requestId=${perm.requestId}` ); this.handleTeammatePermissionRequest(run, perm, new Date().toISOString()); } else if (rawUserText.includes('permission_request')) { // Log near-miss: text contains "permission_request" but wasn't parsed logger.warn( `[${run.teamName}] [PERM-TRACE] stdout user message contains "permission_request" but parsePermissionRequest returned null. Text preview: ${rawUserText.slice(0, 300)}` ); } } for (const block of content) { if (block?.type !== 'tool_result' || typeof block.tool_use_id !== 'string') continue; this.finishRuntimeToolActivity( run, block.tool_use_id, block.content, block.is_error === true ); } this.handleNativeTeammateUserMessage(run, msg); return; } if (msg.type === 'assistant') { const content = this.extractStreamContentBlocks(msg); const hasCapturedVisibleSendMessage = this.hasCapturedVisibleSendMessage( content, run.teamName ); if (run.leadRelayCapture) { if (hasCapturedVisibleSendMessage) { run.leadRelayCapture.hasVisibleSendMessage = true; } if (this.hasCapturedUserVisibleSendMessage(content, run.teamName)) { run.leadRelayCapture.hasUserVisibleSendMessage = true; } } const textParts = content .filter((part) => part.type === 'text' && typeof part.text === 'string') .map((part) => part.text as string); if (textParts.length > 0) { const text = textParts.join('\n'); const messageTimestamp = typeof msg.timestamp === 'string' && msg.timestamp.trim().length > 0 && Number.isFinite(Date.parse(msg.timestamp)) ? msg.timestamp : undefined; // Auth failures sometimes show up as assistant text (e.g. "401", "Please run /login") // rather than stderr or a result.subtype=error. Detect early to avoid false "ready". this.handleAuthFailureInOutput(run, text, 'assistant'); if (this.hasApiError(text) && !this.isAuthFailureWarning(text, 'assistant')) { this.failProvisioningWithApiError(run, text); return; } logger.debug(`[${run.teamName}] assistant: ${text.slice(0, 200)}`); // During provisioning (before provisioningComplete), accumulate for live UI preview. // Emission is handled by the throttled emitLogsProgress() in the stdout data handler. if (!run.provisioningComplete) { this.appendProvisioningAssistantText(run, msg, text); } // Once relay capture is settled, later assistant chunks belong to the normal live // message flow. Keeping them in the capture branch would drop them on the floor // until relayLeadInboxMessages() finally clears run.leadRelayCapture. if (run.leadRelayCapture && !run.leadRelayCapture.settled) { const capture = run.leadRelayCapture; if (this.isSyntheticLeadTextChunk(msg)) { capture.textJoinMode = 'stream'; } else if (!capture.textJoinMode) { capture.textJoinMode = 'block'; } capture.textParts.push(text); if (capture.idleHandle) { clearTimeout(capture.idleHandle); } capture.idleHandle = setTimeout(() => { const combined = this.joinLeadRelayCaptureText(capture); capture.resolveOnce(combined); }, capture.idleMs); } else if (run.provisioningComplete) { // Push each assistant text block as a separate live message (per-message pattern). // When the same assistant message includes SendMessage, skip narration because // captureSendMessages() handles the visible outbound message separately. if ( !run.silentUserDmForward && !run.suppressPostCompactReminderOutput && !run.suppressGeminiPostLaunchHydrationOutput && !hasCapturedVisibleSendMessage ) { const isSyntheticChunk = this.isSyntheticLeadTextChunk(msg); const displayText = isSyntheticChunk ? text : stripAgentBlocks(text).trim(); if ( (displayText.length > 0 || (isSyntheticChunk && run.liveLeadTextBuffer)) && !isTeamInternalControlMessageText(displayText) ) { this.pushLiveLeadTextMessage( run, displayText, this.getStableLeadThoughtMessageId(msg) ?? undefined, messageTimestamp, { coalesceStreamChunk: isSyntheticChunk } ); } } } else { // Pre-ready: keep showing provisioning narration in the banner, but also mirror it // into the live cache so Messages/Activity can show the earliest assistant output. if (!run.silentUserDmForward && !hasCapturedVisibleSendMessage) { const isSyntheticChunk = this.isSyntheticLeadTextChunk(msg); const displayText = isSyntheticChunk ? text : stripAgentBlocks(text).trim(); if ( (displayText.length > 0 || (isSyntheticChunk && run.liveLeadTextBuffer)) && !isTeamInternalControlMessageText(displayText) ) { this.pushLiveLeadTextMessage( run, displayText, this.getStableLeadThoughtMessageId(msg) ?? undefined, messageTimestamp, { coalesceStreamChunk: isSyntheticChunk } ); } } } } // Accumulate tool_use details from tool-only messages (text + tool_use are separate in stream-json). // These details will be attached to the next text message as toolCalls/toolSummary. // Works in both pre-ready and post-ready phases so early live messages get tool metadata. for (const block of content) { if ( block?.type === 'tool_use' && typeof block.name === 'string' && block.name !== 'SendMessage' ) { const input = (block.input ?? {}) as Record; run.pendingToolCalls.push({ name: block.name, preview: extractToolPreview(block.name, input), toolUseId: typeof block.id === 'string' ? block.id : undefined, }); this.resetLiveLeadTextBuffer(run); this.startRuntimeToolActivity(run, this.getRunLeadName(run), block); } } // Track member spawn events from Task tool_use blocks with team_name. // When the lead calls Task(team_name=X, name=Y), it means member Y is being spawned. this.captureTeamSpawnEvents(run, content); // Capture SendMessage tool_use blocks from assistant output. // Works in both pre-ready and post-ready phases so outbound runtime messages // are visible in our team message artifacts even if Claude's own routing drifts. if (!run.silentUserDmForward || run.silentUserDmForward.mode === 'member_inbox_relay') { this.captureSendMessages(run, content); } // Extract context window usage from message.usage for real-time tracking. // SDKAssistantMessage wraps BetaMessage which contains usage stats. const messageObj = (msg.message ?? msg) as Record; if (messageObj && typeof messageObj === 'object') { const msgId = typeof messageObj.id === 'string' ? messageObj.id : null; const usage = messageObj.usage as Record | undefined; if (usage && typeof usage === 'object') { // Dedup: skip if same message.id (SDK bug: multi-block = same usage repeated) if (!msgId || run.leadContextUsage?.lastUsageMessageId !== msgId) { this.updateLeadContextUsageFromUsage( run, usage, typeof messageObj.model === 'string' ? messageObj.model : undefined ); if (run.leadContextUsage) { run.leadContextUsage.lastUsageMessageId = msgId; } this.emitLeadContextUsage(run); } } } } if (this.handleDeterministicBootstrapEvent(run, msg)) { return; } // Handle control_request — tool approval protocol (only when --dangerously-skip-permissions is NOT set) if (msg.type === 'control_request') { this.handleControlRequest(run, msg); return; } if (msg.type === 'result') { const subtype = typeof msg.subtype === 'string' ? msg.subtype : (() => { const result = msg.result; if (!result || typeof result !== 'object') return undefined; const inner = (result as Record).subtype; return typeof inner === 'string' ? inner : undefined; })(); if (subtype === 'success') { logger.info(`[${run.teamName}] stream-json result: success — turn complete, process alive`); if (!run.provisioningComplete) { run.firstRealTurnSucceeded = true; } // Extract contextWindow from modelUsage if available (SDKResultSuccess.modelUsage) const modelUsageObj = (msg.modelUsage ?? (msg.result as Record | undefined)?.modelUsage) as | Record> | undefined; if (modelUsageObj && typeof modelUsageObj === 'object') { for (const modelData of Object.values(modelUsageObj)) { if ( modelData && typeof modelData === 'object' && typeof modelData.contextWindow === 'number' && modelData.contextWindow > 0 ) { if (!run.leadContextUsage) { run.leadContextUsage = { promptInputTokens: null, outputTokens: null, contextUsedTokens: null, contextWindowTokens: modelData.contextWindow, promptInputSource: 'unavailable', lastUsageMessageId: null, lastEmittedAt: 0, }; } else { run.leadContextUsage.contextWindowTokens = modelData.contextWindow; run.leadContextUsage.lastEmittedAt = 0; // force re-emit } this.emitLeadContextUsage(run); break; } } } // Extract usage from result message itself (final turn usage) const resultUsage = (msg.usage ?? (msg.result as Record | undefined)?.usage) as | Record | undefined; if (resultUsage && typeof resultUsage === 'object') { this.updateLeadContextUsageFromUsage( run, resultUsage, typeof (msg.result as Record | undefined)?.model === 'string' ? ((msg.result as Record).model as string) : undefined ); if (run.leadContextUsage) { run.leadContextUsage.lastEmittedAt = 0; } this.emitLeadContextUsage(run); } if (run.provisioningComplete) { // If this was a post-compact reminder turn completing, clear in-flight and suppress flags. // Preserve pendingPostCompactReminder if re-armed by a compact_boundary during this turn. if (run.postCompactReminderInFlight) { const hadPendingRearm = run.pendingPostCompactReminder; run.postCompactReminderInFlight = false; run.suppressPostCompactReminderOutput = false; logger.info( `[${run.teamName}] post-compact reminder turn completed${ hadPendingRearm ? ' (follow-up reminder pending from re-compact)' : '' }` ); } if (run.geminiPostLaunchHydrationInFlight) { run.geminiPostLaunchHydrationInFlight = false; run.suppressGeminiPostLaunchHydrationOutput = false; logger.info(`[${run.teamName}] Gemini post-launch hydration turn completed`); } this.resetRuntimeToolActivity(run, this.getRunLeadName(run)); this.setLeadActivity(run, 'idle'); } if (run.pendingDirectCrossTeamSendRefresh) { run.pendingDirectCrossTeamSendRefresh = false; this.teamChangeEmitter?.({ type: 'inbox', teamName: run.teamName, detail: 'sentMessages.json', }); } if (run.leadRelayCapture) { const capture = run.leadRelayCapture; const combined = this.joinLeadRelayCaptureText(capture); capture.resolveOnce(combined); } this.resetLiveLeadTextBuffer(run); // Clear silent relay flag after any successful turn. run.activeCrossTeamReplyHints = []; run.pendingInboxRelayCandidates = []; run.silentUserDmForward = null; if (run.silentUserDmForwardClearHandle) { clearTimeout(run.silentUserDmForwardClearHandle); run.silentUserDmForwardClearHandle = null; } // Deferred post-compact context reinjection: inject durable rules on first idle after compact. // Placed AFTER leadRelayCapture/silentUserDmForward cleanup so a previously-deferred // reminder can proceed now that the blocking conditions are cleared. if ( run.provisioningComplete && run.pendingPostCompactReminder && !run.postCompactReminderInFlight ) { void this.injectPostCompactReminder(run); } if ( run.provisioningComplete && run.pendingGeminiPostLaunchHydration && !run.geminiPostLaunchHydrationInFlight ) { void this.injectGeminiPostLaunchHydration(run); } this.completeProvisioningFromSuccessfulResult(run); } else if (subtype === 'error') { const errorMsg = typeof msg.error === 'string' ? msg.error : JSON.stringify(msg.error ?? 'unknown'); logger.warn(`[${run.teamName}] stream-json result: error — ${errorMsg}`); if (run.leadRelayCapture) { run.leadRelayCapture.rejectOnce(errorMsg); } this.resetLiveLeadTextBuffer(run); // Clear silent relay flag after any errored turn. run.pendingDirectCrossTeamSendRefresh = false; run.activeCrossTeamReplyHints = []; run.pendingInboxRelayCandidates = []; run.silentUserDmForward = null; if (run.silentUserDmForwardClearHandle) { clearTimeout(run.silentUserDmForwardClearHandle); run.silentUserDmForwardClearHandle = null; } if (!run.provisioningComplete && !run.cancelRequested) { const progress = updateProgress( run, 'failed', 'CLI reported an error during provisioning', { error: errorMsg, cliLogsTail: extractCliLogsFromRun(run), } ); run.onProgress(progress); // Kill the process on provisioning error run.processKilled = true; killTeamProcess(run.child); this.cleanupRun(run); } else if (run.provisioningComplete) { // Post-provisioning error: process alive, waiting for input. // Always clear all post-compact reminder state on error — prevents a stale pending // reminder from firing on the next unrelated successful turn. if (run.pendingPostCompactReminder || run.postCompactReminderInFlight) { const wasInFlight = run.postCompactReminderInFlight; clearPostCompactReminderState(run); logger.warn( `[${run.teamName}] post-compact reminder ${wasInFlight ? 'turn errored' : 'pending dropped'} — clearing (strict policy)` ); } if (run.pendingGeminiPostLaunchHydration || run.geminiPostLaunchHydrationInFlight) { const wasInFlight = run.geminiPostLaunchHydrationInFlight; clearGeminiPostLaunchHydrationState(run); logger.warn( `[${run.teamName}] Gemini post-launch hydration ${ wasInFlight ? 'turn errored' : 'pending dropped' } — clearing (strict policy)` ); } this.resetRuntimeToolActivity(run, this.getRunLeadName(run)); this.setLeadActivity(run, 'idle'); } } } // Handle compact_boundary — context was compacted, next assistant message will carry fresh usage if (msg.type === 'system') { const sub = typeof msg.subtype === 'string' ? msg.subtype : undefined; if (sub === 'compact_boundary') { if (run.leadContextUsage) { run.leadContextUsage.lastUsageMessageId = null; } // Extract compact metadata for the system message const meta = msg.compact_metadata as Record | undefined; const trigger = typeof meta?.trigger === 'string' ? meta.trigger : 'auto'; const preTokens = typeof meta?.pre_tokens === 'number' ? meta.pre_tokens : null; const tokenInfo = preTokens ? ` (was ~${(preTokens / 1000).toFixed(0)}k tokens)` : ''; const compactMsg: InboxMessage = { from: 'system', text: `Context compacted${tokenInfo}, trigger: ${trigger}`, timestamp: nowIso(), read: true, summary: `Context compacted (${trigger})`, messageId: `compact-${run.runId}-${Date.now()}`, source: 'lead_process', }; this.pushLiveLeadProcessMessage(run.teamName, compactMsg); this.teamChangeEmitter?.({ type: 'inbox', teamName: run.teamName, detail: 'compact_boundary', }); logger.info( `[${run.teamName}] compact_boundary — context will refresh on next turn${tokenInfo}` ); // Schedule post-compact context reinjection on next idle. // If a reminder is already in-flight, re-arm pending so a follow-up fires after it completes. // This handles the case where the reminder prompt itself triggers another compaction. if (run.provisioningComplete && !run.pendingPostCompactReminder) { run.pendingPostCompactReminder = true; logger.info( `[${run.teamName}] post-compact reminder scheduled for next idle${ run.postCompactReminderInFlight ? ' (re-armed during in-flight reminder)' : '' }` ); } } // Show API retry attempts in Live output so the user knows what's happening if (sub === 'api_retry') { const attempt = typeof msg.attempt === 'number' ? msg.attempt : '?'; const maxRetries = typeof msg.max_retries === 'number' ? msg.max_retries : '?'; const errorStatus = typeof msg.error_status === 'number' ? msg.error_status : undefined; const errorLabel = typeof msg.error === 'string' ? msg.error.replace(/_/g, ' ') : undefined; const retryDelay = typeof msg.retry_delay_ms === 'number' ? msg.retry_delay_ms : undefined; const rawErrorMessage = typeof msg.error_message === 'string' && msg.error_message.trim().length > 0 ? msg.error_message.trim() : undefined; const errorMessage = rawErrorMessage ? this.normalizeApiRetryErrorMessage(rawErrorMessage) : undefined; const looksLikeQuotaRetry = errorLabel === 'rate limit' || this.isQuotaRetryMessage(errorMessage); if (looksLikeQuotaRetry && rawErrorMessage) { const observedAt = new Date(); const messageTimestamp = typeof msg.timestamp === 'string' && Number.isFinite(Date.parse(msg.timestamp)) ? new Date(msg.timestamp) : observedAt; peekAutoResumeService()?.handleRateLimitMessage( run.teamName, rawErrorMessage, observedAt, messageTimestamp ); } // Use a human label for known quota/rate-limit retries instead of a misleading 500 bucket. const statusLabel = looksLikeQuotaRetry ? 'rate limited' : errorLabel ? `${errorLabel}${errorStatus ? ` (${errorStatus})` : ''}` : `error ${errorStatus ?? 'unknown'}`; const delayLabel = retryDelay ? ` — next retry in ${Math.round(retryDelay / 1000)}s` : ''; const retryText = `API retry ${attempt}/${maxRetries}: ${statusLabel}${ errorMessage ? ` — ${errorMessage}` : '' }${delayLabel}`; if (!run.provisioningComplete) { const warningText = errorMessage ? `**API retry ${attempt}/${maxRetries}: ${statusLabel}**\n\n\`\`\`\n${this.toMarkdownCodeSafe( errorMessage )}\n\`\`\`\n\n${retryDelay ? `Next retry in ${Math.round(retryDelay / 1000)}s.` : 'Retrying...'}` : `**API retry ${attempt}/${maxRetries}: ${statusLabel}**\n\n${ retryDelay ? `Next retry in ${Math.round(retryDelay / 1000)}s.` : 'Retrying...' }`; if (run.apiRetryWarningIndex != null) { run.provisioningOutputParts[run.apiRetryWarningIndex] = warningText; } else { run.apiRetryWarningIndex = run.provisioningOutputParts.length; run.provisioningOutputParts.push(warningText); } run.lastRetryAt = Date.now(); appendProvisioningTrace( run, run.progress.state, retryText, errorMessage ? `error=${errorMessage}` : undefined ); run.progress = { ...run.progress, updatedAt: nowIso(), message: retryText, messageSeverity: 'error' as const, assistantOutput: buildProvisioningLiveOutput(run) ?? run.progress.assistantOutput, }; run.onProgress(run.progress); } } } // Catch-all: detect API errors in unrecognised message types. // Guards against future protocol additions that carry error payloads // (e.g. type: "error") which would otherwise be silently dropped. if (typeof msg.type === 'string' && !HANDLED_STREAM_JSON_TYPES.has(msg.type)) { const raw = JSON.stringify(msg); logger.warn( `[${run.teamName}] Unhandled stream-json type "${msg.type}": ${raw.slice(0, 300)}` ); if ( !run.provisioningComplete && this.hasApiError(raw) && !this.isAuthFailureWarning(raw, 'stdout') ) { this.emitApiErrorWarning(run, raw); } } } private completeProvisioningFromSuccessfulResult(run: ProvisioningRun): void { if (run.provisioningComplete || run.cancelRequested) { return; } void this.handleProvisioningTurnComplete(run).catch((err: unknown) => { logger.error( `[${run.teamName}] handleProvisioningTurnComplete threw unexpectedly: ${ err instanceof Error ? err.message : String(err) }` ); }); } /** * Injects a post-compact context reminder into the lead process via stdin. * Reinjects durable lead rules (constraints, communication protocol, board MCP ops) * plus a fresh task board snapshot so the lead recovers full operational context * after context compaction. * * Policy: strict drop-after-attempt — one compact cycle gives at most one reminder turn. * If the injection fails (stdin not writable, process killed), we do not retry. */ private async injectPostCompactReminder(run: ProvisioningRun): Promise { // Consume the pending flag immediately — strict one-shot policy. run.pendingPostCompactReminder = false; // Guard: process must be alive and writable. if (!run.child?.stdin?.writable || run.processKilled || run.cancelRequested) { logger.warn( `[${run.teamName}] post-compact reminder skipped — process not writable or killed` ); return; } // Guard: don't inject if another turn is actively processing (race with user send / inbox relay). if (run.leadActivityState !== 'idle') { logger.info( `[${run.teamName}] post-compact reminder deferred — lead is ${run.leadActivityState}, not idle` ); // Re-arm so it triggers on next idle. run.pendingPostCompactReminder = true; return; } // Guard: don't inject while a relay capture is in-flight. if (run.leadRelayCapture) { logger.info(`[${run.teamName}] post-compact reminder deferred — relay capture in-flight`); run.pendingPostCompactReminder = true; return; } // Guard: don't inject while a silent DM forward is in progress. if (run.silentUserDmForward) { logger.info( `[${run.teamName}] post-compact reminder deferred — silent DM forward in progress` ); run.pendingPostCompactReminder = true; return; } // Read current team config for up-to-date members (may have changed since launch). let currentMembers: TeamCreateRequest['members'] = run.request.members; let leadName = 'team-lead'; try { const config = await this.readConfigForObservation(run.teamName); if (config?.members) { const configLead = config.members.find((m) => isLeadMember(m)); leadName = configLead?.name?.trim() || 'team-lead'; // Convert config members (excluding lead) to TeamCreateRequest member format. const configTeammates = config.members .filter((m) => !isLeadMember(m) && m?.name) .map((m) => ({ name: m.name, role: m.role ?? undefined, })); // When config.members only has the lead (pre-created config without // TeamCreate), fall back to run.request.members for the teammate list. if (configTeammates.length > 0) { currentMembers = configTeammates; } } else { leadName = run.request.members.find((m) => m.role?.toLowerCase().includes('lead'))?.name || 'team-lead'; } } catch { // Fallback to launch-time members if config is unavailable. leadName = run.request.members.find((m) => m.role?.toLowerCase().includes('lead'))?.name || 'team-lead'; logger.warn( `[${run.teamName}] post-compact reminder: config unavailable, using launch-time members` ); } const isSolo = currentMembers.length === 0; // Build persistent lead context. const persistentContext = buildPersistentLeadContext({ teamName: run.teamName, leadName, isSolo, members: currentMembers, compact: true, }); // Best-effort: fetch fresh task board snapshot. let taskBoardBlock = ''; try { const taskReader = new TeamTaskReader(); const tasks = await taskReader.getTasks(run.teamName); taskBoardBlock = buildTaskBoardSnapshot(tasks); } catch { // If tasks can't be read, inject without the snapshot. logger.warn(`[${run.teamName}] post-compact reminder: task board snapshot unavailable`); } // Re-check guards after async work. if (!run.child?.stdin?.writable || run.processKilled || run.cancelRequested) { logger.warn( `[${run.teamName}] post-compact reminder aborted — process state changed during preparation` ); return; } if (run.leadActivityState !== 'idle') { logger.info( `[${run.teamName}] post-compact reminder deferred — lead activity changed to ${run.leadActivityState as string}` ); // Re-arm so it triggers on next idle. run.pendingPostCompactReminder = true; return; } const message = [ `Apply these standing rules and current team state before responding:`, ``, `You are "${leadName}", the team lead of team "${run.teamName}".`, `You are running in a non-interactive CLI session. Do not ask questions.`, `CRITICAL: Execute ALL steps directly yourself in sequence. Do NOT delegate any step to a sub-agent via the Agent tool. The ONLY valid use of the Agent tool is spawning individual teammates.`, ``, persistentContext, taskBoardBlock.trim() ? `\n${taskBoardBlock}` : '', ``, `Do NOT start new work or execute tasks in this turn. Reply with one concise user-facing team status line about board readiness and teammate availability. Only report board readiness and teammate availability.`, ] .filter(Boolean) .join('\n'); const payload = JSON.stringify({ type: 'user', message: { role: 'user', content: [{ type: 'text', text: message }], }, }); run.postCompactReminderInFlight = true; run.suppressPostCompactReminderOutput = true; this.setLeadActivity(run, 'active'); try { const stdin = run.child.stdin; await new Promise((resolve, reject) => { stdin.write(payload + '\n', (err) => { if (err) reject(err); else resolve(); }); }); logger.info(`[${run.teamName}] post-compact reminder injected`); } catch (error) { // Strict drop-after-attempt — do not re-arm. clearPostCompactReminderState(run); this.resetRuntimeToolActivity(run, this.getRunLeadName(run)); this.setLeadActivity(run, 'idle'); logger.warn( `[${run.teamName}] post-compact reminder injection failed: ${ error instanceof Error ? error.message : String(error) }` ); } } private async injectGeminiPostLaunchHydration(run: ProvisioningRun): Promise { run.pendingGeminiPostLaunchHydration = false; if ( run.geminiPostLaunchHydrationSent || !run.child?.stdin?.writable || run.processKilled || run.cancelRequested ) { logger.warn( `[${run.teamName}] Gemini post-launch hydration skipped — process not writable, killed, or already sent` ); return; } if (run.leadActivityState !== 'idle') { logger.info( `[${run.teamName}] Gemini post-launch hydration deferred — lead is ${run.leadActivityState}, not idle` ); run.pendingGeminiPostLaunchHydration = true; return; } if (run.leadRelayCapture) { logger.info( `[${run.teamName}] Gemini post-launch hydration deferred — relay capture in-flight` ); run.pendingGeminiPostLaunchHydration = true; return; } if (run.silentUserDmForward) { logger.info( `[${run.teamName}] Gemini post-launch hydration deferred — silent DM forward in progress` ); run.pendingGeminiPostLaunchHydration = true; return; } let currentMembers: TeamCreateRequest['members'] = run.effectiveMembers; let leadName = run.effectiveMembers.find((m) => m.role?.toLowerCase().includes('lead'))?.name || 'team-lead'; try { const config = await this.readConfigForObservation(run.teamName); if (config?.members) { const configLead = config.members.find((m) => isLeadMember(m)); leadName = configLead?.name?.trim() || leadName; const configTeammates = config.members .filter((m) => !isLeadMember(m) && m?.name) .map((m) => ({ name: m.name, role: m.role ?? undefined, })); if (configTeammates.length > 0) { const launchMembersByName = new Map( run.effectiveMembers.map((member) => [member.name, member] as const) ); currentMembers = configTeammates.map((member) => ({ ...launchMembersByName.get(member.name), ...member, })); } } } catch { logger.warn( `[${run.teamName}] Gemini post-launch hydration: config unavailable, using launch-time members` ); } let tasks: TeamTask[] = []; try { tasks = await new TeamTaskReader().getTasks(run.teamName); } catch { logger.warn( `[${run.teamName}] Gemini post-launch hydration: task board snapshot unavailable` ); } if ( run.geminiPostLaunchHydrationSent || !run.child?.stdin?.writable || run.processKilled || run.cancelRequested ) { logger.warn( `[${run.teamName}] Gemini post-launch hydration aborted — process state changed during preparation` ); return; } if (run.leadActivityState !== 'idle') { logger.info( `[${run.teamName}] Gemini post-launch hydration deferred — lead activity changed to ${run.leadActivityState as string}` ); run.pendingGeminiPostLaunchHydration = true; return; } const message = buildGeminiPostLaunchHydrationPrompt(run, leadName, currentMembers, tasks); const promptSize = getPromptSizeSummary(message); logger.info( `[${run.teamName}] Gemini post-launch hydration prepared (${promptSize.chars} chars / ${promptSize.lines} lines)` ); const payload = JSON.stringify({ type: 'user', message: { role: 'user', content: [{ type: 'text', text: message }], }, }); run.geminiPostLaunchHydrationInFlight = true; run.geminiPostLaunchHydrationSent = true; run.suppressGeminiPostLaunchHydrationOutput = true; this.setLeadActivity(run, 'active'); try { const stdin = run.child.stdin; await new Promise((resolve, reject) => { stdin.write(payload + '\n', (err) => { if (err) reject(err); else resolve(); }); }); logger.info(`[${run.teamName}] Gemini post-launch hydration injected`); } catch (error) { run.geminiPostLaunchHydrationInFlight = false; run.geminiPostLaunchHydrationSent = false; run.suppressGeminiPostLaunchHydrationOutput = false; this.resetRuntimeToolActivity(run, this.getRunLeadName(run)); this.setLeadActivity(run, 'idle'); logger.warn( `[${run.teamName}] Gemini post-launch hydration injection failed: ${ error instanceof Error ? error.message : String(error) }` ); } } /** * Handles a control_request message from CLI stream-json output. * `can_use_tool` → emits to renderer for manual approval. * All other subtypes (hook_callback, etc.) → auto-allowed to prevent deadlock. */ private handleControlRequest(run: ProvisioningRun, msg: Record): void { const requestId = typeof msg.request_id === 'string' ? msg.request_id : null; if (!requestId) { logger.warn(`[${run.teamName}] control_request missing request_id, ignoring`); return; } const request = msg.request as Record | undefined; const subtype = request?.subtype; // Non-`can_use_tool` subtypes (hook_callback, etc.) are auto-allowed to prevent // CLI deadlock — hooks are user-configured and should not block on manual approval. if (subtype !== 'can_use_tool') { logger.debug( `[${run.teamName}] control_request subtype=${String(subtype)}, auto-allowing to prevent deadlock` ); this.autoAllowControlRequest(run, requestId); return; } const toolName = typeof request?.tool_name === 'string' ? request.tool_name : 'Unknown'; const toolInput = (request?.input ?? {}) as Record; const providerId = toolInput.provider === 'codex' ? 'codex' : undefined; const approval: ToolApprovalRequest = { requestId, runId: run.runId, teamName: run.teamName, ...(providerId ? { providerId } : {}), source: 'lead', toolName, toolInput, receivedAt: new Date().toISOString(), teamColor: run.request.color, teamDisplayName: run.request.displayName, }; // Check auto-allow rules before prompting user const autoResult = shouldAutoAllow( this.getToolApprovalSettings(run.teamName), toolName, toolInput ); if (autoResult.autoAllow) { logger.info(`[${run.teamName}] Auto-allowing ${toolName} (${autoResult.reason})`); this.autoAllowControlRequest(run, requestId); this.emitToolApprovalEvent({ autoResolved: true, requestId, runId: run.runId, teamName: run.teamName, reason: 'auto_allow_category', } as ToolApprovalAutoResolved); return; } run.pendingApprovals.set(requestId, approval); this.emitToolApprovalEvent(approval); this.startApprovalTimeout(run, requestId); // Show OS notification when window is not focused this.maybeShowToolApprovalOsNotification(run, approval); } /** * Handles a teammate permission_request received via inbox message. * Converts it to a ToolApprovalRequest and feeds it into the existing approval flow. */ private handleTeammatePermissionRequest( run: ProvisioningRun, perm: ParsedPermissionRequest, messageTimestamp: string ): void { // Skip if already tracked (idempotency — multiple paths can trigger this: // early inbox scan, stdout parsing, native message blocks, relay Category 4) if (run.processedPermissionRequestIds.has(perm.requestId)) return; if (run.pendingApprovals.has(perm.requestId)) return; run.processedPermissionRequestIds.add(perm.requestId); logger.warn( `[${run.teamName}] [PERM-TRACE] handleTeammatePermissionRequest: agent=${perm.agentId} tool=${perm.toolName} requestId=${perm.requestId}` ); const approval: ToolApprovalRequest = { requestId: perm.requestId, runId: run.runId, teamName: run.teamName, source: perm.agentId, toolName: perm.toolName, toolInput: perm.input, receivedAt: messageTimestamp || new Date().toISOString(), teamColor: run.request.color, teamDisplayName: run.request.displayName, permissionSuggestions: perm.permissionSuggestions.length > 0 ? perm.permissionSuggestions : undefined, }; const autoResult = shouldAutoAllow( this.getToolApprovalSettings(run.teamName), perm.toolName, perm.input ); if (autoResult.autoAllow) { logger.info( `[${run.teamName}] Auto-allowing teammate ${perm.agentId} ${perm.toolName} (${autoResult.reason})` ); void this.respondToTeammatePermission( run, perm.agentId, perm.requestId, true, undefined, perm.permissionSuggestions, perm.toolName, perm.input ); this.emitToolApprovalEvent({ autoResolved: true, requestId: perm.requestId, runId: run.runId, teamName: run.teamName, reason: 'auto_allow_category', } as ToolApprovalAutoResolved); return; } run.pendingApprovals.set(perm.requestId, approval); this.emitToolApprovalEvent(approval); this.startApprovalTimeout(run, perm.requestId); this.maybeShowToolApprovalOsNotification(run, approval); } private syncOpenCodeRuntimeToolApprovals(input: { teamName: string; runId: string; laneId: string; cwd: string; members: Record; expectedMembers: TeamRuntimeMemberSpec[]; memberNames?: readonly string[]; teamColor?: string; teamDisplayName?: string; }): void { const entries = openCodeRuntimeApprovalProvider.collectPendingApprovals(input); this.runtimeToolApprovalCoordinator.sync( { teamName: input.teamName, runId: input.runId, laneId: input.laneId, memberNames: input.memberNames, providerId: 'opencode', }, entries ); } /** * Shows a native OS notification for a pending tool approval when the app * is not in focus. On macOS, adds Allow/Deny action buttons that respond * directly from the notification without switching to the app. */ private maybeShowToolApprovalOsNotification( run: ProvisioningRun | undefined, approval: ToolApprovalRequest ): void { const win = this.mainWindowRef; if (win && !win.isDestroyed() && win.isFocused()) return; const config = ConfigManager.getInstance().getConfig(); if (!config.notifications.enabled || !config.notifications.notifyOnToolApproval) return; // Respect snooze — consistent with other notification types const snoozedUntil = config.notifications.snoozedUntil; if (snoozedUntil && Date.now() < snoozedUntil) return; // eslint-disable-next-line @typescript-eslint/no-require-imports const { Notification: ElectronNotification } = require('electron') as Partial< typeof import('electron') >; if (!ElectronNotification?.isSupported?.()) return; const isMac = process.platform === 'darwin'; const isLinux = process.platform === 'linux'; const iconPath = isMac ? undefined : getAppIconPath(); const teamLabel = approval.teamDisplayName ?? run?.request.displayName ?? approval.teamName; const body = this.formatToolApprovalBody(approval.toolName, approval.toolInput); // Actions (Allow/Deny buttons) supported on macOS and Windows. // Linux libnotify doesn't fire the 'action' event — users get click-to-focus. const supportsActions = !isLinux; const notification = new ElectronNotification({ title: `Tool Approval — ${teamLabel}`, body, sound: config.notifications.soundEnabled ? 'default' : undefined, ...(iconPath ? { icon: iconPath } : {}), ...(supportsActions ? { actions: [ { type: 'button' as const, text: 'Allow' }, { type: 'button' as const, text: 'Deny' }, ], } : {}), }); // Track by requestId so we can close it when approval is resolved via UI this.activeApprovalNotifications.set(approval.requestId, notification); const cleanup = (): void => { this.activeApprovalNotifications.delete(approval.requestId); }; notification.on('click', () => { cleanup(); // Use current mainWindowRef (not captured `win`) in case window was recreated const currentWin = this.mainWindowRef; if (currentWin && !currentWin.isDestroyed()) { currentWin.show(); currentWin.focus(); } }); notification.on('close', cleanup); // Action buttons: Allow (index 0) / Deny (index 1) // 'action' event fires on macOS and Windows (not Linux) if (supportsActions) { notification.on('action', (_event, index) => { cleanup(); const allow = index === 0; logger.info( `[${approval.teamName}] Tool approval ${allow ? 'allowed' : 'denied'} via OS notification` ); void this.respondToToolApproval( approval.teamName, approval.runId, approval.requestId, allow, allow ? undefined : 'Denied via notification' ).catch((err) => { logger.error( `[${approval.teamName}] Failed to respond via notification: ${err instanceof Error ? err.message : String(err)}` ); }); }); } notification.show(); } /** Dismiss the OS notification for a resolved/dismissed approval. */ dismissApprovalNotification(requestId: string): void { const notification = this.activeApprovalNotifications.get(requestId); if (notification) { notification.close(); this.activeApprovalNotifications.delete(requestId); } } private clearOpenCodeRuntimeToolApprovals( teamName: string, options: { runId?: string; laneId?: string; emitDismiss?: boolean } = {} ): void { this.runtimeToolApprovalCoordinator.clear(teamName, { ...options, providerId: 'opencode', }); } private formatToolApprovalBody(toolName: string, toolInput: Record): string { switch (toolName) { case 'AskUserQuestion': return this.formatAskUserQuestionApprovalBody(toolInput); case 'Bash': return `Bash: ${typeof toolInput.command === 'string' ? toolInput.command.slice(0, 150) : 'command'}`; case 'Write': case 'Edit': case 'Read': case 'NotebookEdit': return `${toolName}: ${typeof toolInput.file_path === 'string' ? toolInput.file_path : 'file'}`; default: return `${toolName}: ${JSON.stringify(toolInput).slice(0, 150)}`; } } private formatAskUserQuestionApprovalBody(toolInput: Record): string { const rawQuestions = Array.isArray(toolInput.questions) ? toolInput.questions : []; const questions = rawQuestions .map((item) => { if (!item || typeof item !== 'object') return null; const question = 'question' in item && typeof item.question === 'string' ? item.question.trim() : null; return question && question.length > 0 ? question.replace(/\s+/g, ' ') : null; }) .filter((question): question is string => Boolean(question)); if (questions.length === 0) { return 'Question: User input is required'; } const firstQuestion = questions[0]; const truncatedQuestion = firstQuestion.length > 140 ? `${firstQuestion.slice(0, 137)}...` : firstQuestion; return questions.length === 1 ? `Question: ${truncatedQuestion}` : `Questions (${questions.length}): ${truncatedQuestion}`; } /** * Immediately sends an "allow" control_response for a non-tool control_request. * Prevents CLI deadlock for hook_callback and other non-`can_use_tool` subtypes. */ private autoAllowControlRequest(run: ProvisioningRun, requestId: string): void { if (!run.child?.stdin?.writable) { logger.warn(`[${run.teamName}] Cannot auto-allow control_request: stdin not writable`); return; } const response = { type: 'control_response', response: { subtype: 'success', request_id: requestId, response: { behavior: 'allow', updatedInput: {} }, }, }; run.child.stdin.write(JSON.stringify(response) + '\n', (err) => { if (err) { logger.error( `[${run.teamName}] Failed to auto-allow control_request ${requestId}: ${err.message}` ); } }); } private tryClaimResponse(requestId: string): boolean { if (this.inFlightResponses.has(requestId)) return false; this.inFlightResponses.add(requestId); return true; } private startApprovalTimeout(run: ProvisioningRun, requestId: string): void { const { timeoutAction, timeoutSeconds } = this.getToolApprovalSettings(run.teamName); if (timeoutAction === 'wait') return; const timeoutMs = timeoutSeconds * 1000; const timer = setTimeout(() => { this.pendingTimeouts.delete(requestId); if (!run.pendingApprovals.has(requestId)) return; if (!this.tryClaimResponse(requestId)) return; // Read CURRENT settings (not captured closure) in case user changed action const currentAction = this.getToolApprovalSettings(run.teamName).timeoutAction; if (currentAction === 'wait') { // Settings changed to 'wait' but timer fired before reEvaluatePendingApprovals cleared it this.inFlightResponses.delete(requestId); return; } const allow = currentAction === 'allow'; logger.info(`[${run.teamName}] Timeout ${allow ? 'allowing' : 'denying'} ${requestId}`); const approval = run.pendingApprovals.get(requestId); if (approval && approval.source !== 'lead') { // Teammate request — apply permission_suggestions to project settings. this.respondToTeammatePermission( run, approval.source, requestId, allow, allow ? undefined : 'Timed out - auto-denied by settings', approval.permissionSuggestions, approval.toolName, approval.toolInput ).finally(() => { run.pendingApprovals.delete(requestId); this.inFlightResponses.delete(requestId); this.dismissApprovalNotification(requestId); this.emitToolApprovalEvent({ autoResolved: true, requestId, runId: run.runId, teamName: run.teamName, reason: allow ? 'timeout_allow' : 'timeout_deny', } as ToolApprovalAutoResolved); }); return; } if (allow) { this.autoAllowControlRequest(run, requestId); } else { this.autoDenyControlRequest(run, requestId); } run.pendingApprovals.delete(requestId); this.inFlightResponses.delete(requestId); this.dismissApprovalNotification(requestId); this.emitToolApprovalEvent({ autoResolved: true, requestId, runId: run.runId, teamName: run.teamName, reason: allow ? 'timeout_allow' : 'timeout_deny', } as ToolApprovalAutoResolved); }, timeoutMs); this.pendingTimeouts.set(requestId, timer); } private clearApprovalTimeout(requestId: string): void { const timer = this.pendingTimeouts.get(requestId); if (timer) { clearTimeout(timer); this.pendingTimeouts.delete(requestId); } } private autoDenyControlRequest(run: ProvisioningRun, requestId: string): void { if (!run.child?.stdin?.writable) { logger.warn(`[${run.teamName}] Cannot auto-deny control_request: stdin not writable`); return; } const response = { type: 'control_response', response: { subtype: 'success', request_id: requestId, response: { behavior: 'deny', message: 'Timed out — auto-denied by settings' }, }, }; run.child.stdin.write(JSON.stringify(response) + '\n', (err) => { if (err) { logger.error( `[${run.teamName}] Failed to auto-deny control_request ${requestId}: ${err.message}` ); } }); } private reEvaluatePendingApprovals(): void { for (const [, run] of this.runs) { const settings = this.getToolApprovalSettings(run.teamName); const toRemove: string[] = []; for (const [requestId, approval] of run.pendingApprovals) { const result = shouldAutoAllow(settings, approval.toolName, approval.toolInput); if (result.autoAllow) { this.clearApprovalTimeout(requestId); if (!this.tryClaimResponse(requestId)) continue; if (approval.source !== 'lead') { void this.respondToTeammatePermission( run, approval.source, requestId, true, undefined, approval.permissionSuggestions, approval.toolName, approval.toolInput ); } else { this.autoAllowControlRequest(run, requestId); } this.dismissApprovalNotification(requestId); toRemove.push(requestId); this.emitToolApprovalEvent({ autoResolved: true, requestId, runId: run.runId, teamName: run.teamName, reason: 'auto_allow_category', } as ToolApprovalAutoResolved); } else if (settings.timeoutAction !== 'wait' && !this.pendingTimeouts.has(requestId)) { // Settings changed from 'wait' to allow/deny — start timer for already pending items this.startApprovalTimeout(run, requestId); } else if (settings.timeoutAction === 'wait' && this.pendingTimeouts.has(requestId)) { // Settings changed TO 'wait' — clear existing timers this.clearApprovalTimeout(requestId); } } for (const requestId of toRemove) { run.pendingApprovals.delete(requestId); this.inFlightResponses.delete(requestId); } } this.runtimeToolApprovalCoordinator.reEvaluate(); } private async answerRuntimeToolApproval( entry: RuntimeToolApprovalEntry, allow: boolean, _message?: string ): Promise { if (entry.providerId !== 'opencode') { throw new Error(`Runtime approval provider is not supported: ${entry.providerId}`); } const adapter = this.getOpenCodeRuntimeAdapter(); if (!adapter?.answerRuntimePermission) { throw new Error('OpenCode runtime permission answer bridge is not available'); } const previousLaunchState = await this.launchStateStore.read(entry.approval.teamName); const result = await adapter.answerRuntimePermission({ runId: entry.approval.runId, laneId: entry.laneId, teamName: entry.approval.teamName, cwd: entry.cwd ?? '', providerId: 'opencode', memberName: entry.memberName, requestId: entry.providerRequestId, decision: allow ? 'allow' : 'reject', expectedMembers: entry.expectedMembers ?? [], previousLaunchState, }); if (entry.laneId === 'primary') { const launchInput: TeamRuntimeLaunchInput = { runId: entry.approval.runId, laneId: entry.laneId, teamName: entry.approval.teamName, cwd: entry.cwd ?? '', providerId: 'opencode', skipPermissions: false, expectedMembers: entry.expectedMembers ?? [], previousLaunchState, }; const { result: committed } = await this.persistOpenCodeRuntimeAdapterLaunchResult( result, launchInput ); if (committed.teamLaunchState === 'partial_failure') { this.runtimeAdapterRunByTeam.delete(entry.approval.teamName); } else { this.runtimeAdapterRunByTeam.set(entry.approval.teamName, { runId: entry.approval.runId, providerId: 'opencode', cwd: entry.cwd, members: committed.members, }); this.setAliveRunId(entry.approval.teamName, entry.approval.runId); } this.syncOpenCodeRuntimeToolApprovals({ teamName: entry.approval.teamName, runId: entry.approval.runId, laneId: entry.laneId, cwd: entry.cwd ?? '', members: committed.members, expectedMembers: entry.expectedMembers ?? [], teamDisplayName: entry.approval.teamDisplayName, teamColor: entry.approval.teamColor, }); } else { await this.applyOpenCodeSecondaryPermissionAnswerResult(entry, result); } this.teamChangeEmitter?.({ type: 'process', teamName: entry.approval.teamName, runId: entry.approval.runId, detail: allow ? 'permission-allowed' : 'permission-denied', }); } /** * Respond to a pending tool approval — sends control_response to CLI stdin. * Validates runId match and requestId existence before writing. */ async respondToToolApproval( teamName: string, runId: string, requestId: string, allow: boolean, message?: string ): Promise { const handledByRuntime = await this.runtimeToolApprovalCoordinator.respond( teamName, runId, requestId, allow, message ); if (handledByRuntime) { return; } // Look in both provisioning and alive runs — control_requests arrive during provisioning too const currentRunId = this.getTrackedRunId(teamName); if (!currentRunId) throw new Error(`No active process for team "${teamName}"`); const run = this.runs.get(currentRunId); if (!run) throw new Error(`Run not found for team "${teamName}"`); if (run.runId !== runId) { throw new Error(`Stale approval: runId mismatch (expected ${run.runId}, got ${runId})`); } // Clear timeout and claim response FIRST (before pendingApprovals check) // to handle the race where timeout already responded and deleted the approval this.clearApprovalTimeout(requestId); if (!this.tryClaimResponse(requestId)) { // Another response is already being written; leave the pending approval tracked // until that write succeeds or fails. return; } if (!run.pendingApprovals.has(requestId)) { // Approval was removed (e.g. by reEvaluatePendingApprovals) — clean up claim and exit this.inFlightResponses.delete(requestId); return; } const approval = run.pendingApprovals.get(requestId)!; // Teammate permission requests: apply permission_suggestions to project settings if (approval.source !== 'lead') { try { await this.respondToTeammatePermission( run, approval.source, requestId, allow, message, approval.permissionSuggestions, approval.toolName, approval.toolInput ); this.inFlightResponses.delete(requestId); run.pendingApprovals.delete(requestId); this.dismissApprovalNotification(requestId); } catch (error) { this.inFlightResponses.delete(requestId); if (run.pendingApprovals.has(requestId)) { this.startApprovalTimeout(run, requestId); } throw error; } return; } if (!run.child?.stdin?.writable) { this.inFlightResponses.delete(requestId); this.startApprovalTimeout(run, requestId); throw new Error(`Team "${teamName}" process stdin is not writable`); } // IMPORTANT: request_id is NESTED inside response, NOT top-level // (asymmetry with control_request — confirmed by Python SDK, Elixir SDK and issue #29991) const allowResponse: Record = { behavior: 'allow', updatedInput: {} }; // For AskUserQuestion: pass user's answers via updatedInput so the CLI // can deliver them without re-prompting. Format follows --permission-prompt-tool spec. if (allow && message) { const pending = run.pendingApprovals.get(requestId); if (pending?.toolName === 'AskUserQuestion') { try { const answers = JSON.parse(message) as Record; allowResponse.updatedInput = { ...pending.toolInput, answers }; } catch { // If message isn't JSON, use as-is for the first question const questions = (pending.toolInput.questions as { question?: string }[]) ?? []; const answers: Record = {}; if (questions[0]?.question) answers[questions[0].question] = message; allowResponse.updatedInput = { ...pending.toolInput, answers }; } } } const response = allow ? { type: 'control_response', response: { subtype: 'success', request_id: requestId, response: allowResponse, }, } : { type: 'control_response', response: { subtype: 'success', request_id: requestId, response: { behavior: 'deny', message: message ?? 'User denied' }, }, }; const stdin = run.child.stdin; const responseJson = JSON.stringify(response) + '\n'; logger.info( `[${teamName}] Writing control_response for ${requestId}: ${allow ? 'allow' : 'deny'}` ); try { await new Promise((resolve, reject) => { // Safety timeout — if stdin.write callback is never called (e.g. process died // between the writable check and the write), reject instead of hanging forever. const writeTimeout = setTimeout(() => { reject(new Error(`Timeout writing control_response to stdin (process may have exited)`)); }, 5000); stdin.write(responseJson, (err) => { clearTimeout(writeTimeout); if (err) { logger.error(`[${teamName}] Failed to write control_response: ${err.message}`); reject(err); } else { logger.info(`[${teamName}] control_response written successfully for ${requestId}`); resolve(); } }); }); } catch (error) { this.inFlightResponses.delete(requestId); if (run.pendingApprovals.has(requestId)) { this.startApprovalTimeout(run, requestId); } throw error; } run.pendingApprovals.delete(requestId); this.inFlightResponses.delete(requestId); this.dismissApprovalNotification(requestId); } /** * Respond to a teammate's permission_request by applying permission_suggestions. * * FACT: Claude Code teammate runtime sends permission_request via the inbox protocol. * FACT: Teammates wait for permission_response in their own inbox. * FACT: control_response via the lead stdin does not reliably reach teammate request ids. * FACT: permission_suggestions.destination "localSettings" refers to {cwd}/.claude/settings.local.json. * FACT: Claude Code CLI reads this file via --setting-sources user,project,local. * * When allow=true: applies permission_suggestions, then replies to the teammate. * When allow=false: replies with an error so the teammate does not hang. */ private async respondToTeammatePermission( run: ProvisioningRun, agentId: string, requestId: string, allow: boolean, message?: string, permissionSuggestions?: import('@shared/utils/inboxNoise').PermissionSuggestion[], toolName?: string, toolInput?: Record ): Promise { if (!allow) { logger.info(`[${run.teamName}] Denied teammate ${agentId} permission ${requestId}`); this.sendTeammatePermissionResponse(run, agentId, requestId, { allow: false, message, toolName, }); return; } const suggestions = permissionSuggestions ?? []; const sendSuccessResponse = (): void => { this.sendTeammatePermissionResponse(run, agentId, requestId, { allow: true, message, permissionUpdates: suggestions, toolName, toolInput, }); }; // Apply permission_suggestions: add tool rules to project settings file. if (suggestions.length === 0) { logger.info( `[${run.teamName}] No permission_suggestions for ${requestId}; sending allow responses only` ); } else { // Resolve project cwd from team config let projectCwd: string | undefined; try { const config = await this.readConfigForStrictDecision(run.teamName); projectCwd = config?.projectPath ?? config?.members?.[0]?.cwd; } catch { // best-effort } if (!projectCwd) { logger.warn( `[${run.teamName}] Cannot resolve project cwd for permission rule; sending allow responses only` ); } else { for (const suggestion of suggestions) { // Handle "setMode" suggestions (e.g. Write/Edit tools suggest acceptEdits mode) // FACT: Write/Edit permission_requests have permission_suggestions: // { type: "setMode", mode: "acceptEdits", destination: "session" } // Since we can't change session mode of a subprocess, we translate to addRules. if (suggestion.type === 'setMode') { const mode = typeof suggestion.mode === 'string' ? suggestion.mode : ''; let toolNames: string[] = []; if (mode === 'acceptEdits') { toolNames = ['Edit', 'Write', 'NotebookEdit']; } else if (mode === 'bypassPermissions') { // Broad approval - add common tools toolNames = ['Edit', 'Write', 'NotebookEdit', 'Bash', 'Read', 'Grep', 'Glob']; } if (toolNames.length > 0) { const settingsPath = path.join(projectCwd, '.claude', 'settings.local.json'); try { await this.addPermissionRulesToSettings(settingsPath, toolNames, 'allow'); logger.info( `[${run.teamName}] Applied setMode "${mode}" for ${agentId}: ${toolNames.join(', ')} in ${settingsPath}` ); } catch (error) { logger.error( `[${run.teamName}] Failed to apply setMode: ${ error instanceof Error ? error.message : String(error) }` ); } } continue; } if (suggestion.type !== 'addRules' || !Array.isArray(suggestion.rules)) continue; let toolNames = suggestion.rules .map((r) => r.toolName) .filter((name): name is string => typeof name === 'string' && name.length > 0); if (toolNames.length === 0) continue; // Expand teammate-safe operational tools only. // This removes the bootstrap/task workflow race without accidentally granting // admin/runtime tools like team_stop or kanban_clear. if ( toolNames.some((name) => AGENT_TEAMS_NAMESPACED_TEAMMATE_OPERATIONAL_TOOL_NAMES.includes(name) ) ) { const merged = new Set([ ...toolNames, ...AGENT_TEAMS_NAMESPACED_TEAMMATE_OPERATIONAL_TOOL_NAMES, ]); toolNames = Array.from(merged); } const behavior = suggestion.behavior ?? 'allow'; // FACT: observed destinations are "localSettings" (project-level .claude/settings.local.json) const settingsPath = suggestion.destination === 'localSettings' ? path.join(projectCwd, '.claude', 'settings.local.json') : path.join(projectCwd, '.claude', 'settings.local.json'); // default to local try { await this.addPermissionRulesToSettings(settingsPath, toolNames, behavior); logger.info( `[${run.teamName}] Added permission rules for ${agentId}: ${toolNames.join(', ')} -> ${behavior} in ${settingsPath}` ); } catch (error) { logger.error( `[${run.teamName}] Failed to add permission rules: ${ error instanceof Error ? error.message : String(error) }` ); } } } } sendSuccessResponse(); // Also attempt control_response via stdin - the lead runtime MAY forward it // to the teammate subprocess. This was broken before (missing updatedInput: {}) // but is now fixed. Belt-and-suspenders: settings handle future calls, // control_response may unblock the CURRENT waiting prompt. if (allow && run.child?.stdin?.writable) { const updatedInput = this.buildTeammatePermissionUpdatedInput(toolName, toolInput, message) ?? {}; const controlResponse = { type: 'control_response', response: { subtype: 'success', request_id: requestId, response: { behavior: 'allow', updatedInput }, }, }; run.child.stdin.write(JSON.stringify(controlResponse) + '\n', (err) => { if (err) { logger.warn( `[${run.teamName}] control_response via stdin for teammate ${agentId} failed (non-critical): ${err.message}` ); } }); } } private sendTeammatePermissionResponse( run: ProvisioningRun, agentId: string, requestId: string, params: { allow: boolean; message?: string; permissionUpdates?: unknown[]; toolName?: string; toolInput?: Record; } ): void { const payload = params.allow ? { type: 'permission_response', request_id: requestId, subtype: 'success', response: { updated_input: this.buildTeammatePermissionUpdatedInput( params.toolName, params.toolInput, params.message ), permission_updates: params.permissionUpdates ?? [], }, } : { type: 'permission_response', request_id: requestId, subtype: 'error', error: params.message ?? 'Permission denied', }; this.persistInboxMessage(run.teamName, agentId, { from: run.request?.members.find((member) => member.role?.toLowerCase().includes('lead'))?.name ?? 'team-lead', to: agentId, text: JSON.stringify(payload), timestamp: nowIso(), read: false, summary: params.allow ? `Approved ${params.toolName ?? 'tool'} request` : `Denied ${params.toolName ?? 'tool'} request`, messageId: `permission-response-${run.runId}-${requestId}-${Date.now()}`, source: 'lead_process', }); this.teamChangeEmitter?.({ type: 'inbox', teamName: run.teamName, detail: `inboxes/${agentId}.json`, }); } private buildTeammatePermissionUpdatedInput( toolName: string | undefined, toolInput: Record | undefined, message: string | undefined ): Record | undefined { if (!toolInput) return undefined; if (toolName !== 'AskUserQuestion' || message === undefined) return toolInput; const answers = this.parseAskUserQuestionAnswers(message, toolInput); return Object.keys(answers).length > 0 ? { ...toolInput, answers } : toolInput; } private parseAskUserQuestionAnswers( message: string, toolInput: Record ): Record { try { const parsed = JSON.parse(message) as unknown; if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { return Object.fromEntries( Object.entries(parsed as Record).filter( (entry): entry is [string, string] => typeof entry[1] === 'string' ) ); } } catch { // Fall back to using the raw message as the first answer. } const questions = Array.isArray(toolInput.questions) ? (toolInput.questions as { question?: unknown }[]) : []; const firstQuestion = questions.find((question) => typeof question.question === 'string'); return typeof firstQuestion?.question === 'string' ? { [firstQuestion.question]: message } : {}; } /** * Safely add tool names to the permissions.allow (or deny) array in a Claude settings file. * Creates the file and parent directories if they don't exist. * Merges with existing entries — never overwrites. */ private async addPermissionRulesToSettings( settingsPath: string, toolNames: string[], behavior: string ): Promise { const dir = path.dirname(settingsPath); await fs.promises.mkdir(dir, { recursive: true }); // Read existing settings (or start with empty object) let settings: Record = {}; try { const raw = await fs.promises.readFile(settingsPath, 'utf-8'); const parsed = JSON.parse(raw) as unknown; if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { settings = parsed as Record; } } catch { // File doesn't exist or invalid JSON — start fresh } // Ensure permissions object exists if (!settings.permissions || typeof settings.permissions !== 'object') { settings.permissions = {}; } const perms = settings.permissions as Record; // Target array: "allow" or "deny" based on behavior const key = behavior === 'deny' ? 'deny' : 'allow'; if (!Array.isArray(perms[key])) { perms[key] = []; } const list = perms[key] as string[]; // Add tool names that aren't already in the list const existing = new Set(list); let added = 0; for (const name of toolNames) { if (!existing.has(name)) { list.push(name); added++; } } if (added === 0) return 0; // Nothing new to add await atomicWriteAsync(settingsPath, JSON.stringify(settings, null, 2) + '\n'); return added; } private async seedLeadBootstrapPermissionRules( teamName: string, projectCwd: string ): Promise { const settingsPath = path.join(projectCwd, '.claude', 'settings.local.json'); try { const allTools = [ ...AGENT_TEAMS_NAMESPACED_LEAD_BOOTSTRAP_TOOL_NAMES, 'Edit', 'Write', 'NotebookEdit', ]; const added = await this.addPermissionRulesToSettings(settingsPath, allTools, 'allow'); logger.info( `[${teamName}] Seeded lead bootstrap MCP rules in ${settingsPath} (${added} added)` ); } catch (error) { logger.warn( `[${teamName}] Failed to seed lead bootstrap MCP rules: ${ error instanceof Error ? error.message : String(error) }` ); } } /** * Called once provisioning has a promotable readiness signal. * For deterministic runs with a deferred first task, that signal must be result.success. * Process stays alive for subsequent tasks. */ private async handleProvisioningTurnComplete(run: ProvisioningRun): Promise { // Guard: must be set synchronously BEFORE any await to prevent // double-invocation from filesystem monitor + stream-json racing. if ( run.provisioningComplete || run.cancelRequested || run.processKilled || run.progress.state === 'failed' ) return; if ( this.hasPendingDeterministicFirstRealTurn(run) || !this.isProvisioningRunStillPromotable(run) ) { return; } // Prevent false "ready" when auth failure was printed in CLI output but the filesystem monitor // already observed files on disk. We only re-check stderr plus a trailing non-JSON stdout // fragment here to avoid late false positives from assistant/result stream-json payloads. const preCompleteText = this.getPreCompleteCliErrorText(run); if ( preCompleteText && this.hasApiError(preCompleteText) && !this.isAuthFailureWarning(preCompleteText, 'pre-complete') && // Skip if we already showed a warning for this error — the SDK had a chance to retry // and the CLI reported success. Killing now would be a false positive. !run.apiErrorWarningEmitted ) { this.failProvisioningWithApiError(run, preCompleteText); return; } if (preCompleteText && this.isAuthFailureWarning(preCompleteText, 'pre-complete')) { this.handleAuthFailureInOutput(run, preCompleteText, 'pre-complete'); return; } run.provisioningComplete = true; this.scheduleDeterministicBootstrapCompletionRecovery(run); this.resetRuntimeToolActivity(run, this.getRunLeadName(run)); this.setLeadActivity(run, 'idle'); // Clear provisioning timeout — no longer needed if (run.timeoutHandle) { clearTimeout(run.timeoutHandle); run.timeoutHandle = null; } this.stopFilesystemMonitor(run); this.stopStallWatchdog(run); if (run.isLaunch) { await this.updateConfigPostLaunch( run.teamName, run.request.cwd, run.detectedSessionId, run.request.color, { providerId: run.request.providerId, model: run.request.model, effort: run.request.effort, members: run.allEffectiveMembers, } ); await this.cleanupPrelaunchBackup(run.teamName); // Best-effort: detect CLI-suffixed member names (alice-2, bob-2) that indicate // a stale config.json was present during launch (double-launch race). try { const postLaunchConfigPath = path.join(getTeamsBasePath(), run.teamName, 'config.json'); const raw = await tryReadRegularFileUtf8(postLaunchConfigPath, { timeoutMs: TEAM_JSON_READ_TIMEOUT_MS, maxBytes: TEAM_CONFIG_MAX_BYTES, }); if (raw) { const config = JSON.parse(raw) as { members?: { name?: string; agentType?: string }[]; }; const suffixed = (config.members ?? []).filter( (m) => typeof m.name === 'string' && /-\d+$/.test(m.name) && !isLeadMember(m) ); if (suffixed.length > 0) { logger.warn( `[${run.teamName}] Post-launch: detected suffixed members: ` + `${suffixed.map((m) => m.name).join(', ')}. ` + 'This usually means the team was launched with stale config.json.' ); } } } catch { /* best-effort */ } // Audit: flag any expected member not registered in config.json after launch. await this.refreshMemberSpawnStatusesFromLeadInbox(run); await this.maybeAuditMemberSpawnStatuses(run, { force: true }); await this.finalizeMissingRegisteredMembersAsFailed(run); const persistedLaunchSnapshot = await this.reconcileFinalLaunchReportingSnapshot( run, await this.launchMixedSecondaryLaneIfNeeded(run) ); const failedSpawnMembers = persistedLaunchSnapshot ? persistedLaunchSnapshot.expectedMembers .filter( (memberName) => persistedLaunchSnapshot.members[memberName]?.launchState === 'failed_to_start' ) .map((memberName) => ({ name: memberName, error: persistedLaunchSnapshot.members[memberName]?.hardFailureReason, updatedAt: persistedLaunchSnapshot.members[memberName]?.lastEvaluatedAt ?? nowIso(), })) : this.getFailedSpawnMembers(run); const launchSummary = persistedLaunchSnapshot?.summary ?? this.getMemberLaunchSummary(run); const hasSpawnFailures = failedSpawnMembers.length > 0; const hasPendingBootstrap = !hasSpawnFailures && this.hasPendingLaunchMembers(run, launchSummary, persistedLaunchSnapshot); if ( this.isProvisioningRunPromotedToAlive(run) || !this.isProvisioningRunStillPromotable(run) ) { return; } const readyMessage = hasSpawnFailures ? `Launch completed with teammate errors — ${failedSpawnMembers .map((member) => member.name) .join(', ')} failed to start` : hasPendingBootstrap ? this.buildAggregatePendingLaunchMessage( 'Launch completed', run, launchSummary, persistedLaunchSnapshot ) : 'Team launched — process alive and ready'; const progress = updateProgress(run, 'ready', readyMessage, { cliLogsTail: extractCliLogsFromRun(run), messageSeverity: hasSpawnFailures || hasPendingBootstrap ? 'warning' : undefined, }); run.onProgress(progress); this.provisioningRunByTeam.delete(run.teamName); this.setAliveRunId(run.teamName, run.runId); logger.info(`[${run.teamName}] Launch complete. Process alive for subsequent tasks.`); if (!run.deterministicBootstrap && shouldUseGeminiStagedLaunch(run.request.providerId)) { run.pendingGeminiPostLaunchHydration = true; } // Force a post-ready detail refresh so Messages reload persisted lead_session // texts from JSONL even if the last visible assistant output only reached disk. this.teamChangeEmitter?.({ type: 'lead-message', teamName: run.teamName, runId: run.runId, detail: 'lead-session-sync', }); if (!hasSpawnFailures && !hasPendingBootstrap) { // Fire "Team Launched" notification only for clean launches. void this.fireTeamLaunchedNotification(run); } else if (hasSpawnFailures) { void this.fireTeamLaunchIncompleteNotification( run, failedSpawnMembers, launchSummary, persistedLaunchSnapshot ); } if (hasSpawnFailures) { const failureNotice = [ `Системное замечание: часть команды не запустилась.`, `Не стартовали тиммейты: ${failedSpawnMembers.map((member) => `@${member.name}`).join(', ')}.`, `Не считай их доступными, пока их запуск не будет повторён успешно.`, ].join(' '); await this.sendMessageToRun(run, failureNotice).catch((error: unknown) => logger.warn( `[${run.teamName}] failed to send teammate-start failure notice to lead: ${ error instanceof Error ? error.message : String(error) }` ) ); } // Pick up any direct messages that arrived before/while reconnecting. void this.relayLeadInboxMessages(run.teamName).catch((e: unknown) => logger.warn(`[${run.teamName}] post-reconnect relay failed: ${String(e)}`) ); // Solo teams have no teammate processes to resume work; kick off task execution // as a separate turn AFTER the launch is marked ready so the UI doesn't mix // long-running task output into the "Launching team" live output stream. if ( run.request.members.length === 0 && !shouldUseGeminiStagedLaunch(run.request.providerId) ) { void (async () => { try { const taskReader = new TeamTaskReader(); const tasks = await taskReader.getTasks(run.teamName); const active = tasks.filter(isTaskBoardSnapshotWorkCandidate); if (active.length === 0) return; const board = buildTaskBoardSnapshot(tasks); const message = [ `Reconnected and ready. Begin executing tasks now.`, `Execute tasks sequentially and keep the board + user updated:`, `- Identify the next READY task (pending or needsFix, not blocked by incomplete dependencies).`, `- If the task is unassigned, set yourself as owner.`, `- BEFORE doing any work on a task: mark it started (in_progress).`, `- Immediately SendMessage "user" that you started task # (what you're doing + next step).`, `- While working: after each meaningful milestone/decision/blocker, add a task comment on #. If user-relevant, also SendMessage "user".`, `- On completion: add a final task comment with your full results (findings, report, analysis, code changes summary, or any deliverable), then mark the task completed, then SendMessage "user" with a brief summary of the outcome (2-4 sentences) and "Full details in task comment ". The task comment is the primary delivery channel — the user reads results on the task board.`, `- Do NOT start the next task until the current task is completed (default: one task in_progress at a time).`, board.trim(), ] .filter(Boolean) .join('\n\n'); await this.sendMessageToRun(run, message); } catch (error) { logger.warn( `[${run.teamName}] Failed to kick off solo task resumption: ${ error instanceof Error ? error.message : String(error) }` ); } })(); } if ( run.pendingGeminiPostLaunchHydration && !run.geminiPostLaunchHydrationInFlight && !run.cancelRequested ) { void this.injectGeminiPostLaunchHydration(run); } return; } // Quick verification: config should exist by now const configProbe = await this.waitForValidConfig(run, 5000); if (!configProbe.ok) { logger.warn( `[${run.teamName}] Provisioning turn completed but no config.json found — marking ready anyway` ); } if (configProbe.ok && configProbe.location === 'default') { const configuredTeamsBasePath = getTeamsBasePath(); const progress = updateProgress(run, 'failed', 'Provisioning failed validation', { error: `TeamCreate produced config.json under a different Claude root (${configProbe.configPath}). ` + `This app is configured to read teams from ${configuredTeamsBasePath}. ` + 'Align the app Claude root setting with the CLI, then retry.', cliLogsTail: extractCliLogsFromRun(run), }); run.onProgress(progress); run.processKilled = true; killTeamProcess(run.child); this.cleanupRun(run); return; } // Persist teammates metadata separately from config.json. await this.persistMembersMeta(run.teamName, run.request); await this.updateConfigPostLaunch( run.teamName, run.request.cwd, run.detectedSessionId, run.request.color, { providerId: run.request.providerId, model: run.request.model, effort: run.request.effort, members: run.allEffectiveMembers, } ); // Audit: flag any expected member not registered in config.json after provisioning. await this.refreshMemberSpawnStatusesFromLeadInbox(run); await this.maybeAuditMemberSpawnStatuses(run, { force: true }); await this.finalizeMissingRegisteredMembersAsFailed(run); const persistedLaunchSnapshot = await this.reconcileFinalLaunchReportingSnapshot( run, await this.launchMixedSecondaryLaneIfNeeded(run) ); const failedSpawnMembers = persistedLaunchSnapshot ? persistedLaunchSnapshot.expectedMembers .filter( (memberName) => persistedLaunchSnapshot.members[memberName]?.launchState === 'failed_to_start' ) .map((memberName) => ({ name: memberName, error: persistedLaunchSnapshot.members[memberName]?.hardFailureReason, updatedAt: persistedLaunchSnapshot.members[memberName]?.lastEvaluatedAt ?? nowIso(), })) : this.getFailedSpawnMembers(run); const launchSummary = persistedLaunchSnapshot?.summary ?? this.getMemberLaunchSummary(run); const hasSpawnFailures = failedSpawnMembers.length > 0; const hasPendingBootstrap = !hasSpawnFailures && this.hasPendingLaunchMembers(run, launchSummary, persistedLaunchSnapshot); if (this.isProvisioningRunPromotedToAlive(run) || !this.isProvisioningRunStillPromotable(run)) { return; } const progress = updateProgress( run, 'ready', hasSpawnFailures ? `Provisioning completed with teammate errors — ${failedSpawnMembers .map((member) => member.name) .join(', ')} failed to start` : hasPendingBootstrap ? this.buildAggregatePendingLaunchMessage( 'Team provisioned', run, launchSummary, persistedLaunchSnapshot ) : 'Team provisioned — process alive and ready', { cliLogsTail: extractCliLogsFromRun(run), messageSeverity: hasSpawnFailures || hasPendingBootstrap ? 'warning' : undefined, } ); run.onProgress(progress); if (hasSpawnFailures) { this.writeLaunchFailureArtifactPackBestEffort(run, { reason: run.isLaunch ? 'launch_completed_with_teammate_errors' : 'provisioning_completed_with_teammate_errors', launchSnapshot: persistedLaunchSnapshot, }); } this.provisioningRunByTeam.delete(run.teamName); this.setAliveRunId(run.teamName, run.runId); logger.info(`[${run.teamName}] Provisioning complete. Process alive for subsequent tasks.`); if (!run.deterministicBootstrap && shouldUseGeminiStagedLaunch(run.request.providerId)) { run.pendingGeminiPostLaunchHydration = true; } // Force a post-ready detail refresh so Messages reload persisted lead_session // texts from JSONL even if the last visible assistant output only reached disk. this.teamChangeEmitter?.({ type: 'lead-message', teamName: run.teamName, runId: run.runId, detail: 'lead-session-sync', }); if (!hasSpawnFailures && !hasPendingBootstrap) { // Fire "Team Launched" notification only for clean launches. void this.fireTeamLaunchedNotification(run); } else if (hasSpawnFailures) { void this.fireTeamLaunchIncompleteNotification( run, failedSpawnMembers, launchSummary, persistedLaunchSnapshot ); } if (hasSpawnFailures) { const failureNotice = [ `Системное замечание: часть команды не запустилась.`, `Не стартовали тиммейты: ${failedSpawnMembers.map((member) => `@${member.name}`).join(', ')}.`, `Не считай их доступными, пока их запуск не будет повторён успешно.`, ].join(' '); await this.sendMessageToRun(run, failureNotice).catch((error: unknown) => logger.warn( `[${run.teamName}] failed to send teammate-start failure notice to lead: ${ error instanceof Error ? error.message : String(error) }` ) ); } // Pick up any direct messages that arrived during provisioning. void this.relayLeadInboxMessages(run.teamName).catch((e: unknown) => logger.warn(`[${run.teamName}] post-provisioning relay failed: ${String(e)}`) ); if ( run.pendingGeminiPostLaunchHydration && !run.geminiPostLaunchHydrationInFlight && !run.cancelRequested ) { void this.injectGeminiPostLaunchHydration(run); } } // --------------------------------------------------------------------------- // Team Launched notification // --------------------------------------------------------------------------- /** * Fires a "team_launched" notification when a team transitions to ready state. * Uses the existing addTeamNotification() pipeline. */ private async fireTeamLaunchedNotification(run: ProvisioningRun): Promise { if (run.teamLaunchedNotificationFired) { return; } try { const config = ConfigManager.getInstance().getConfig(); const suppressToast = !config.notifications.notifyOnTeamLaunched; const displayName = run.request.displayName || run.teamName; const joinedCount = run.expectedMembers?.length ?? 0; const allJoined = joinedCount > 0 && this.areAllExpectedLaunchMembersConfirmed(run); if (run.isLaunch && joinedCount > 0 && !allJoined) { return; } run.teamLaunchedNotificationFired = true; const body = run.isLaunch ? allJoined ? `Team "${displayName}" has been launched - all ${joinedCount} teammates joined and are ready for tasks.` : `Team "${displayName}" has been launched and is ready for tasks.` : `Team "${displayName}" has been provisioned and is ready for tasks.`; await NotificationManager.getInstance().addTeamNotification({ teamEventType: 'team_launched', teamName: run.teamName, teamDisplayName: displayName, from: 'system', summary: run.isLaunch ? 'Team launched' : 'Team provisioned', body, dedupeKey: `team_launched:${run.teamName}:${run.runId}`, target: { kind: 'team', teamName: run.teamName, section: 'overview' }, projectPath: run.request.cwd, suppressToast, }); } catch (error) { run.teamLaunchedNotificationFired = false; logger.warn( `[${run.teamName}] Failed to fire team_launched notification: ${ error instanceof Error ? error.message : String(error) }` ); } } private async fireTeamLaunchIncompleteNotification( run: ProvisioningRun, failedMembers: readonly { name: string }[], launchSummary: { confirmedCount: number; pendingCount: number; failedCount: number; runtimeAlivePendingCount: number; runtimeProcessPendingCount?: number; }, snapshot?: PersistedTeamLaunchSnapshot | null ): Promise { try { const config = ConfigManager.getInstance().getConfig(); const suppressToast = !config.notifications.notifyOnTeamLaunched; const displayName = run.request.displayName || run.teamName; const expectedMembers = [ ...new Set( [ ...(snapshot?.expectedMembers ?? []), ...(run.expectedMembers ?? []), ...run.allEffectiveMembers.map((member) => member.name).filter(Boolean), ].filter(Boolean) ), ]; const expectedCount = expectedMembers.length; if (expectedCount === 0) return; const failedNames = this.getLaunchIncompleteFailedNames( run, expectedMembers, failedMembers, snapshot ); if (failedNames.length === 0) { return; } const pendingNames = this.getLaunchIncompletePendingNames( run, expectedMembers, failedNames, snapshot ); const joinedCount = this.getLaunchIncompleteJoinedCount( run, expectedMembers, failedNames.length + pendingNames.length, launchSummary, snapshot ); const missingCount = Math.max(0, launchSummary.pendingCount + launchSummary.failedCount); const bodyParts = [`${joinedCount}/${expectedCount} joined`]; if (failedNames.length > 0) { bodyParts.push(`failed: ${this.formatLaunchIncompleteMemberMentions(failedNames)}`); } if (pendingNames.length > 0) { bodyParts.push(`still joining: ${this.formatLaunchIncompleteMemberMentions(pendingNames)}`); } if (bodyParts.length === 1 && missingCount > 0 && joinedCount < expectedCount) { const genericMissingCount = Math.min(missingCount, expectedCount - joinedCount); bodyParts.push( `${genericMissingCount} teammate${genericMissingCount === 1 ? '' : 's'} not joined yet` ); } await NotificationManager.getInstance().addTeamNotification({ teamEventType: 'team_launch_incomplete', teamName: run.teamName, teamDisplayName: displayName, from: 'system', summary: 'Team launch incomplete', body: bodyParts.join(' · '), dedupeKey: `team_launch_incomplete:${run.teamName}:${run.runId}`, target: { kind: 'team', teamName: run.teamName, section: 'members' }, projectPath: run.request.cwd, suppressToast, }); } catch (error) { logger.warn( `[${run.teamName}] Failed to fire team_launch_incomplete notification: ${ error instanceof Error ? error.message : String(error) }` ); } } private getLaunchIncompleteMemberEvidence( run: ProvisioningRun, snapshot: PersistedTeamLaunchSnapshot | null | undefined, memberName: string ): { live?: MemberSpawnStatusEntry; persisted?: PersistedTeamLaunchMemberState; } { return { live: run.memberSpawnStatuses?.get(memberName), persisted: snapshot?.members[memberName], }; } private formatLaunchIncompleteMemberMentions(names: readonly string[]): string { return names.map((name) => `@${name}`).join(', '); } private getLaunchIncompleteFailedNames( run: ProvisioningRun, expectedMembers: readonly string[], failedMembers: readonly { name: string }[], snapshot?: PersistedTeamLaunchSnapshot | null ): string[] { const failedNames = new Set(failedMembers.map((member) => member.name).filter(Boolean)); for (const memberName of expectedMembers) { const { live, persisted } = this.getLaunchIncompleteMemberEvidence(run, snapshot, memberName); const liveResolved = live?.launchState === 'confirmed_alive' || live?.bootstrapConfirmed === true || live?.launchState === 'skipped_for_launch' || live?.skippedForLaunch === true; const persistedResolved = persisted?.launchState === 'confirmed_alive' || persisted?.bootstrapConfirmed === true || persisted?.launchState === 'skipped_for_launch' || persisted?.skippedForLaunch === true; if (liveResolved || persistedResolved) { failedNames.delete(memberName); continue; } if ( live?.launchState === 'failed_to_start' || persisted?.launchState === 'failed_to_start' || live?.hardFailure === true || persisted?.hardFailure === true ) { failedNames.add(memberName); } } return [...failedNames].sort((left, right) => left.localeCompare(right)); } private getLaunchIncompletePendingNames( run: ProvisioningRun, expectedMembers: readonly string[], failedNames: readonly string[], snapshot?: PersistedTeamLaunchSnapshot | null ): string[] { const failed = new Set(failedNames); return expectedMembers .filter((memberName) => { if (failed.has(memberName)) { return false; } const { live, persisted } = this.getLaunchIncompleteMemberEvidence( run, snapshot, memberName ); const hasEvidence = live !== undefined || persisted !== undefined; if (!hasEvidence) { return false; } const confirmed = live?.launchState === 'confirmed_alive' || persisted?.launchState === 'confirmed_alive' || live?.bootstrapConfirmed === true || persisted?.bootstrapConfirmed === true; if (confirmed) { return false; } const skipped = live?.launchState === 'skipped_for_launch' || persisted?.launchState === 'skipped_for_launch' || live?.skippedForLaunch === true || persisted?.skippedForLaunch === true; return !skipped; }) .sort((left, right) => left.localeCompare(right)); } private getLaunchIncompleteJoinedCount( run: ProvisioningRun, expectedMembers: readonly string[], namedMissingCount: number, launchSummary: { confirmedCount: number; }, snapshot?: PersistedTeamLaunchSnapshot | null ): number { const evidenceConfirmedCount = expectedMembers.filter((memberName) => { const { live, persisted } = this.getLaunchIncompleteMemberEvidence(run, snapshot, memberName); return ( live?.launchState === 'confirmed_alive' || persisted?.launchState === 'confirmed_alive' || live?.bootstrapConfirmed === true || persisted?.bootstrapConfirmed === true ); }).length; const namedMissingUpperBound = expectedMembers.length - namedMissingCount; const rawJoinedCount = namedMissingCount > 0 ? Math.min( namedMissingUpperBound, Math.max(evidenceConfirmedCount, launchSummary.confirmedCount) ) : Math.max(evidenceConfirmedCount, launchSummary.confirmedCount); return Math.max(0, Math.min(expectedMembers.length, rawJoinedCount)); } // --------------------------------------------------------------------------- // Same-team native delivery dedup (Layer 2) // --------------------------------------------------------------------------- private collectConfirmedSameTeamPairs( messages: InboxMessage[], fingerprints: NativeSameTeamFingerprint[], leadName: string ): { confirmedMessageIds: Set; matchedFingerprintIds: Set } { const confirmedMessageIds = new Set(); const matchedFingerprintIds = new Set(); if (fingerprints.length === 0) { return { confirmedMessageIds, matchedFingerprintIds }; } // Build group key: from + normalizedText (summary checked during pairing, not grouping) const groupKey = (from: string, text: string) => `${from}\0${text}`; // Group fingerprints by (from, text), sorted FIFO by seenAt within each group const fpByGroup = new Map(); for (const fp of fingerprints) { const key = groupKey(fp.from, fp.text); let group = fpByGroup.get(key); if (!group) { group = []; fpByGroup.set(key, group); } group.push(fp); } for (const group of fpByGroup.values()) { group.sort((a, b) => a.seenAt - b.seenAt); } // Collect eligible inbox messages, grouped by (from, text), sorted FIFO by timestamp type EligibleMsg = InboxMessage & { messageId: string; parsedTs: number }; const msgByGroup = new Map(); for (const m of messages) { if (m.read) continue; if (m.source) continue; if (!this.hasStableMessageId(m)) continue; const fromName = m.from?.trim() ?? ''; if (!fromName || fromName === leadName || fromName === 'user') continue; const parsedTs = Date.parse(m.timestamp); if (!Number.isFinite(parsedTs)) continue; const key = groupKey(fromName, normalizeSameTeamText(m.text)); let group = msgByGroup.get(key); if (!group) { group = []; msgByGroup.set(key, group); } group.push({ ...m, parsedTs } as EligibleMsg); } for (const group of msgByGroup.values()) { group.sort((a, b) => a.parsedTs - b.parsedTs); } // FIFO pair within each group: first fingerprint → first message, second → second, etc. // This prevents delayed native delivery from pairing with the wrong inbox row // when identical messages (e.g. "Done") are sent close together. for (const [key, fps] of fpByGroup) { const msgs = msgByGroup.get(key); if (!msgs || msgs.length === 0) continue; const limit = Math.min(fps.length, msgs.length); for (let i = 0; i < limit; i++) { const fp = fps[i]; const m = msgs[i]; // Summary validation: if both sides have summary, they must match if (fp.summary && m.summary?.trim() && fp.summary !== m.summary.trim()) continue; // Time window validation if (Math.abs(m.parsedTs - fp.seenAt) > TeamProvisioningService.SAME_TEAM_MATCH_WINDOW_MS) { continue; } confirmedMessageIds.add(m.messageId); matchedFingerprintIds.add(fp.id); } } return { confirmedMessageIds, matchedFingerprintIds }; } private rememberSameTeamNativeFingerprints( teamName: string, blocks: ParsedTeammateContent[] ): void { const teamKey = teamName.trim(); const existing = this.recentSameTeamNativeFingerprints.get(teamKey) ?? []; const now = Date.now(); const cutoff = now - TeamProvisioningService.SAME_TEAM_NATIVE_FINGERPRINT_TTL_MS; const fresh = existing.filter((fp) => fp.seenAt > cutoff); for (const block of blocks) { fresh.push({ id: randomUUID(), from: block.teammateId.trim(), text: normalizeSameTeamText(block.content), summary: (block.summary ?? '').trim(), seenAt: now, }); } this.recentSameTeamNativeFingerprints.set(teamKey, fresh); } private consumeMatchedSameTeamFingerprints(teamName: string, matchedIds: Set): void { if (matchedIds.size === 0) return; const current = this.recentSameTeamNativeFingerprints.get(teamName.trim()) ?? []; if (current.length === 0) return; const remaining = current.filter((fp) => !matchedIds.has(fp.id)); if (remaining.length > 0) { this.recentSameTeamNativeFingerprints.set(teamName.trim(), remaining); } else { this.recentSameTeamNativeFingerprints.delete(teamName.trim()); } } private getFreshSameTeamNativeFingerprints(teamName: string): NativeSameTeamFingerprint[] { const all = this.recentSameTeamNativeFingerprints.get(teamName) ?? []; if (all.length === 0) return []; const cutoff = Date.now() - TeamProvisioningService.SAME_TEAM_NATIVE_FINGERPRINT_TTL_MS; const fresh = all.filter((fp) => fp.seenAt > cutoff); if (fresh.length !== all.length) { if (fresh.length > 0) { this.recentSameTeamNativeFingerprints.set(teamName, fresh); } else { this.recentSameTeamNativeFingerprints.delete(teamName); } } return fresh; } private isPotentialSameTeamCliMessage(m: InboxMessage, leadName: string): boolean { if (m.source) return false; const fromName = m.from?.trim() ?? ''; if (!fromName || fromName === leadName || fromName === 'user') return false; const toName = m.to?.trim(); if (toName && toName !== leadName) return false; return true; } private shouldDeferSameTeamMessage( m: InboxMessage, leadName: string, runStartedAtMs: number ): boolean { if (!this.isPotentialSameTeamCliMessage(m, leadName)) return false; const messageTs = Date.parse(m.timestamp); if (!Number.isFinite(messageTs) || messageTs < 0) return false; if ( Number.isFinite(runStartedAtMs) && messageTs < runStartedAtMs - TeamProvisioningService.SAME_TEAM_RUN_START_SKEW_MS ) { return false; } const ageMs = Date.now() - messageTs; if (ageMs < 0) return false; return ageMs < TeamProvisioningService.SAME_TEAM_NATIVE_DELIVERY_GRACE_MS; } private async confirmSameTeamNativeMatches( teamName: string, leadName: string, messages: InboxMessage[] ): Promise<{ nativeMatchedMessageIds: Set; persisted: boolean }> { const fingerprints = this.getFreshSameTeamNativeFingerprints(teamName); const { confirmedMessageIds, matchedFingerprintIds } = this.collectConfirmedSameTeamPairs( messages, fingerprints, leadName ); if (confirmedMessageIds.size === 0) { return { nativeMatchedMessageIds: confirmedMessageIds, persisted: true }; } const toMarkRead = Array.from(confirmedMessageIds, (messageId) => ({ messageId })); let persisted = false; try { await this.markInboxMessagesRead(teamName, leadName, toMarkRead); persisted = true; } catch { // keep fingerprints alive for next attempt } if (persisted) { // Durable: inbox says read=true. Safe to add in-memory dedup and consume fingerprints. const relayedIds = this.relayedLeadInboxMessageIds.get(teamName) ?? new Set(); for (const messageId of confirmedMessageIds) { relayedIds.add(messageId); } this.relayedLeadInboxMessageIds.set(teamName, this.trimRelayedSet(relayedIds)); this.consumeMatchedSameTeamFingerprints(teamName, matchedFingerprintIds); } // If NOT persisted: don't add to relayedIds, don't consume fingerprints. // Next relay cycle will see the message in unread, re-match, and retry persist. return { nativeMatchedMessageIds: confirmedMessageIds, persisted }; } private async reconcileSameTeamNativeDeliveries( teamName: string, leadName: string ): Promise { let leadInboxMessages: Awaited> = []; try { leadInboxMessages = await this.inboxReader.getMessagesFor(teamName, leadName); } catch { return; } const { nativeMatchedMessageIds, persisted } = await this.confirmSameTeamNativeMatches( teamName, leadName, leadInboxMessages ); // If native was matched but persist failed, schedule a quick retry // so we don't wait for the 16s deferred timer to retry the disk write. if (nativeMatchedMessageIds.size > 0 && !persisted) { this.scheduleSameTeamPersistRetry(teamName); } } private scheduleSameTeamDeferredRetry(teamName: string): void { const key = `same-team-deferred:${teamName}`; if (this.pendingTimeouts.has(key)) return; const timer = setTimeout(() => { this.pendingTimeouts.delete(key); void this.relayLeadInboxMessages(teamName).catch((e: unknown) => logger.warn(`[${teamName}] same-team deferred retry failed: ${String(e)}`) ); }, TeamProvisioningService.SAME_TEAM_NATIVE_DELIVERY_GRACE_MS + 1_000); this.pendingTimeouts.set(key, timer); } /** * Best-effort durable follow-up after native delivery was matched but inbox read-state * could not be persisted. If the run dies before this retry succeeds, a later reconnect * may still relay the row once because in-memory dedupe is not durable. */ private scheduleSameTeamPersistRetry(teamName: string): void { const key = `same-team-persist:${teamName}`; if (this.pendingTimeouts.has(key)) return; const timer = setTimeout(() => { this.pendingTimeouts.delete(key); void this.relayLeadInboxMessages(teamName).catch((e: unknown) => logger.warn(`[${teamName}] same-team persist retry failed: ${String(e)}`) ); }, TeamProvisioningService.SAME_TEAM_PERSIST_RETRY_MS); this.pendingTimeouts.set(key, timer); } private shouldFinalizeIncompleteLaunchState(run: ProvisioningRun): boolean { return ( run.isLaunch && run.launchStateClearedForRun !== false && !run.provisioningComplete && !run.cancelRequested && run.launchCleanupStateFinalized !== true ); } private buildIncompleteLaunchCleanupReason( run: ProvisioningRun, fallback = 'Launch ended before teammate bootstrap completed.' ): string { return typeof run.progress.error === 'string' && run.progress.error.trim() ? run.progress.error.trim() : run.progress.state === 'failed' && run.progress.message.trim() ? run.progress.message.trim() : fallback; } private markIncompleteLaunchStateFinalized(run: ProvisioningRun, cleanupReason: string): void { logger.warn(`[${run.teamName}] Launch cleanup finalizing unconfirmed bootstrap members`, { runId: run.runId, progressState: run.progress.state, progressMessage: run.progress.message, progressError: run.progress.error ?? null, cleanupReason, unconfirmedMembers: this.getUnconfirmedBootstrapMemberNames(run), ...this.buildStdoutCarryDiagnostic(run), }); this.markUnconfirmedBootstrapMembersFailed(run, cleanupReason, { cleanupRequested: true, preserveExistingFailure: true, }); run.launchCleanupStateFinalized = true; } private async finalizeIncompleteLaunchStateBeforeCleanup( run: ProvisioningRun, fallbackReason?: string ): Promise { if (!this.shouldFinalizeIncompleteLaunchState(run)) { return; } const cleanupReason = this.buildIncompleteLaunchCleanupReason(run, fallbackReason); this.markIncompleteLaunchStateFinalized(run, cleanupReason); try { await this.persistLaunchStateSnapshot(run, 'finished'); } catch (error) { run.launchCleanupStateFinalized = false; logger.warn( `[${run.teamName}] Failed to finalize launch state before cleanup: ${ error instanceof Error ? error.message : String(error) }` ); } } /** * Remove a run from tracking maps. */ private cleanupRun(run: ProvisioningRun): void { const currentTrackedRunId = this.getTrackedRunId(run.teamName); const hasNewerTrackedRun = currentTrackedRunId !== null && currentTrackedRunId !== run.runId; const retainedClaudeLogs = hasNewerTrackedRun ? null : buildRetainedClaudeLogsSnapshot(run); if (!hasNewerTrackedRun) { peekAutoResumeService()?.cancelPendingAutoResume(run.teamName); } if (!hasNewerTrackedRun && this.shouldFinalizeIncompleteLaunchState(run)) { const cleanupReason = this.buildIncompleteLaunchCleanupReason(run); this.markIncompleteLaunchStateFinalized(run, cleanupReason); void this.persistLaunchStateSnapshot(run, 'finished'); } if ( !hasNewerTrackedRun && (run.progress.state === 'failed' || (run.isLaunch && run.launchStateClearedForRun !== false && !run.provisioningComplete && !run.cancelRequested)) ) { this.writeLaunchFailureArtifactPackBestEffort(run, { reason: run.progress.state === 'failed' ? 'launch_progress_failed' : 'launch_cleanup_unconfirmed_bootstrap', }); } this.resetRuntimeToolActivity(run); this.setLeadActivity(run, 'offline'); run.pendingDirectCrossTeamSendRefresh = false; if (run.timeoutHandle) { clearTimeout(run.timeoutHandle); run.timeoutHandle = null; } this.stopStallWatchdog(run); if (run.silentUserDmForwardClearHandle) { clearTimeout(run.silentUserDmForwardClearHandle); run.silentUserDmForwardClearHandle = null; } clearPostCompactReminderState(run); clearGeminiPostLaunchHydrationState(run); this.stopFilesystemMonitor(run); // Remove stream listeners to prevent data handlers firing on a cleaned-up run if (run.child) { run.child.stdout?.removeAllListeners('data'); run.child.stderr?.removeAllListeners('data'); } if (this.provisioningRunByTeam.get(run.teamName) === run.runId) { this.provisioningRunByTeam.delete(run.teamName); } if (this.aliveRunByTeam.get(run.teamName) === run.runId) { this.deleteAliveRunId(run.teamName); } if (!hasNewerTrackedRun) { this.clearSecondaryRuntimeRuns(run.teamName); } if (!hasNewerTrackedRun) { this.invalidateRuntimeSnapshotCaches(run.teamName); this.invalidateMemberSpawnStatusesCache(run.teamName); this.leadInboxRelayInFlight.delete(run.teamName); this.relayedLeadInboxMessageIds.delete(run.teamName); this.pendingCrossTeamFirstReplies.delete(run.teamName); this.recentCrossTeamLeadDeliveryMessageIds.delete(run.teamName); this.recentSameTeamNativeFingerprints.delete(run.teamName); this.clearSameTeamRetryTimers(run.teamName); this.clearLeadInboxFollowUpRelayTimer(run.teamName); } for (const memberName of run.memberSpawnStatuses.keys()) { const key = this.getMemberLaunchGraceKey(run, memberName); const timer = this.pendingTimeouts.get(key); if (timer) { clearTimeout(timer); this.pendingTimeouts.delete(key); } } run.activeCrossTeamReplyHints = []; run.pendingInboxRelayCandidates = []; if (!hasNewerTrackedRun) { for (const key of Array.from(this.memberInboxRelayInFlight.keys())) { if (key.startsWith(`${run.teamName}:`)) { this.memberInboxRelayInFlight.delete(key); } } for (const key of Array.from(this.openCodeMemberInboxRelayInFlight.keys())) { if (key.startsWith(`opencode:${run.teamName}:`)) { this.openCodeMemberInboxRelayInFlight.delete(key); } } for (const key of Array.from(this.openCodePromptDeliveryWatchdogTimers.keys())) { if (key.startsWith(`opencode-delivery:${run.teamName}:`)) { const timer = this.openCodePromptDeliveryWatchdogTimers.get(key); if (timer) clearTimeout(timer); this.openCodePromptDeliveryWatchdogTimers.delete(key); this.openCodePromptDeliveryWatchdogDeadlines.delete(key); } } for ( let index = this.openCodePromptDeliveryWatchdogQueue.length - 1; index >= 0; index -= 1 ) { if (this.openCodePromptDeliveryWatchdogQueue[index]?.teamName === run.teamName) { this.openCodePromptDeliveryWatchdogQueue.splice(index, 1); } } for (const key of Array.from(this.relayedMemberInboxMessageIds.keys())) { if (key.startsWith(`${run.teamName}:`)) { this.relayedMemberInboxMessageIds.delete(key); } } this.liveLeadProcessMessages.delete(run.teamName); } else { this.pruneLiveLeadMessagesForCleanedRun(run); } // Dismiss any pending tool approvals for this run if (run.pendingApprovals.size > 0) { for (const requestId of run.pendingApprovals.keys()) { this.clearApprovalTimeout(requestId); this.inFlightResponses.delete(requestId); this.dismissApprovalNotification(requestId); } this.emitToolApprovalEvent({ dismissed: true, teamName: run.teamName, runId: run.runId }); run.pendingApprovals.clear(); } // Clean up the generated MCP config file (best-effort, fire-and-forget) if (run.mcpConfigPath) { void this.mcpConfigBuilder.removeConfigFile(run.mcpConfigPath); run.mcpConfigPath = null; } this.removeRunMemberMcpConfigFilesLater(run); if (run.bootstrapSpecPath) { void removeDeterministicBootstrapSpecFile(run.bootstrapSpecPath); run.bootstrapSpecPath = null; } if (run.bootstrapUserPromptPath) { void removeDeterministicBootstrapUserPromptFile(run.bootstrapUserPromptPath); run.bootstrapUserPromptPath = null; } if (!hasNewerTrackedRun) { if (retainedClaudeLogs) { this.retainedClaudeLogsByTeam.set(run.teamName, retainedClaudeLogs); } else { this.retainedClaudeLogsByTeam.delete(run.teamName); } } // Remove from runs Map to free memory (stdoutBuffer, stderrBuffer, claudeLogLines) if (run.progress) { this.retainProvisioningProgress(run.runId, run.progress); } this.runs.delete(run.runId); } /** * Polls the filesystem to track provisioning progress in real time. * Emits progress updates as team files appear (config, inboxes, tasks). */ private startFilesystemMonitor(run: ProvisioningRun, request: TeamCreateRequest): void { const configuredTeamDir = path.join(getTeamsBasePath(), run.teamName); const defaultTeamDir = path.join(getAutoDetectedClaudeBasePath(), 'teams', run.teamName); const tasksDir = path.join(getTasksBasePath(), run.teamName); const primaryProvisioningMembers = Array.isArray(run.effectiveMembers) ? run.effectiveMembers : request.members; const primaryProvisioningMemberCount = primaryProvisioningMembers.length; const resolveTeamDir = async (): Promise => { const configPath = path.join(configuredTeamDir, 'config.json'); try { await fs.promises.access(configPath, fs.constants.F_OK); return configuredTeamDir; } catch { // fallback to default location } if (path.resolve(configuredTeamDir) !== path.resolve(defaultTeamDir)) { const defaultConfigPath = path.join(defaultTeamDir, 'config.json'); try { await fs.promises.access(defaultConfigPath, fs.constants.F_OK); return defaultTeamDir; } catch { // not found in either location } } return null; }; const countFiles = async (dir: string, ext: string): Promise => { try { const entries = await fs.promises.readdir(dir); return entries.filter((e) => e.endsWith(ext) && !e.startsWith('.')).length; } catch { return 0; } }; const poll = async (): Promise => { if (run.cancelRequested || run.processKilled || run.progress.state === 'ready') { return; } try { if (run.fsPhase === 'waiting_config') { const teamDir = await resolveTeamDir(); if (teamDir) { run.fsPhase = 'waiting_members'; const progress = updateProgress( run, 'assembling', 'Team config created, waiting for members', { configReady: true } ); run.onProgress(progress); } } if (run.fsPhase === 'waiting_members') { if (run.deterministicBootstrap) { const registeredNames = await this.getRegisteredTeamMemberNames(run.teamName); const registeredMembers = registeredNames ? primaryProvisioningMembers.filter((member) => registeredNames.has(member.name)) .length : 0; if (registeredMembers >= primaryProvisioningMemberCount) { run.fsPhase = 'all_files_found'; return; } } if (primaryProvisioningMemberCount === 0) { if (run.deterministicBootstrap) { run.fsPhase = 'all_files_found'; } else { run.fsPhase = 'waiting_tasks'; const progress = updateProgress(run, 'finalizing', 'Solo team, preparing workspace'); run.onProgress(progress); } } else { const teamDir = (await resolveTeamDir()) ?? configuredTeamDir; const inboxDir = path.join(teamDir, 'inboxes'); const inboxCount = await countFiles(inboxDir, '.json'); if (inboxCount >= primaryProvisioningMemberCount) { run.fsPhase = 'waiting_tasks'; const progress = updateProgress( run, 'finalizing', `Prepared communication channels for all ${inboxCount} members, preparing workspace` ); run.onProgress(progress); } else if (inboxCount > 0) { const progress = updateProgress( run, 'assembling', `Prepared communication channels for ${inboxCount}/${primaryProvisioningMemberCount} members` ); run.onProgress(progress); } } } if (run.fsPhase === 'waiting_tasks') { if (run.waitingTasksSince === null) { run.waitingTasksSince = Date.now(); } const taskCount = await countFiles(tasksDir, '.json'); const taskFound = taskCount > 0; const taskFallbackExpired = !taskFound && Date.now() - run.waitingTasksSince >= TASK_WAIT_FALLBACK_MS; if (taskFound || taskFallbackExpired) { run.fsPhase = 'all_files_found'; // Legacy filesystem fallback - deterministic bootstrap waits for stream-json success. // The process stays alive for subsequent tasks. if (!run.deterministicBootstrap && !run.provisioningComplete) { void this.handleProvisioningTurnComplete(run); } } } } catch (error) { logger.debug( `FS monitor poll error: ${error instanceof Error ? error.message : String(error)}` ); } }; run.fsMonitorHandle = setInterval(() => { void poll(); }, FS_MONITOR_POLL_MS); // Best-effort monitor; should not keep the process alive. run.fsMonitorHandle.unref(); // Run first poll immediately void poll(); } private stopFilesystemMonitor(run: ProvisioningRun): void { if (run.fsMonitorHandle) { clearInterval(run.fsMonitorHandle); run.fsMonitorHandle = null; } } private isProvisioningRunFailed(run: ProvisioningRun): boolean { return run.progress.state === 'failed'; } private async handleProcessExit(run: ProvisioningRun, code: number | null): Promise { if (run.finalizingByTimeout) { return; } if (run.progress.state === 'failed' || run.cancelRequested) { return; } // Skip if respawn after auth failure is in progress — the old process is being replaced if (run.authRetryInProgress) { logger.info( `[${run.teamName}] Process exited (code ${code ?? '?'}) during auth-failure respawn — ignoring` ); return; } if ( (typeof run.stdoutParserCarry === 'string' ? run.stdoutParserCarry.trim() : '') && !run.stdoutParserCarryIsCompleteJson && run.stdoutParserCarryLooksLikeClaudeJson ) { logger.warn( `[${run.teamName}] Process closed with incomplete stream-json stdout carry`, this.buildStdoutCarryDiagnostic(run) ); } this.flushStdoutParserCarry(run); run.processClosed = true; if ( this.isProvisioningRunFailed(run) || run.cancelRequested || run.processKilled || run.authRetryInProgress ) { return; } // IMPORTANT: stopStallWatchdog MUST be AFTER authRetryInProgress guard above! // During respawn, the old process exit fires but run.stallCheckHandle already // points to the NEW process's watchdog. Stopping it here would kill the wrong timer. // The authRetryInProgress guard returns early, keeping the new watchdog alive. this.stopStallWatchdog(run); // === Process exited AFTER provisioning completed === // This means the team went offline (crash, kill, or natural exit). if (run.provisioningComplete) { const message = code === 0 ? 'Team process exited normally' : `Team process exited unexpectedly (code ${code ?? 'unknown'})`; logger.info(`[${run.teamName}] ${message}`); const progress = updateProgress(run, 'disconnected', message, { cliLogsTail: extractCliLogsFromRun(run), }); run.onProgress(progress); this.cleanupRun(run); return; } // === Process exited DURING provisioning === // Try to verify if files were created before the process died. updateProgress(run, 'verifying', 'Process exited — verifying provisioning results'); run.onProgress(run.progress); if (run.cancelRequested) { return; } const configProbe = await this.waitForValidConfig(run); if (run.cancelRequested) { return; } if (configProbe.ok && configProbe.location === 'default') { const configuredTeamsBasePath = getTeamsBasePath(); const progress = updateProgress(run, 'failed', 'Provisioning failed validation', { error: `TeamCreate produced config.json under a different Claude root (${configProbe.configPath}). ` + `This app is configured to read teams from ${configuredTeamsBasePath}. ` + 'Align the app Claude root setting with the CLI, then retry.', cliLogsTail: extractCliLogsFromRun(run), }); run.onProgress(progress); this.cleanupRun(run); return; } const visibleInList = configProbe.ok && configProbe.location === 'configured' ? await this.waitForTeamInList(run.teamName, run) : false; if (run.cancelRequested) { return; } if (configProbe.ok && visibleInList) { // Files exist but process died — provisioned but not alive. const warnings: string[] = [ `CLI process exited (code ${code ?? 'unknown'}) — team provisioned but not alive`, ]; const missingInboxes = await this.waitForMissingInboxes(run); if (run.cancelRequested) { return; } if (missingInboxes.length > 0) { warnings.push('Some inboxes not created yet'); } if (!run.isLaunch) { await this.persistMembersMeta(run.teamName, run.request); } // Mark as disconnected since the process is dead const progress = updateProgress( run, 'disconnected', 'Team provisioned but process is no longer alive', { warnings, cliLogsTail: extractCliLogsFromRun(run), } ); await this.finalizeIncompleteLaunchStateBeforeCleanup(run, warnings[0]); run.onProgress(progress); this.cleanupRun(run); return; } if (code === 0) { const configuredConfigPath = path.join(getTeamsBasePath(), run.teamName, 'config.json'); const defaultTeamsBasePath = path.join(getAutoDetectedClaudeBasePath(), 'teams'); const defaultConfigPath = path.join(defaultTeamsBasePath, run.teamName, 'config.json'); const combinedLogs = buildCombinedLogs(run.stdoutBuffer, run.stderrBuffer); const cleanupHint = logsSuggestShutdownOrCleanup(combinedLogs) ? ' CLI output suggests the team was shut down / cleaned up, so no persisted config was left on disk.' : ''; const errorMessage = !configProbe.ok ? `No valid config.json found at ${configuredConfigPath}${ path.resolve(defaultTeamsBasePath) === path.resolve(getTeamsBasePath()) ? '' : ` (also checked ${defaultConfigPath})` } within ${Math.round(VERIFY_TIMEOUT_MS / 1000)}s.${cleanupHint}` : 'Team did not appear in team:list after provisioning'; const progress = updateProgress(run, 'failed', 'Provisioning failed validation', { error: errorMessage, cliLogsTail: extractCliLogsFromRun(run), }); run.onProgress(progress); this.cleanupRun(run); return; } const failurePresentation = buildCliExitFailurePresentation(run, code, { cliCommandLabel: getConfiguredCliCommandLabel(), }); const runtimeFailureLabel = getRunRuntimeFailureLabel(run); const progress = updateProgress( run, 'failed', failurePresentation.message ?? `${runtimeFailureLabel} exited with an error`, { error: failurePresentation.error, cliLogsTail: extractCliLogsFromRun(run), } ); run.onProgress(progress); this.cleanupRun(run); logger.warn( `Provisioning failed for ${run.teamName}: ${progress.error ?? failurePresentation.error}` ); } private async waitForValidConfig( run: ProvisioningRun, timeoutMs: number = VERIFY_TIMEOUT_MS ): Promise { const probes = run.teamsBasePathsToProbe.map((probe) => ({ ...probe, configPath: path.join(probe.basePath, run.teamName, 'config.json'), })); const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { if (run.cancelRequested) { return { ok: false }; } for (const probe of probes) { try { const raw = await tryReadRegularFileUtf8(probe.configPath, { timeoutMs: TEAM_JSON_READ_TIMEOUT_MS, maxBytes: TEAM_CONFIG_MAX_BYTES, }); if (!raw) { continue; } const parsed = JSON.parse(raw) as unknown; if (parsed && typeof parsed === 'object') { const candidate = parsed as { name?: unknown }; if (typeof candidate.name === 'string' && candidate.name.trim().length > 0) { return { ok: true, location: probe.location, configPath: probe.configPath }; } } } catch { // Best-effort polling until deadline. } } await sleep(VERIFY_POLL_MS); } return { ok: false }; } private async waitForTeamInList(teamName: string, run?: ProvisioningRun): Promise { const deadline = Date.now() + VERIFY_TIMEOUT_MS; while (Date.now() < deadline) { if (run?.cancelRequested) { return false; } try { const teams = await this.configReader.listTeams(); if (teams.some((team) => team.teamName === teamName)) { return true; } } catch { // Keep polling until deadline. } await sleep(VERIFY_POLL_MS); } return false; } private async waitForMissingInboxes(run: ProvisioningRun): Promise { if (run.expectedMembers.length === 0) { return []; } const inboxDir = path.join(getTeamsBasePath(), run.teamName, 'inboxes'); const deadline = Date.now() + VERIFY_TIMEOUT_MS; let missing = new Set(run.expectedMembers); while (Date.now() < deadline && missing.size > 0) { if (run.cancelRequested || run.progress.state === 'cancelled') { return Array.from(missing); } const nextMissing = new Set(); for (const member of missing) { const inboxPath = path.join(inboxDir, `${member}.json`); if (!(await this.pathExists(inboxPath))) { nextMissing.add(member); } } missing = nextMissing; if (missing.size === 0) { break; } await sleep(VERIFY_POLL_MS); } return Array.from(missing); } private async tryCompleteAfterTimeout(run: ProvisioningRun): Promise { if (run.cancelRequested) { return false; } const configProbe = await this.waitForValidConfig(run); if (!configProbe.ok || configProbe.location !== 'configured') { return false; } const visibleInList = await this.waitForTeamInList(run.teamName); if (!visibleInList) { return false; } const warnings: string[] = [ 'CLI timed out after config was created — team provisioned but process killed', ]; const missingInboxes = await this.waitForMissingInboxes(run); if (run.cancelRequested) { return false; } if (missingInboxes.length > 0) { warnings.push('Some inboxes not created yet'); } if (!run.isLaunch) { await this.persistMembersMeta(run.teamName, run.request); } // Persist team color even on timeout path await this.updateConfigPostLaunch( run.teamName, run.request.cwd, run.detectedSessionId, run.request.color, { providerId: run.request.providerId, model: run.request.model, effort: run.request.effort, members: run.allEffectiveMembers, } ); await this.refreshMemberSpawnStatusesFromLeadInbox(run); await this.maybeAuditMemberSpawnStatuses(run, { force: true }); await this.finalizeMissingRegisteredMembersAsFailed(run); await this.persistLaunchStateSnapshot(run, 'finished'); // Process was killed by timeout — mark as disconnected, not ready const progress = updateProgress(run, 'disconnected', 'Team provisioned but process timed out', { warnings, }); run.onProgress(progress); this.cleanupRun(run); return true; } private async pathExists(filePath: string): Promise { try { await fs.promises.access(filePath, fs.constants.F_OK); return true; } catch { return false; } } private async buildProvisioningEnv( providerId: TeamProviderId | undefined = 'anthropic', providerBackendId?: string | null, options?: { includeCodexTeammateAuth?: boolean; teamRuntimeAuth?: TeamRuntimeAuthContext; } ): Promise { const shellEnv = await resolveInteractiveShellEnvBestEffort({ source: 'team-provisioning', timeoutMs: 1_500, fallbackEnv: process.env, background: false, }); // getHomeDir() uses Electron's app.getPath('home') which handles Unicode // correctly on Windows. Prefer it over process.env which may be garbled. const electronHome = getHomeDir(); const isWindows = process.platform === 'win32'; const home = shellEnv.HOME?.trim() || electronHome; let osUsername = ''; try { osUsername = os.userInfo().username; } catch { // os.userInfo() can throw SystemError in restricted environments (no passwd entry, Docker, etc.) } const user = shellEnv.USER?.trim() || process.env.USER?.trim() || process.env.USERNAME?.trim() || osUsername || 'unknown'; // Shell: on Windows there is no SHELL env var; use COMSPEC (cmd.exe / powershell). // On Unix, prefer the user's login shell from env or fall back to /bin/zsh. const shell = isWindows ? (process.env.COMSPEC ?? 'powershell.exe') : shellEnv.SHELL?.trim() || process.env.SHELL?.trim() || '/bin/zsh'; const env: NodeJS.ProcessEnv = { ...process.env, ...shellEnv, HOME: home, USERPROFILE: home, USER: user, LOGNAME: shellEnv.LOGNAME?.trim() || process.env.LOGNAME?.trim() || user, TERM: shellEnv.TERM?.trim() || process.env.TERM?.trim() || 'xterm-256color', // Only set CLAUDE_CONFIG_DIR when the user configured a custom path. // Setting it to the default ~/.claude changes the macOS Keychain namespace // for OAuth credential lookup, causing auth failures. (See issue #27) ...(getClaudeBasePath() !== getAutoDetectedClaudeBasePath() ? { CLAUDE_CONFIG_DIR: getClaudeBasePath() } : {}), CLAUDE_CODE_EXPERIMENTAL_AGENT_TEAMS: '1', }; normalizeTeamRuntimeNodeEnv(env); const resolvedProviderId = resolveTeamProviderId(providerId); const providerEnvResult = await buildProviderAwareCliEnv({ providerId, providerBackendId, shellEnv, env, }); const providerConnectionIssue = providerEnvResult.connectionIssues[resolvedProviderId]; const providerEnv = providerEnvResult.env; const writableEnvResult = await prepareAgentChildProcessWritableEnv(providerEnv, { home }); if (writableEnvResult.warning) { logger.warn(`[TeamProvisioningService] ${writableEnvResult.warning}`); } if (options?.includeCodexTeammateAuth && resolvedProviderId !== 'codex') { await this.providerConnectionService.augmentConfiguredConnectionEnv( providerEnv, 'codex', getConfiguredRuntimeBackend('codex') ); } Object.assign(providerEnv, await this.buildRuntimeTurnSettledEnvironment(resolvedProviderId)); const controlApiBaseUrl = await this.resolveControlApiBaseUrl(); if (controlApiBaseUrl) { providerEnv.CLAUDE_TEAM_CONTROL_URL = controlApiBaseUrl; } // SHELL is a Unix concept — only set it on non-Windows platforms. if (!isWindows) { providerEnv.SHELL = shell; } // XDG directories are a freedesktop.org (Linux/macOS) convention. // On Windows, these are unused by most tools and can cause confusion. if (!isWindows) { const xdgConfigHome = shellEnv.XDG_CONFIG_HOME?.trim() || process.env.XDG_CONFIG_HOME?.trim() || `${home}/.config`; const xdgStateHome = shellEnv.XDG_STATE_HOME?.trim() || process.env.XDG_STATE_HOME?.trim() || `${home}/.local/state`; providerEnv.XDG_CONFIG_HOME = xdgConfigHome; providerEnv.XDG_STATE_HOME = xdgStateHome; } if (providerConnectionIssue) { return { env: providerEnv, authSource: 'configured_api_key_missing', geminiRuntimeAuth: null, providerArgs: providerEnvResult.providerArgs, warning: providerConnectionIssue, }; } if (resolvedProviderId === 'codex') { return { env: providerEnv, authSource: 'codex_runtime', geminiRuntimeAuth: null, providerArgs: providerEnvResult.providerArgs, }; } if (resolvedProviderId === 'gemini') { return { env: providerEnv, authSource: 'gemini_runtime', geminiRuntimeAuth: await resolveGeminiRuntimeAuth(providerEnv), providerArgs: providerEnvResult.providerArgs, }; } const teamRuntimeAuth = options?.teamRuntimeAuth; const helperAllowed = resolvedProviderId === 'anthropic' && teamRuntimeAuth?.allowAnthropicApiKeyHelper === true && typeof teamRuntimeAuth.teamName === 'string' && teamRuntimeAuth.teamName.trim().length > 0 && typeof teamRuntimeAuth.authMaterialId === 'string' && teamRuntimeAuth.authMaterialId.trim().length > 0 && !isWindows && process.env[DISABLE_ANTHROPIC_TEAM_API_KEY_HELPER_ENV] !== '1'; if (helperAllowed) { const apiKey = await this.providerConnectionService.getConfiguredAnthropicApiKeyForTeamRuntime( providerEnv ); if (apiKey) { const helper = await materializeAnthropicTeamApiKeyHelper({ teamName: teamRuntimeAuth.teamName!, authMaterialId: teamRuntimeAuth.authMaterialId!, apiKey, baseClaudeDir: getClaudeBasePath(), }); try { await verifyAnthropicTeamApiKeyHelperMaterial({ helperPath: helper.helperPath, expectedApiKey: apiKey, }); } catch (error) { await cleanupAnthropicTeamApiKeyHelperMaterial({ directory: helper.directory }); throw error; } for (const key of ANTHROPIC_HELPER_MODE_COMPETING_AUTH_ENV_KEYS) { delete providerEnv[key]; } Object.assign(providerEnv, helper.envPatch); return { env: providerEnv, authSource: 'anthropic_api_key_helper', geminiRuntimeAuth: null, providerArgs: [...(providerEnvResult.providerArgs ?? []), ...helper.settingsArgs], anthropicApiKeyHelper: helper, }; } } // 1. Explicit ANTHROPIC_API_KEY - works with `-p` mode directly if ( typeof providerEnv.ANTHROPIC_API_KEY === 'string' && providerEnv.ANTHROPIC_API_KEY.trim().length > 0 ) { return { env: providerEnv, authSource: 'anthropic_api_key', geminiRuntimeAuth: null, providerArgs: providerEnvResult.providerArgs, }; } // 2. Anthropic-compatible runtimes (Ollama/LM Studio/gateways) expect a bearer // token and often require ANTHROPIC_API_KEY to stay empty. if (hasAnthropicCompatibleAuthTokenEnv(providerEnv)) { return { env: providerEnv, authSource: 'anthropic_auth_token', geminiRuntimeAuth: null, providerArgs: providerEnvResult.providerArgs, }; } // 3. Proxy token (ANTHROPIC_AUTH_TOKEN) - `-p` mode does NOT read this var, // so we must copy it into ANTHROPIC_API_KEY for it to work. if ( typeof providerEnv.ANTHROPIC_AUTH_TOKEN === 'string' && providerEnv.ANTHROPIC_AUTH_TOKEN.trim().length > 0 ) { providerEnv.ANTHROPIC_API_KEY = providerEnv.ANTHROPIC_AUTH_TOKEN; return { env: providerEnv, authSource: 'anthropic_auth_token', geminiRuntimeAuth: null, providerArgs: providerEnvResult.providerArgs, }; } // 4. No explicit API key - let the CLI handle its own OAuth auth. // Claude CLI reads credentials from its own storage and refreshes // tokens in-memory. Injecting CLAUDE_CODE_OAUTH_TOKEN from the // credentials file causes 401 errors because the stored token is // often stale (CLI refreshes in-memory but rarely writes back). return { env: providerEnv, authSource: 'none', geminiRuntimeAuth: null, providerArgs: providerEnvResult.providerArgs, }; } private async buildCrossProviderMemberArgs( primaryProviderId: TeamProviderId, memberSpecs: TeamCreateRequest['members'], options?: { teamRuntimeAuth?: TeamRuntimeAuthContext } ): Promise { const crossProviderIds = new Set(); for (const member of memberSpecs) { const memberId = resolveTeamProviderId( normalizeTeamMemberProviderId(member.providerId) ?? primaryProviderId ); if (memberId !== primaryProviderId) { crossProviderIds.add(memberId); } } const args: string[] = []; const providerArgsByProvider = new Map(); const envPatch: NodeJS.ProcessEnv = {}; let usesAnthropicApiKeyHelper = false; for (const providerId of crossProviderIds) { let env: ProvisioningEnvResolution; try { env = await this.buildProvisioningEnv(providerId, undefined, { teamRuntimeAuth: options?.teamRuntimeAuth, }); } catch (error) { console.error( `[TeamProvisioningService] Failed to build cross-provider args for provider "${providerId}"`, error ); // Best-effort: don't block launch if cross-provider env resolution fails // before the provider can report a concrete auth/readiness issue. continue; } if (env.warning) { throw new Error(`${getTeamProviderLabel(providerId)}: ${env.warning}`); } args.push(...(await this.buildRuntimeTurnSettledHookSettingsArgs(providerId))); const providerArgs = env.providerArgs ?? []; providerArgsByProvider.set(providerId, providerArgs); if (providerId === 'codex') { Object.assign(envPatch, buildCodexCrossProviderSafeEnvPatch(env.env)); } if (env.anthropicApiKeyHelper) { usesAnthropicApiKeyHelper = true; Object.assign(envPatch, env.anthropicApiKeyHelper.envPatch); } else if ( providerId === 'anthropic' && isAnthropicDirectCredentialAuthSource(env.authSource) ) { Object.assign( envPatch, buildAnthropicCrossProviderDirectAuthEnvPatch(env.env, env.authSource) ); } const flattenedArgs = providerId === 'anthropic' && env.anthropicApiKeyHelper ? filterOutSettingsPathArgs(providerArgs, env.anthropicApiKeyHelper.settingsPath) : providerArgs; if (flattenedArgs.length > 0) { args.push(...flattenedArgs); } } return { args, providerArgsByProvider, envPatch, usesAnthropicApiKeyHelper }; } private async resolveControlApiBaseUrl(): Promise { if (!this.controlApiBaseUrlResolver) { return null; } try { const baseUrl = await this.controlApiBaseUrlResolver(); if (!baseUrl) { throw new Error('Team control API resolver returned no base URL after startup.'); } process.env.CLAUDE_TEAM_CONTROL_URL = baseUrl; return baseUrl; } catch (error) { const message = error instanceof Error ? error.message : String(error); logger.error(`Failed to resolve team control API base URL: ${message}`); throw new Error( `Team control API failed to start or publish its base URL. Team runtime commands require the desktop Control API. ${message}` ); } } /** * Immediately update projectPath in config.json at launch start, before CLI spawn. * Ensures TeamDetailView shows the correct project path even if provisioning * is interrupted. On failure, restorePrelaunchConfig() reverts to the backup. */ private async updateConfigProjectPath(teamName: string, cwd: string): Promise { const configPath = path.join(getTeamsBasePath(), teamName, 'config.json'); try { const raw = await tryReadRegularFileUtf8(configPath, { timeoutMs: TEAM_JSON_READ_TIMEOUT_MS, maxBytes: TEAM_CONFIG_MAX_BYTES, }); if (!raw) { throw new Error('config.json unreadable'); } const config = JSON.parse(raw) as Record; config.projectPath = cwd; const pathHistory = Array.isArray(config.projectPathHistory) ? (config.projectPathHistory as string[]).filter((p) => typeof p === 'string' && p !== cwd) : []; pathHistory.push(cwd); config.projectPathHistory = pathHistory.slice(-500); await atomicWriteAsync(configPath, JSON.stringify(config, null, 2)); TeamConfigReader.invalidateTeam(teamName); logger.info(`[${teamName}] Updated config.projectPath immediately: ${cwd}`); } catch (error) { // Non-fatal: updateConfigPostLaunch will update it later if provisioning succeeds. logger.warn( `[${teamName}] Failed to update projectPath early: ${error instanceof Error ? error.message : String(error)}` ); } } private applyEffectiveLaunchStateToConfig( teamName: string, config: Record, launchState?: { providerId?: TeamProviderId; model?: string; effort?: TeamCreateRequest['effort']; members?: TeamCreateRequest['members']; } ): void { if (!launchState || !Array.isArray(config.members)) { return; } const effectiveLeadProviderId = normalizeTeamMemberProviderId(launchState.providerId) ?? 'anthropic'; const effectiveLeadModel = launchState.model?.trim() || undefined; const effectiveLeadEffort = isTeamEffortLevel(launchState.effort) ? launchState.effort : undefined; const membersByName = new Map( (launchState.members ?? []).map((member) => [member.name.toLowerCase(), member] as const) ); const nextMembers = (config.members as Record[]).map((member) => { if (!member || typeof member !== 'object') { return member; } const rawName = typeof member.name === 'string' ? member.name.trim() : ''; const nextMember = { ...member }; const assignRuntimeState = (state: { providerId?: TeamProviderId; model?: string; effort?: TeamCreateRequest['effort']; }): void => { const providerId = normalizeTeamMemberProviderId(state.providerId); if (providerId) { nextMember.provider = providerId; nextMember.providerId = providerId; } else { delete nextMember.provider; delete nextMember.providerId; } const model = state.model?.trim() || undefined; if (model) { nextMember.model = model; } else { delete nextMember.model; } const effort = isTeamEffortLevel(state.effort) ? state.effort : undefined; if (effort) { nextMember.effort = effort; } else { delete nextMember.effort; } }; if (isLeadMember(nextMember) || rawName.toLowerCase() === 'team-lead') { assignRuntimeState({ providerId: effectiveLeadProviderId, model: effectiveLeadModel, effort: effectiveLeadEffort, }); return nextMember; } const effectiveMember = membersByName.get(rawName.toLowerCase()); if (!effectiveMember) { return nextMember; } assignRuntimeState({ providerId: effectiveMember.providerId, model: effectiveMember.model, effort: effectiveMember.effort, }); return nextMember; }); const existingNames = new Set( nextMembers .map((member) => (typeof member.name === 'string' ? member.name.trim().toLowerCase() : '')) .filter(Boolean) ); for (const member of launchState.members ?? []) { const name = member.name?.trim(); if (!name || existingNames.has(name.toLowerCase())) { continue; } const providerId = normalizeTeamMemberProviderId(member.providerId); if (providerId !== 'opencode') { continue; } nextMembers.push(this.buildOpenCodeConfigMemberFromLaunchMember(teamName, member)); existingNames.add(name.toLowerCase()); } config.members = nextMembers; } private buildOpenCodeConfigMemberFromLaunchMember( teamName: string, member: TeamCreateRequest['members'][number] ): Record { const name = member.name.trim(); const configMember: Record = { name, agentId: `${name}@${teamName}`, agentType: 'general-purpose', role: member.role?.trim() || undefined, workflow: member.workflow?.trim() || undefined, isolation: member.isolation === 'worktree' ? 'worktree' : undefined, providerId: 'opencode', model: member.model?.trim() || undefined, effort: isTeamEffortLevel(member.effort) ? member.effort : undefined, mcpPolicy: normalizeTeamMemberMcpPolicy(member.mcpPolicy), cwd: member.cwd?.trim() || undefined, joinedAt: Date.now(), }; return Object.fromEntries( Object.entries(configMember).filter(([, value]) => value !== undefined) ); } /** * Single atomic read-mutate-write for post-launch config updates. * Combines session history append and projectPath update to avoid * race conditions with the CLI writing to the same file. */ private async updateConfigPostLaunch( teamName: string, projectPath: string, detectedSessionId: string | null, color?: string, launchState?: { providerId?: TeamProviderId; model?: string; effort?: TeamCreateRequest['effort']; members?: TeamCreateRequest['members']; } ): Promise { const MAX_SESSION_HISTORY = 5000; const MAX_PROJECT_PATH_HISTORY = 500; const configPath = path.join(getTeamsBasePath(), teamName, 'config.json'); try { const raw = await tryReadRegularFileUtf8(configPath, { timeoutMs: TEAM_JSON_READ_TIMEOUT_MS, maxBytes: TEAM_CONFIG_MAX_BYTES, }); if (!raw) { throw new Error('config.json unreadable'); } const config = JSON.parse(raw) as Record; const sessionHistory = Array.isArray(config.sessionHistory) ? (config.sessionHistory as string[]) : []; // Preserve old leadSessionId in history before overwriting const oldLeadSessionId = config.leadSessionId; if (typeof oldLeadSessionId === 'string' && oldLeadSessionId.trim().length > 0) { if (!sessionHistory.includes(oldLeadSessionId)) { sessionHistory.push(oldLeadSessionId); } } // Update leadSessionId to the new session detected from stream-json let newSessionId = detectedSessionId; // Fallback: if stream-json didn't provide session_id, scan project dir for newest JSONL if (!newSessionId && projectPath.trim()) { const scannedId = await this.scanForNewestSession(projectPath, sessionHistory); if (scannedId) { newSessionId = scannedId; logger.info(`[${teamName}] Detected new session via project dir scan: ${scannedId}`); } } if (newSessionId) { config.leadSessionId = newSessionId; if (!sessionHistory.includes(newSessionId)) { sessionHistory.push(newSessionId); } logger.info(`[${teamName}] Updated leadSessionId: ${newSessionId}`); } if (sessionHistory.length > MAX_SESSION_HISTORY) { config.sessionHistory = sessionHistory.slice(-MAX_SESSION_HISTORY); } else { config.sessionHistory = sessionHistory; } // Save current language setting const langCode = ConfigManager.getInstance().getConfig().general.agentLanguage || 'system'; config.language = langCode; // Persist team color chosen by the user during creation if (color && color.trim().length > 0) { config.color = color.trim(); } // Ensure projectPath if (projectPath.trim()) { config.projectPath = projectPath; const pathHistory = Array.isArray(config.projectPathHistory) ? (config.projectPathHistory as string[]).filter( (p) => typeof p === 'string' && p !== projectPath ) : []; pathHistory.push(projectPath); config.projectPathHistory = pathHistory.length > MAX_PROJECT_PATH_HISTORY ? pathHistory.slice(-MAX_PROJECT_PATH_HISTORY) : pathHistory; } this.applyEffectiveLaunchStateToConfig(teamName, config, launchState); await atomicWriteAsync(configPath, JSON.stringify(config, null, 2)); TeamConfigReader.invalidateTeam(teamName); } catch (error) { logger.warn( `[${teamName}] Failed to update config post-launch: ${ error instanceof Error ? error.message : String(error) }` ); } } private async cleanupCliAutoSuffixedMembers(teamName: string): Promise { const configPath = path.join(getTeamsBasePath(), teamName, 'config.json'); const removedFromConfig: string[] = []; try { const raw = await tryReadRegularFileUtf8(configPath, { timeoutMs: TEAM_JSON_READ_TIMEOUT_MS, maxBytes: TEAM_CONFIG_MAX_BYTES, }); if (raw) { const parsed = JSON.parse(raw) as Record; const membersRaw = Array.isArray(parsed.members) ? (parsed.members as Record[]) : []; if (membersRaw.length > 0) { const teammateNames = membersRaw .map((m) => (typeof m.name === 'string' ? m.name.trim() : '')) .filter( (n) => n.length > 0 && n.toLowerCase() !== 'team-lead' && n.toLowerCase() !== 'user' ); const keepName = createCliAutoSuffixNameGuard(teammateNames); const nextMembers: Record[] = []; for (const m of membersRaw) { const name = typeof m.name === 'string' ? m.name.trim() : ''; if (!name) continue; if (isLeadMember(m) || name === 'user') { nextMembers.push(m); continue; } if (!keepName(name)) { removedFromConfig.push(name); continue; } nextMembers.push(m); } if (removedFromConfig.length > 0) { parsed.members = nextMembers; await atomicWriteAsync(configPath, JSON.stringify(parsed, null, 2)); TeamConfigReader.invalidateTeam(teamName); logger.warn( `[${teamName}] Removed CLI auto-suffixed members from config.json: ${removedFromConfig.join(', ')}` ); } } } } catch { // best-effort } let activeNamesForInboxCleanup = new Set(); try { const metaMembers = await this.membersMetaStore.getMembers(teamName); if (metaMembers.length > 0) { const activeNames = metaMembers .filter((m) => !m.removedAt) .map((m) => m.name.trim()) .filter( (n) => n.length > 0 && n.toLowerCase() !== 'team-lead' && n.toLowerCase() !== 'user' ); const keepName = createCliAutoSuffixNameGuard(activeNames); const removedFromMeta: string[] = []; const nextMeta = metaMembers.filter((m) => { const name = m.name?.trim() ?? ''; if (!name) return false; const lower = name.toLowerCase(); if (lower === 'user' || isLeadMember(m)) return true; if (!m.removedAt && !keepName(name)) { removedFromMeta.push(name); return false; } return true; }); if (removedFromMeta.length > 0) { await this.membersMetaStore.writeMembers(teamName, nextMeta); logger.warn( `[${teamName}] Removed CLI auto-suffixed members from members.meta.json: ${removedFromMeta.join(', ')}` ); } activeNamesForInboxCleanup = new Set( nextMeta .filter((m) => !m.removedAt) .map((m) => m.name.trim()) .filter( (n) => n.length > 0 && n.toLowerCase() !== 'team-lead' && n.toLowerCase() !== 'user' ) ); } } catch { // best-effort } // Also attempt inbox cleanup (merge alice-2.json into alice.json). if (activeNamesForInboxCleanup.size > 0) { try { await this.mergeAndRemoveDuplicateInboxes(teamName, activeNamesForInboxCleanup); } catch { // best-effort } } } /** * Fallback: scan the project directory for the newest JSONL file * that isn't already in sessionHistory. Returns the session ID or null. */ private async scanForNewestSession( projectPath: string, knownSessions: string[] ): Promise { try { const projectId = encodePath(projectPath); const baseDir = extractBaseDir(projectId); const projectDir = path.join(getProjectsBasePath(), baseDir); const entries = await fs.promises.readdir(projectDir); const knownSet = new Set(knownSessions); let newest: { id: string; mtime: number } | null = null; for (const entry of entries) { if (!entry.endsWith('.jsonl')) continue; const sessionId = entry.replace('.jsonl', ''); if (knownSet.has(sessionId)) continue; const filePath = path.join(projectDir, entry); const stat = await fs.promises.stat(filePath); if (!newest || stat.mtimeMs > newest.mtime) { newest = { id: sessionId, mtime: stat.mtimeMs }; } } return newest?.id ?? null; } catch { return null; } } private async assertConfigLeadOnlyForLaunch(teamName: string): Promise { const configPath = path.join(getTeamsBasePath(), teamName, 'config.json'); const raw = await tryReadRegularFileUtf8(configPath, { timeoutMs: TEAM_JSON_READ_TIMEOUT_MS, maxBytes: TEAM_CONFIG_MAX_BYTES, }); if (!raw) { throw new Error('config.json unreadable'); } let parsed: unknown; try { parsed = JSON.parse(raw) as unknown; } catch { throw new Error('config.json could not be parsed'); } if (!parsed || typeof parsed !== 'object') { throw new Error('config.json has invalid shape'); } const config = parsed as Record; const members = Array.isArray(config.members) ? (config.members as Record[]) : []; if (members.length === 0) return; for (const member of members) { const name = typeof member.name === 'string' ? member.name.trim() : ''; if (!name) continue; const lower = name.toLowerCase(); if (isLeadMember(member) || lower === 'user') continue; const leadAgentId = config.leadAgentId; if ( typeof leadAgentId === 'string' && typeof member.agentId === 'string' && member.agentId === leadAgentId ) { continue; } throw new Error( `Refusing to launch: config.json still contains teammates (e.g. "${name}"), which can trigger CLI auto-suffixes like "${name}-2".` ); } } private async normalizeTeamConfigForLaunch(teamName: string, configRaw: string): Promise { const configPath = path.join(getTeamsBasePath(), teamName, 'config.json'); const backupPath = `${configPath}.prelaunch.bak`; let parsed: unknown; try { parsed = JSON.parse(configRaw) as unknown; } catch { return; } if (!parsed || typeof parsed !== 'object') { return; } const config = parsed as Record; const members = Array.isArray(config.members) ? (config.members as Record[]) : []; if (members.length === 0) { return; } // Keep only the lead entry. const leadMembers = members.filter((member) => { const agentType = member.agentType; if (typeof agentType === 'string' && isLeadAgentType(agentType)) { return true; } // Also check by name (CLI may set agentType to "general-purpose" for leads) const name = typeof member.name === 'string' ? member.name.trim().toLowerCase() : ''; if (name === 'team-lead') return true; const leadAgentId = config.leadAgentId; return ( typeof leadAgentId === 'string' && typeof member.agentId === 'string' && member.agentId === leadAgentId ); }); // If already lead-only, no-op. if (leadMembers.length === members.length) { return; } // Try to determine base teammate names for inbox cleanup (prefer meta). const baseNames = new Set(); try { const metaMembers = await this.membersMetaStore.getMembers(teamName); for (const member of metaMembers) { const name = member.name.trim(); const lower = name.toLowerCase(); if (name.length > 0 && !member.removedAt && lower !== 'team-lead' && lower !== 'user') { baseNames.add(name); } } } catch { // ignore } if (baseNames.size === 0) { const allConfigNames = new Set(); for (const member of members) { const name = typeof member.name === 'string' ? member.name.trim() : ''; const agentType = typeof member.agentType === 'string' ? member.agentType : ''; if ( name && agentType && !isLeadAgentType(agentType) && name !== 'team-lead' && name !== 'user' ) { allConfigNames.add(name); } } const allConfigNamesLower = new Set(Array.from(allConfigNames).map((n) => n.toLowerCase())); for (const name of allConfigNames) { const match = /^(.+)-(\d+)$/.exec(name); if (!match?.[1] || !match[2]) { baseNames.add(name); continue; } const suffix = Number(match[2]); // Only exclude CLI-suffixed names (alice-2) when the base name (alice) also exists // (and only for -2+ to avoid excluding legitimate "dev-1"-style names). if (!Number.isFinite(suffix) || suffix < 2) { baseNames.add(name); continue; } if (!allConfigNamesLower.has(match[1].toLowerCase())) { baseNames.add(name); } } } // Backup current config on disk for crash recovery / debugging. try { await atomicWriteAsync(backupPath, configRaw); } catch (error) { logger.warn( `[${teamName}] Failed to write config prelaunch backup: ${ error instanceof Error ? error.message : String(error) }` ); } // Write normalized config atomically. config.members = leadMembers; try { await atomicWriteAsync(configPath, JSON.stringify(config, null, 2)); TeamConfigReader.invalidateTeam(teamName); logger.info( `[${teamName}] Normalized config.json for launch: kept ${leadMembers.length} lead member(s)` ); } catch (error) { logger.warn( `[${teamName}] Failed to normalize config.json for launch: ${ error instanceof Error ? error.message : String(error) }` ); return; } // Best-effort: merge and remove suffixed inboxes like alice-2.json to avoid UI duplicates. await this.mergeAndRemoveDuplicateInboxes(teamName, baseNames); } /** * Restore config.json from prelaunch backup if launch fails after normalization. */ private async restorePrelaunchConfig(teamName: string): Promise { const configPath = path.join(getTeamsBasePath(), teamName, 'config.json'); const backupPath = `${configPath}.prelaunch.bak`; try { const backupRaw = await tryReadRegularFileUtf8(backupPath, { timeoutMs: TEAM_JSON_READ_TIMEOUT_MS, maxBytes: TEAM_CONFIG_MAX_BYTES, }); if (!backupRaw) { return; } await atomicWriteAsync(configPath, backupRaw); TeamConfigReader.invalidateTeam(teamName); logger.info(`[${teamName}] Restored config.json from prelaunch backup after launch failure`); } catch { logger.debug(`[${teamName}] No prelaunch backup to restore (or read failed)`); } } /** * Remove the prelaunch backup file after a successful launch. */ async cleanupPrelaunchBackup(teamName: string): Promise { const configPath = path.join(getTeamsBasePath(), teamName, 'config.json'); const backupPath = `${configPath}.prelaunch.bak`; try { await fs.promises.unlink(backupPath); } catch { // Backup may not exist — that's fine } } private async mergeAndRemoveDuplicateInboxes( teamName: string, baseNames: Set ): Promise { if (baseNames.size === 0) return; const inboxDir = path.join(getTeamsBasePath(), teamName, 'inboxes'); let entries: string[]; try { entries = await fs.promises.readdir(inboxDir); } catch { return; } const existing = new Set(entries.filter((e) => e.endsWith('.json') && !e.startsWith('.'))); for (const baseName of baseNames) { const canonicalFile = `${baseName}.json`; if (!existing.has(canonicalFile)) { continue; } const duplicates = Array.from(existing) .filter((file) => file.startsWith(`${baseName}-`) && file.endsWith('.json')) .filter((file) => /-\d+\.json$/.test(file)); if (duplicates.length === 0) { continue; } const canonicalPath = path.join(inboxDir, canonicalFile); let canonicalRaw: string; try { const raw = await tryReadRegularFileUtf8(canonicalPath, { timeoutMs: TEAM_JSON_READ_TIMEOUT_MS, maxBytes: TEAM_INBOX_MAX_BYTES, }); if (!raw) { continue; } canonicalRaw = raw; } catch { // If cannot read, skip cleanup for this base. continue; } let canonicalParsed: unknown; try { canonicalParsed = JSON.parse(canonicalRaw) as unknown; } catch { canonicalParsed = []; } const canonicalList = Array.isArray(canonicalParsed) ? (canonicalParsed as unknown[]) : []; const merged = [...canonicalList]; for (const dupFile of duplicates) { const dupPath = path.join(inboxDir, dupFile); let dupRaw: string; try { const raw = await tryReadRegularFileUtf8(dupPath, { timeoutMs: TEAM_JSON_READ_TIMEOUT_MS, maxBytes: TEAM_INBOX_MAX_BYTES, }); if (!raw) { continue; } dupRaw = raw; } catch { continue; } let dupParsed: unknown; try { dupParsed = JSON.parse(dupRaw) as unknown; } catch { dupParsed = []; } if (Array.isArray(dupParsed)) { const dupList = dupParsed as unknown[]; merged.push(...dupList); } } // Dedup by messageId when available, then sort by timestamp desc. const dedupById = new Map(); const noId: unknown[] = []; for (const item of merged) { if (!item || typeof item !== 'object') { continue; } const msg = item as { messageId?: unknown }; if (typeof msg.messageId === 'string' && msg.messageId.trim().length > 0) { dedupById.set(msg.messageId, item); } else { noId.push(item); } } const mergedDeduped = [...Array.from(dedupById.values()), ...noId]; mergedDeduped.sort((a, b) => { const at = a && typeof a === 'object' ? Date.parse((a as { timestamp?: string }).timestamp ?? '') : NaN; const bt = b && typeof b === 'object' ? Date.parse((b as { timestamp?: string }).timestamp ?? '') : NaN; const atNaN = Number.isNaN(at); const btNaN = Number.isNaN(bt); if (atNaN && btNaN) return 0; if (atNaN) return 1; if (btNaN) return -1; return bt - at; }); try { await atomicWriteAsync(canonicalPath, JSON.stringify(mergedDeduped, null, 2)); } catch { continue; } for (const dupFile of duplicates) { try { await fs.promises.unlink(path.join(inboxDir, dupFile)); existing.delete(dupFile); } catch { // Best-effort cleanup. } } } } private async persistMembersMeta(teamName: string, request: TeamCreateRequest): Promise { const teammateMembers = request.members.filter((member) => { const trimmed = member.name.trim(); const lower = trimmed.toLowerCase(); return trimmed.length > 0 && lower !== 'team-lead' && lower !== 'user'; }); if (teammateMembers.length === 0) { return; } const joinedAt = Date.now(); try { const membersToWrite = this.buildMembersMetaWritePayload( teammateMembers.map((member) => ({ ...member, joinedAt, })) ); await this.membersMetaStore.writeMembers(teamName, membersToWrite, { providerBackendId: request.providerBackendId, }); } catch (error) { logger.warn( `[${teamName}] Failed to persist members.meta.json: ${ error instanceof Error ? error.message : String(error) }` ); } } private async resolveLaunchExpectedMembers( teamName: string, configRaw: string, leadProviderId?: TeamProviderId ): Promise<{ members: TeamCreateRequest['members']; source: 'members-meta' | 'inboxes' | 'config-fallback'; warning?: string; }> { return this.resolveLaunchExpectedMembersFromCompatibility( await this.probeLaunchCompatibility(teamName, configRaw, leadProviderId) ); } private resolveLaunchExpectedMembersFromCompatibility(report: TeamLaunchCompatibilityReport): { members: TeamCreateRequest['members']; source: 'members-meta' | 'inboxes' | 'config-fallback'; warning?: string; } { if (report.level === 'unsafe') { throw new Error(report.blockers[0] ?? getMixedLaunchFallbackRecoveryError()); } return { members: report.members, source: report.rosterSource === 'members-meta' ? 'members-meta' : report.rosterSource === 'inboxes' ? 'inboxes' : 'config-fallback', ...(report.warnings.length > 0 ? { warning: report.warnings.join(' ') } : {}), }; } private async probeLaunchCompatibility( teamName: string, configRaw: string, leadProviderId?: TeamProviderId ): Promise { // Keep this probe read-only: launch-state/bootstrap-state may inform existing resume guards, // but compatibility repair must not mutate or trust stale runtime projections. await Promise.allSettled([ this.launchStateStore.read(teamName), readBootstrapLaunchSnapshot(teamName), ]); try { const metaMembers = await this.membersMetaStore.getMembers(teamName); const members = this.buildLaunchMembersFromMeta(metaMembers); if (members.length > 0) { return { level: 'ready', rosterSource: 'members-meta', members, warnings: [], blockers: [], }; } } catch (error) { logger.warn( `[${teamName}] Failed to read members.meta.json: ${ error instanceof Error ? error.message : String(error) }` ); } const configMembers = this.extractTeammateSpecsFromConfig(teamName, configRaw); try { const allInboxNames = Array.from( new Set( (await this.inboxReader.listInboxNames(teamName)) .map((name) => name.trim()) .filter((name) => name.length > 0) ) ); const inboxNameSetLower = new Set(allInboxNames.map((n) => n.toLowerCase())); const inboxNames = allInboxNames .filter((name) => name !== 'team-lead' && name !== 'user') .filter((name) => !this.isCrossTeamPseudoRecipientName(name)) .filter((name) => !this.isCrossTeamToolRecipientName(name)) .filter((name) => !this.looksLikeQualifiedExternalRecipientName(name)) .filter((name) => { const match = /^(.+)-(\d+)$/.exec(name); if (!match?.[1] || !match[2]) return true; const suffix = Number(match[2]); // Only filter CLI-suffixed names (alice-2) when the base name (alice) also exists. // Important: do NOT filter names like dev-1 (common intentional naming). Only consider -2+ as auto-suffix. if (!Number.isFinite(suffix) || suffix < 2) return true; return !inboxNameSetLower.has(match[1].toLowerCase()); }); if (inboxNames.length > 0) { const configHasOpenCodeMember = configMembers.some((member) => { const providerId = normalizeOptionalTeamProviderId(member.providerId); const model = typeof member.model === 'string' ? member.model.trim() : ''; return providerId === 'opencode' || inferTeamProviderIdFromModel(model) === 'opencode'; }); if (configHasOpenCodeMember) { return this.buildConfigLaunchCompatibilityReport( teamName, configMembers, leadProviderId, { ignoredInboxNames: true, } ); } const configMembersByName = new Map( configMembers.map((member) => [member.name.toLowerCase(), member] as const) ); const members = inboxNames.map((name) => { const configMember = configMembersByName.get(name.toLowerCase()); return { name, role: configMember?.role, workflow: configMember?.workflow, isolation: configMember?.isolation, cwd: configMember?.cwd, providerId: configMember?.providerId, model: configMember?.model, effort: configMember?.effort, mcpPolicy: configMember?.mcpPolicy, }; }); const memberOverridesUsed = members.some( (member) => member.providerId || member.model || member.effort || member.isolation ); if ( this.hasIncompleteOpenCodeLaunchCompatibilityMember(members) || this.isUnsafeMixedLaunchFallback({ leadProviderId, members, }) ) { return { level: 'unsafe', rosterSource: 'inboxes', members: [], warnings: [], blockers: [ `[${teamName}] ${getMixedLaunchFallbackRecoveryError()} Fallback source: inboxes.`, ], }; } return { level: 'ready', rosterSource: 'inboxes', members, warnings: memberOverridesUsed ? [ 'Launch roster was recovered from inboxes and merged with config.json provider/model/effort overrides. ' + 'Multimodel reconnect is best-effort in this fallback path.', ] : [], blockers: [], }; } } catch (error) { logger.warn( `[${teamName}] Failed to read inbox member names: ${ error instanceof Error ? error.message : String(error) }` ); } if (configMembers.length > 0) { return this.buildConfigLaunchCompatibilityReport(teamName, configMembers, leadProviderId); } let configParseFailed = false; try { JSON.parse(configRaw); } catch { configParseFailed = true; } return { level: 'ready', rosterSource: 'missing', members: [], warnings: configParseFailed ? [ 'Config could not be parsed during launch roster discovery. ' + 'Launch will continue without explicit teammate names.', ] : [], blockers: [], }; } private buildConfigLaunchCompatibilityReport( teamName: string, configMembers: TeamCreateRequest['members'], leadProviderId?: TeamProviderId, options: { ignoredInboxNames?: boolean } = {} ): TeamLaunchCompatibilityReport { if (this.hasIncompleteOpenCodeLaunchCompatibilityMember(configMembers)) { return { level: 'unsafe', rosterSource: 'config', members: [], warnings: [], blockers: [ `[${teamName}] ${getMixedLaunchFallbackRecoveryError()} Fallback source: config.`, ], }; } const lanePlan = this.runtimeLaneCoordinator.planProvisioningMembers({ leadProviderId, members: configMembers, hasOpenCodeRuntimeAdapter: true, }); if (this.runtimeLaneCoordinator.isMixedSideLanePlan(lanePlan)) { const sideLanesHaveExplicitProviderModels = lanePlan.sideLanes.every( (lane) => normalizeOptionalTeamProviderId(lane.member.providerId) === 'opencode' && typeof lane.member.model === 'string' && lane.member.model.trim().length > 0 ); if (!sideLanesHaveExplicitProviderModels) { return { level: 'unsafe', rosterSource: 'config', members: [], warnings: [], blockers: [ `[${teamName}] ${getMixedLaunchFallbackRecoveryError()} Fallback source: config.`, ], }; } } return { level: 'repairable', rosterSource: 'config', members: configMembers, warnings: [ options.ignoredInboxNames ? 'members.meta.json is missing; launch used complete config.json member metadata instead of inbox fallback to preserve mixed provider/model layout.' : 'members.meta.json and inboxes are empty; launch fell back to config.json members. ' + 'Run a fresh team bootstrap to persist stable member metadata.', ], blockers: [], repairAction: 'materialize-members-meta', }; } private buildLaunchMembersFromMeta(metaMembers: TeamMember[]): TeamCreateRequest['members'] { const byName = new Map(); for (const member of metaMembers) { const rawName = member.name?.trim() ?? ''; const lower = rawName.toLowerCase(); if (isLeadMember(member) || lower === 'user') { continue; } const name = rawName; if (!name) continue; if (member.removedAt) continue; const role = typeof member.role === 'string' ? member.role.trim() || undefined : undefined; const workflow = typeof member.workflow === 'string' ? member.workflow.trim() || undefined : undefined; const isolation = member.isolation === 'worktree' ? 'worktree' : undefined; const providerId = normalizeOptionalTeamProviderId(member.providerId); const model = typeof member.model === 'string' ? member.model.trim() || undefined : undefined; const effort = isTeamEffortLevel(member.effort) ? member.effort : undefined; const cwd = typeof member.cwd === 'string' ? member.cwd.trim() || undefined : undefined; const mcpPolicy = normalizeTeamMemberMcpPolicy(member.mcpPolicy); const prev = byName.get(name); if (!prev) { byName.set(name, { name, role, workflow, isolation, cwd, providerId, model, effort, mcpPolicy, }); } else { byName.set(name, { ...prev, role: prev.role || role, workflow: prev.workflow || workflow, isolation: prev.isolation || isolation, cwd: prev.cwd || cwd, providerId: prev.providerId || providerId, model: prev.model || model, effort: prev.effort || effort, mcpPolicy: prev.mcpPolicy || mcpPolicy, }); } } const allNames = Array.from(byName.keys()); const keepName = createCliAutoSuffixNameGuard(allNames); for (const name of allNames) { if (!keepName(name)) { byName.delete(name); } } return Array.from(byName.values()).sort((a, b) => a.name.localeCompare(b.name)); } private async materializeLaunchCompatibilityRepair( request: TeamLaunchRequest, report: TeamLaunchCompatibilityReport ): Promise { if (report.repairAction !== 'materialize-members-meta' || report.members.length === 0) { return; } const joinedAt = Date.now(); const membersToWrite = this.buildMembersMetaWritePayload( report.members.map((member) => ({ ...member, joinedAt, })) ); await this.membersMetaStore.writeMembers(request.teamName, membersToWrite, { providerBackendId: request.providerBackendId, }); } private isUnsafeMixedLaunchFallback(params: { leadProviderId?: TeamProviderId; members: TeamCreateRequest['members']; }): boolean { const lanePlan = this.runtimeLaneCoordinator.planProvisioningMembers({ leadProviderId: params.leadProviderId, members: params.members, hasOpenCodeRuntimeAdapter: true, }); return this.runtimeLaneCoordinator.isMixedSideLanePlan(lanePlan); } private hasIncompleteOpenCodeLaunchCompatibilityMember( members: TeamCreateRequest['members'] ): boolean { return members.some((member) => { const providerId = normalizeOptionalTeamProviderId(member.providerId); const model = typeof member.model === 'string' ? member.model.trim() : ''; const inferredProviderId = inferTeamProviderIdFromModel(model); return ( (providerId === 'opencode' && model.length === 0) || (!providerId && inferredProviderId === 'opencode') ); }); } private assertMixedLaunchFallbackSafe(params: { teamName: string; leadProviderId?: TeamProviderId; source: 'inboxes' | 'config-fallback'; members: TeamCreateRequest['members']; }): void { if ( this.isUnsafeMixedLaunchFallback({ leadProviderId: params.leadProviderId, members: params.members, }) ) { throw new Error( `[${params.teamName}] ${getMixedLaunchFallbackRecoveryError()} Fallback source: ${params.source}.` ); } } private extractTeammateSpecsFromConfig( teamName: string, configRaw: string ): TeamCreateRequest['members'] { try { const parsed = JSON.parse(configRaw) as { members?: { name?: string; role?: string; workflow?: string; isolation?: string; agentType?: string; providerId?: string; provider?: string; model?: string; effort?: string; mcpPolicy?: unknown; cwd?: string; removedAt?: unknown; }[]; }; if (!Array.isArray(parsed.members)) { return []; } const byName = new Map(); for (const member of parsed.members) { const rawName = typeof member?.name === 'string' ? member.name.trim() : ''; const lower = rawName.toLowerCase(); if (!member || isLeadMember(member) || lower === 'user') continue; const name = rawName; if (!name) continue; if (member.removedAt != null) continue; byName.set(name, { name, role: typeof member.role === 'string' ? member.role.trim() || undefined : undefined, workflow: typeof member.workflow === 'string' ? member.workflow.trim() || undefined : undefined, isolation: member.isolation === 'worktree' ? ('worktree' as const) : undefined, cwd: typeof member.cwd === 'string' ? member.cwd.trim() || undefined : undefined, providerId: normalizeTeamMemberProviderId(member.providerId ?? member.provider), model: typeof member.model === 'string' ? member.model.trim() || undefined : undefined, effort: isTeamEffortLevel(member.effort) ? member.effort : undefined, mcpPolicy: normalizeTeamMemberMcpPolicy(member.mcpPolicy), }); } // Defense: ignore CLI auto-suffixed duplicates (alice-2) when base name exists. const allNames = Array.from(byName.keys()); const keepName = createCliAutoSuffixNameGuard(allNames); for (const name of allNames) { if (!keepName(name)) { byName.delete(name); } } return Array.from(byName.values()).sort((a, b) => a.name.localeCompare(b.name)); } catch { logger.warn(`[${teamName}] Failed to parse config.json for launch fallback members`); return []; } } /** * Two-stage preflight check: * 1. `claude --version` verifies the binary is executable. * 2. Runtime control-plane commands verify provider auth/team-launch readiness. * * Do not use `-p` here: full print mode can initialize MCP/plugin/LSP startup context * before the first response, which makes Create Team preflight slow and flaky. */ private async probeClaudeRuntime( claudePath: string, cwd: string, env: NodeJS.ProcessEnv, providerId: TeamProviderId | undefined = 'anthropic', providerArgs: string[] = [] ): Promise<{ warning?: string }> { const resolvedProviderId = resolveTeamProviderId(providerId); const cliCommandLabel = getConfiguredCliCommandLabel(); if (!(await pathExistsAsDirectory(cwd))) { return { warning: `Working directory does not exist: ${cwd}`, }; } try { const versionProbe = await this.spawnProbe( claudePath, ['--version'], cwd, env, PREFLIGHT_BINARY_TIMEOUT_MS ); if (versionProbe.exitCode !== 0) { const errorText = buildCombinedLogs(versionProbe.stdout, versionProbe.stderr) || `${cliCommandLabel} exited with code ${versionProbe.exitCode ?? 'unknown'} during warm-up`; return { warning: `${cliCommandLabel} binary failed to start correctly. Details: ${errorText}`, }; } } catch (error) { const message = error instanceof Error ? error.message : String(error); if (isMissingCwdSpawnError(message) && !(await pathExistsAsDirectory(cwd))) { return { warning: `Working directory does not exist: ${cwd}`, }; } return { warning: `${cliCommandLabel} binary failed to start. Details: ${message}`, }; } if (resolvedProviderId === 'gemini') { const authState = await resolveGeminiRuntimeAuth(env); if (authState.authenticated) { return {}; } return { warning: authState.statusMessage ?? 'Gemini provider is not configured for runtime use. Set GEMINI_API_KEY or Google ADC credentials (plus GOOGLE_CLOUD_PROJECT when needed) and retry.', }; } if (resolvedProviderId === 'anthropic' || resolvedProviderId === 'codex') { return await this.probeProviderRuntimeControlPlane({ claudePath, cwd, env, providerId: resolvedProviderId, providerArgs, }); } return {}; } private buildRuntimeProviderReadinessWarning( providerId: TeamProviderId, providerStatus: Partial | null | undefined ): string | null { const providerLabel = getTeamProviderLabel(providerId); const detail = [providerStatus?.statusMessage?.trim(), providerStatus?.detailMessage?.trim()] .filter((entry): entry is string => Boolean(entry)) .join(' '); if (!providerStatus) { return `${providerLabel} provider is not configured for runtime use. Runtime status did not include this provider.`; } if (providerStatus.supported === false) { return `${providerLabel} provider is not configured for runtime use.${ detail ? ` ${detail}` : '' }`; } if (providerStatus.authenticated === false) { return `${providerLabel} provider is not authenticated.${detail ? ` ${detail}` : ''}`; } if (providerStatus.capabilities?.teamLaunch === false) { return `${providerLabel} provider is not configured for runtime use. Team launch is unavailable.${ detail ? ` ${detail}` : '' }`; } return null; } private extractAuthStatusReadiness( providerId: TeamProviderId, parsed: AuthStatusCommandResponse ): { authenticated: boolean | null; providerStatus: Partial | null; } { const providerStatus = parsed.providers?.[providerId] ?? (parsed.provider === providerId || !parsed.provider ? parsed.status : null) ?? null; if (typeof providerStatus?.authenticated === 'boolean') { return { authenticated: providerStatus.authenticated, providerStatus, }; } if (typeof parsed.loggedIn === 'boolean') { return { authenticated: parsed.loggedIn, providerStatus, }; } return { authenticated: null, providerStatus, }; } private async probeProviderRuntimeControlPlane({ claudePath, cwd, env, providerId, providerArgs, }: { claudePath: string; cwd: string; env: NodeJS.ProcessEnv; providerId: TeamProviderId; providerArgs: string[]; }): Promise<{ warning?: string }> { const cliCommandLabel = getConfiguredCliCommandLabel(); const providerLabel = getTeamProviderLabel(providerId); try { const runtimeStatus = await execCli( claudePath, buildProviderCliCommandArgs(providerArgs, [ 'runtime', 'status', '--json', '--summary', '--provider', providerId, ]), { cwd, env, timeout: PROVIDER_RUNTIME_STATUS_TIMEOUT_MS, } ); const parsed = extractJsonObjectFromCli(runtimeStatus.stdout); const providerStatus = parsed.providers?.[providerId] ?? null; const warning = this.buildRuntimeProviderReadinessWarning(providerId, providerStatus); appendPreflightDebugLog('provider_runtime_control_plane_status', { providerId, cwd, ready: !warning, authenticated: providerStatus?.authenticated, teamLaunch: providerStatus?.capabilities?.teamLaunch, oneShot: providerStatus?.capabilities?.oneShot, warning, }); return warning ? { warning } : {}; } catch (runtimeStatusError) { const runtimeStatusMessage = runtimeStatusError instanceof Error ? runtimeStatusError.message : String(runtimeStatusError); try { const authStatus = await execCli( claudePath, buildProviderCliCommandArgs(providerArgs, [ 'auth', 'status', '--json', '--provider', providerId, ]), { cwd, env, timeout: 8_000, } ); const parsed = extractJsonObjectFromCli(authStatus.stdout); const authReadiness = this.extractAuthStatusReadiness(providerId, parsed); const readinessWarning = authReadiness.providerStatus ? this.buildRuntimeProviderReadinessWarning(providerId, authReadiness.providerStatus) : null; if (authReadiness.authenticated === false || readinessWarning) { const authWarning = readinessWarning ?? `${providerLabel} provider is not authenticated. Runtime auth status reported logged out.`; appendPreflightDebugLog('provider_runtime_control_plane_auth_fallback', { providerId, cwd, ready: false, runtimeStatusError: runtimeStatusMessage, warning: authWarning, }); return { warning: authWarning }; } if (authReadiness.authenticated === true) { const warning = `${cliCommandLabel} runtime status was unavailable, but auth status passed. ` + `Proceeding with catalog checks. Details: ${runtimeStatusMessage}`; appendPreflightDebugLog('provider_runtime_control_plane_auth_fallback', { providerId, cwd, ready: true, runtimeStatusError: runtimeStatusMessage, warning, }); return { warning }; } } catch (authStatusError) { const authStatusMessage = authStatusError instanceof Error ? authStatusError.message : String(authStatusError); appendPreflightDebugLog('provider_runtime_control_plane_auth_fallback', { providerId, cwd, ready: false, runtimeStatusError: runtimeStatusMessage, authStatusError: authStatusMessage, }); return { warning: `${cliCommandLabel} runtime status check did not complete. ` + `Proceeding with catalog checks. Details: ${runtimeStatusMessage}; auth status failed: ${authStatusMessage}`, }; } return { warning: `${cliCommandLabel} runtime status was unavailable and auth status did not report ${providerLabel} authentication. ` + `Proceeding with catalog checks. Details: ${runtimeStatusMessage}`, }; } } private async runProviderOneShotDiagnostic( claudePath: string, cwd: string, env: NodeJS.ProcessEnv, providerId: TeamProviderId | undefined = 'anthropic', providerArgs: string[] = [] ): Promise<{ warning?: string }> { const cliCommandLabel = getConfiguredCliCommandLabel(); const resolvedProviderId = resolveTeamProviderId(providerId); if (!(await pathExistsAsDirectory(cwd))) { appendPreflightDebugLog('provider_one_shot_diagnostic_skipped', { providerId: resolvedProviderId, cwd, reason: 'missing_cwd', }); return {}; } const args = buildProviderCliCommandArgs(providerArgs, getPreflightPingArgs(providerId)); const timeoutMs = getPreflightTimeoutMs(providerId); appendPreflightDebugLog('provider_one_shot_diagnostic_start', { providerId: resolvedProviderId, cwd, timeoutMs, args, }); for (let attempt = 1; attempt <= PREFLIGHT_AUTH_MAX_RETRIES; attempt++) { let pingProbe: { exitCode: number | null; stdout: string; stderr: string } | null = null; try { pingProbe = await this.spawnProbe(claudePath, args, cwd, env, timeoutMs, { resolveOnOutputMatch: ({ stdout, stderr }) => { const combined = `${stdout}\n${stderr}`.trim(); return /\bPONG\b/i.test(combined); }, }); } catch (error) { const message = error instanceof Error ? error.message : String(error); if (!isProbeTimeoutMessage(message) && attempt < PREFLIGHT_AUTH_MAX_RETRIES) { appendPreflightDebugLog('provider_one_shot_diagnostic_retry', { providerId: resolvedProviderId, cwd, attempt, reason: truncatePreflightDebugText(message), }); logger.warn( `One-shot diagnostic failed (attempt ${attempt}/${PREFLIGHT_AUTH_MAX_RETRIES}), ` + `retrying in ${PREFLIGHT_AUTH_RETRY_DELAY_MS}ms: ${message}` ); await new Promise((resolve) => setTimeout(resolve, PREFLIGHT_AUTH_RETRY_DELAY_MS)); continue; } const normalizedMessage = normalizeProviderModelProbeFailureReason(message); appendPreflightDebugLog('provider_one_shot_diagnostic_complete', { providerId: resolvedProviderId, cwd, attempt, ok: false, reason: isProbeTimeoutMessage(message) ? 'timeout' : 'error', message: truncatePreflightDebugText(normalizedMessage), }); return { warning: (isProbeTimeoutMessage(message) ? 'One-shot diagnostic timed out after runtime readiness passed. ' : 'One-shot diagnostic did not complete after runtime readiness passed. ') + `This does not mark selected models unavailable. Details: ${normalizedMessage}`, }; } const combinedOutput = buildCombinedLogs(pingProbe.stdout, pingProbe.stderr); const isAuthFailure = this.isAuthFailureWarning(combinedOutput, 'probe'); if (isAuthFailure && attempt < PREFLIGHT_AUTH_MAX_RETRIES) { appendPreflightDebugLog('provider_one_shot_diagnostic_retry', { providerId: resolvedProviderId, cwd, attempt, exitCode: pingProbe.exitCode, reason: 'auth_failure', output: truncatePreflightDebugText(combinedOutput), }); logger.warn( `One-shot diagnostic auth failure detected (attempt ${attempt}/${PREFLIGHT_AUTH_MAX_RETRIES}), ` + `retrying in ${PREFLIGHT_AUTH_RETRY_DELAY_MS}ms - likely stale locks from interrupted process` ); await new Promise((resolve) => setTimeout(resolve, PREFLIGHT_AUTH_RETRY_DELAY_MS)); continue; } if (isAuthFailure || pingProbe.exitCode !== 0) { const normalizedOutput = this.normalizeApiRetryErrorMessage(combinedOutput) || combinedOutput.trim(); const hint = isAuthFailure ? resolvedProviderId === 'codex' ? 'Codex provider is not authenticated for `-p` mode. ' + `Authenticate Codex in ${cliCommandLabel} and retry.` + (attempt > 1 ? ` (failed after ${attempt} attempts)` : '') : `${cliCommandLabel} \`-p\` mode is not authenticated. ` + (cliCommandLabel === 'claude' ? 'Run `claude auth login` (or start `claude` and run `/login`) to authenticate. ' : `Authenticate Anthropic in ${cliCommandLabel} and retry. `) + 'For automation/headless use, set ANTHROPIC_API_KEY.' + (attempt > 1 ? ` (failed after ${attempt} attempts)` : '') : normalizedOutput ? `${cliCommandLabel} preflight check failed (exit code ${pingProbe.exitCode ?? 'unknown'}). Details: ${normalizedOutput}` : `${cliCommandLabel} preflight check failed (exit code ${pingProbe.exitCode ?? 'unknown'}).`; appendPreflightDebugLog('provider_one_shot_diagnostic_complete', { providerId: resolvedProviderId, cwd, attempt, ok: false, exitCode: pingProbe.exitCode, authFailure: isAuthFailure, output: truncatePreflightDebugText(normalizedOutput || combinedOutput), }); return { warning: 'One-shot diagnostic failed after runtime readiness passed. ' + `This does not mark selected models unavailable. Details: ${hint}`, }; } const pongCandidate = pingProbe.stdout.trim() || pingProbe.stderr.trim(); const isPong = new RegExp(`\\b${getProviderModelProbeExpectedOutput()}\\b`, 'i').test( pongCandidate ); if (!isPong) { appendPreflightDebugLog('provider_one_shot_diagnostic_complete', { providerId: resolvedProviderId, cwd, attempt, ok: false, exitCode: pingProbe.exitCode, reason: 'unexpected_output', output: truncatePreflightDebugText(combinedOutput), }); return { warning: 'One-shot diagnostic completed but did not return the expected PONG. ' + 'This does not mark selected models unavailable. ' + `Output: ${combinedOutput || '(empty)'}`, }; } if (attempt > 1) { logger.info( `One-shot diagnostic succeeded on attempt ${attempt} (previous attempt had auth failure)` ); } appendPreflightDebugLog('provider_one_shot_diagnostic_complete', { providerId: resolvedProviderId, cwd, attempt, ok: true, exitCode: pingProbe.exitCode, }); return {}; } return {}; } /** * Run `claude --help` and return the output. Cached for 5 minutes. * Used by the validateCliArgs IPC handler to check user-entered flags. */ async getCliHelpOutput(cwd?: string): Promise { if ( this.helpOutputCache && Date.now() - this.helpOutputCacheTime < TeamProvisioningService.HELP_CACHE_TTL_MS ) { return this.helpOutputCache; } const targetCwd = cwd ?? process.cwd(); const probeResult = await this.getCachedOrProbeResult(targetCwd, 'anthropic'); if (!probeResult?.claudePath) { throw new Error(`${getConfiguredCliCommandLabel()} not found`); } const { env } = await this.buildProvisioningEnv(); const result = await this.spawnProbe( probeResult.claudePath, ['--help'], targetCwd, env, 10_000 ); const output = (result.stdout + '\n' + result.stderr).trim(); if (!output) { throw new Error( `${getConfiguredCliCommandLabel()} --help returned empty output (exit code: ${String(result.exitCode)})` ); } this.helpOutputCache = output; this.helpOutputCacheTime = Date.now(); return output; } private buildAgentTeamsMcpValidationError(output: string): string { const detail = this.normalizeApiRetryErrorMessage(output) || output.trim(); if (!detail) { return 'agent-teams MCP preflight failed before team launch.'; } return `agent-teams MCP preflight failed before team launch. Details: ${detail}`; } private async readAgentTeamsMcpLaunchSpec( mcpConfigPath: string ): Promise { let parsed: AgentTeamsMcpConfigFile; try { const raw = await fs.promises.readFile(mcpConfigPath, 'utf8'); parsed = JSON.parse(raw) as AgentTeamsMcpConfigFile; } catch (error) { throw new Error( this.buildAgentTeamsMcpValidationError( `Failed to read generated MCP config ${mcpConfigPath}: ${ error instanceof Error ? error.message : String(error) }` ) ); } const server = parsed.mcpServers?.['agent-teams']; if (!server) { throw new Error( this.buildAgentTeamsMcpValidationError( `Generated MCP config ${mcpConfigPath} does not contain an "agent-teams" server entry.` ) ); } if (typeof server.command !== 'string' || server.command.trim().length === 0) { throw new Error( this.buildAgentTeamsMcpValidationError( 'Generated agent-teams MCP config is missing a valid launch command.' ) ); } if (server.args !== undefined && !isStringArray(server.args)) { throw new Error( this.buildAgentTeamsMcpValidationError( 'Generated agent-teams MCP config has invalid args; expected a string array.' ) ); } if (server.cwd !== undefined && typeof server.cwd !== 'string') { throw new Error( this.buildAgentTeamsMcpValidationError( 'Generated agent-teams MCP config has invalid cwd; expected a string path.' ) ); } return { command: server.command, args: server.args ?? [], cwd: typeof server.cwd === 'string' ? server.cwd : undefined, env: normalizeRecordStringValues(server.env), }; } private async createAgentTeamsMcpValidationFixture( projectPath: string ): Promise { const claudeDir = await fs.promises.mkdtemp( path.join(os.tmpdir(), 'agent-teams-mcp-validate-') ); const teamName = 'mcp-validation-team'; const memberName = 'mcp-validation-member'; const teamDir = path.join(claudeDir, 'teams', teamName); await fs.promises.mkdir(teamDir, { recursive: true }); await fs.promises.writeFile( path.join(teamDir, 'config.json'), JSON.stringify( { name: teamName, projectPath, members: [ { name: 'team-lead', agentType: 'team-lead', role: 'lead' }, { name: memberName, agentType: 'teammate', role: 'developer' }, ], }, null, 2 ), 'utf8' ); return { claudeDir, teamName, memberName, }; } private async validateAgentTeamsMcpRuntime( _claudePath: string, cwd: string, env: NodeJS.ProcessEnv, mcpConfigPath: string, options: { isCancelled?: () => boolean; } = {} ): Promise { const launchSpec = await this.readAgentTeamsMcpLaunchSpec(mcpConfigPath); const fixture = await this.createAgentTeamsMcpValidationFixture(cwd); let child: ReturnType | null = null; let stdoutBuffer = ''; let stderrBuffer = ''; let nextRequestId = 1; let cancellationTriggered = false; let cancellationTimer: ReturnType | null = null; const cancellationMessage = 'agent-teams MCP preflight cancelled by app shutdown'; const pending = new Map< number, { resolve: (value: unknown) => void; reject: (error: Error) => void; timeoutHandle: ReturnType; } >(); const rejectAll = (error: Error): void => { for (const [id, entry] of pending) { clearTimeout(entry.timeoutHandle); entry.reject(error); pending.delete(id); } }; const getCancellationError = (): Error => new Error(cancellationMessage); const cancelPreflightIfNeeded = (): boolean => { if (cancellationTriggered) { return true; } if (!options.isCancelled?.()) { return false; } cancellationTriggered = true; const error = getCancellationError(); rejectAll(error); if (child?.pid) { killProcessTree(child); } return true; }; const throwIfCancelled = (): void => { if (cancelPreflightIfNeeded()) { throw getCancellationError(); } }; try { throwIfCancelled(); child = spawnCli(launchSpec.command, launchSpec.args, { cwd: launchSpec.cwd ?? cwd, env: { ...env, ...launchSpec.env, AGENT_TEAMS_MCP_CLAUDE_DIR: fixture.claudeDir, }, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true, }); this.transientProbeProcesses.add(child); if (options.isCancelled) { cancellationTimer = setInterval(() => { if (cancelPreflightIfNeeded() && cancellationTimer) { clearInterval(cancellationTimer); cancellationTimer = null; } }, 100); cancellationTimer.unref?.(); } const parseStdoutLine = (line: string): void => { let message: McpJsonRpcResponse; try { message = JSON.parse(line) as McpJsonRpcResponse; } catch (error) { logger.warn( `agent-teams MCP preflight emitted non-JSON stdout line: ${ error instanceof Error ? error.message : String(error) }` ); return; } if (typeof message.id !== 'number') { return; } const entry = pending.get(message.id); if (!entry) { return; } clearTimeout(entry.timeoutHandle); pending.delete(message.id); if (message.error) { entry.reject(new Error(message.error.message ?? 'Unknown MCP JSON-RPC error')); return; } entry.resolve(message.result); }; child.stdout?.setEncoding('utf8'); child.stdout?.on('data', (chunk: string | Buffer) => { stdoutBuffer += chunk.toString(); while (true) { const newlineIndex = stdoutBuffer.indexOf('\n'); if (newlineIndex === -1) { break; } const line = stdoutBuffer.slice(0, newlineIndex).trim(); stdoutBuffer = stdoutBuffer.slice(newlineIndex + 1); if (!line) { continue; } parseStdoutLine(line); } }); child.stderr?.setEncoding('utf8'); child.stderr?.on('data', (chunk: string | Buffer) => { stderrBuffer += chunk.toString(); }); child.once('error', (error) => { rejectAll(error instanceof Error ? error : new Error(String(error))); }); child.once('close', (code, signal) => { if (pending.size === 0) { return; } rejectAll( new Error( `agent-teams MCP process exited unexpectedly during preflight (code=${ code ?? 'null' } signal=${signal ?? 'null'})` ) ); }); const request = ( method: string, params: Record, timeoutMs: number = VERIFY_TIMEOUT_MS ): Promise => new Promise((resolve, reject) => { if (cancelPreflightIfNeeded()) { reject(getCancellationError()); return; } if (!child?.stdin) { reject(new Error('agent-teams MCP stdin is not available')); return; } const id = nextRequestId++; const timeoutHandle = setTimeout(() => { pending.delete(id); reject(new Error(`agent-teams MCP request timed out: ${method}`)); }, timeoutMs); pending.set(id, { resolve: resolve as (value: unknown) => void, reject, timeoutHandle, }); if (cancelPreflightIfNeeded()) { clearTimeout(timeoutHandle); pending.delete(id); reject(getCancellationError()); return; } child.stdin.write( `${JSON.stringify({ jsonrpc: '2.0', id, method, params })}\n`, (error) => { if (!error) { return; } clearTimeout(timeoutHandle); pending.delete(id); reject(error instanceof Error ? error : new Error(String(error))); } ); }); const notify = async (method: string, params?: Record): Promise => { if (!child?.stdin) { throw new Error('agent-teams MCP stdin is not available'); } const stdin = child.stdin; await new Promise((resolve, reject) => { stdin.write( `${JSON.stringify({ jsonrpc: '2.0', method, ...(params ? { params } : {}) })}\n`, (error) => { if (error) { reject(error instanceof Error ? error : new Error(String(error))); return; } resolve(); } ); }); }; await request( 'initialize', { protocolVersion: '2024-11-05', capabilities: {}, clientInfo: { name: 'agent-teams-ai', version: '1.0.0' }, }, MCP_PREFLIGHT_INITIALIZE_TIMEOUT_MS ); throwIfCancelled(); await notify('notifications/initialized'); const toolsList = await request('tools/list', {}); throwIfCancelled(); const availableTools = new Set((toolsList.tools ?? []).map((tool) => tool.name)); const requiredTools = Array.from( new Set([ ...AGENT_TEAMS_TEAMMATE_OPERATIONAL_TOOL_NAMES, 'lead_briefing', 'runtime_bootstrap_checkin', 'runtime_deliver_message', 'runtime_task_event', 'runtime_heartbeat', ]) ); const missingTools = requiredTools.filter((toolName) => !availableTools.has(toolName)); if (missingTools.length > 0) { throw new Error( `agent-teams MCP started but tools/list did not include required tool(s): ${missingTools.join( ', ' )}` ); } const memberBriefing = await request('tools/call', { name: 'member_briefing', arguments: { claudeDir: fixture.claudeDir, teamName: fixture.teamName, memberName: fixture.memberName, runtimeProvider: 'opencode', includeActiveProcesses: false, }, }); throwIfCancelled(); if (memberBriefing.isError) { throw new Error( memberBriefing.content?.[0]?.text ?? 'agent-teams MCP returned an unspecified error for member_briefing' ); } const briefingText = memberBriefing.content?.find((item) => item.type === 'text')?.text ?? ''; if (briefingText.trim().length === 0) { throw new Error('agent-teams MCP returned empty content for member_briefing'); } const leadBriefing = await request('tools/call', { name: 'lead_briefing', arguments: { claudeDir: fixture.claudeDir, teamName: fixture.teamName, }, }); throwIfCancelled(); if (leadBriefing.isError) { throw new Error( leadBriefing.content?.[0]?.text ?? 'agent-teams MCP returned an unspecified error for lead_briefing' ); } const leadBriefingText = leadBriefing.content?.find((item) => item.type === 'text')?.text ?? ''; if (leadBriefingText.trim().length === 0) { throw new Error('agent-teams MCP returned empty content for lead_briefing'); } } catch (error) { if (error instanceof Error && error.message === cancellationMessage) { throw error; } const detail = buildCombinedLogs('', stderrBuffer).trim(); const errorText = error instanceof Error && detail.length > 0 ? `${error.message}\n${detail}` : detail || String(error); throw new Error(this.buildAgentTeamsMcpValidationError(errorText)); } finally { if (cancellationTimer) { clearInterval(cancellationTimer); cancellationTimer = null; } rejectAll(new Error('agent-teams MCP preflight session closed')); if (child) { this.transientProbeProcesses.delete(child); } if (child?.stdin && !child.stdin.destroyed && !child.stdin.writableEnded) { const stdin = child.stdin; await new Promise((resolve) => { try { stdin.end(() => resolve()); } catch { resolve(); } }); } if (child?.pid) { await waitForChildProcessToExit(child, MCP_PREFLIGHT_SHUTDOWN_GRACE_MS); if (isProcessAlive(child.pid)) { killProcessTree(child); await waitForPidsToExit([child.pid], { timeoutMs: MCP_PREFLIGHT_SHUTDOWN_TIMEOUT_MS, pollMs: MCP_PREFLIGHT_SHUTDOWN_POLL_MS, }); await waitForChildProcessToExit(child, MCP_PREFLIGHT_SHUTDOWN_GRACE_MS); } } await fs.promises.rm(fixture.claudeDir, { recursive: true, force: true }).catch(() => {}); } } private async spawnProbe( claudePath: string, args: string[], cwd: string, env: NodeJS.ProcessEnv, timeoutMs: number, options?: { /** * Optional early success predicate. If this returns true based on * buffered stdout/stderr, the probe resolves immediately (and the process * is best-effort terminated) instead of waiting for `close`. */ resolveOnOutputMatch?: (ctx: { stdout: string; stderr: string }) => boolean; } ): Promise<{ exitCode: number | null; stdout: string; stderr: string }> { return new Promise((resolve, reject) => { const child = spawnCli(claudePath, args, { cwd, env, stdio: ['ignore', 'pipe', 'pipe'], }); this.transientProbeProcesses.add(child); const cleanupProbe = (): void => { this.transientProbeProcesses.delete(child); }; let stdoutText = ''; let stderrText = ''; let settled = false; const timeoutHandle = setTimeout(() => { settled = true; cleanupProbe(); killProcessTree(child); reject(new Error(`Timeout running: ${getConfiguredCliCommandLabel()} ${args.join(' ')}`)); }, timeoutMs); timeoutHandle.unref?.(); const maybeResolveEarly = (): void => { if (settled) return; if (!options?.resolveOnOutputMatch) return; const ctx = { stdout: stdoutText.trim(), stderr: stderrText.trim() }; if (!options.resolveOnOutputMatch(ctx)) return; settled = true; clearTimeout(timeoutHandle); cleanupProbe(); // If the process printed the match but hangs during teardown, don't // block the UI; terminate best-effort and resolve. killProcessTree(child); resolve({ exitCode: 0, stdout: ctx.stdout, stderr: ctx.stderr }); }; child.stdout?.on('data', (chunk: Buffer) => { stdoutText += chunk.toString('utf8'); maybeResolveEarly(); }); child.stderr?.on('data', (chunk: Buffer) => { stderrText += chunk.toString('utf8'); maybeResolveEarly(); }); child.once('error', (error) => { if (settled) return; settled = true; clearTimeout(timeoutHandle); cleanupProbe(); reject(error); }); child.once('close', (exitCode) => { if (settled) return; settled = true; clearTimeout(timeoutHandle); cleanupProbe(); resolve({ exitCode, stdout: stdoutText.trim(), stderr: stderrText.trim(), }); }); }); } }