fix(team): reconcile teammate launch liveness
This commit is contained in:
777genius
parent
1d19a59b12
commit
4d5533585c
3 changed files with 704 additions and 33 deletions
|
|
@ -1714,6 +1714,10 @@ function isOpenCodeBridgeLaunchFailureReason(reason?: string): boolean {
|
|||
return reason?.trim() === 'OpenCode bridge reported member launch failure';
|
||||
}
|
||||
|
||||
function isRegisteredRuntimeMetadataFailureReason(reason?: string): boolean {
|
||||
return reason?.trim() === 'registered runtime metadata without live process';
|
||||
}
|
||||
|
||||
function isTmuxNoServerRunningError(error: unknown): boolean {
|
||||
const text = error instanceof Error ? error.message : String(error ?? '');
|
||||
return (
|
||||
|
|
@ -1727,6 +1731,7 @@ function isAutoClearableLaunchFailureReason(reason?: string): boolean {
|
|||
isNeverSpawnedDuringLaunchReason(reason) ||
|
||||
isLaunchGraceWindowFailureReason(reason) ||
|
||||
isConfigRegistrationFailureReason(reason) ||
|
||||
isRegisteredRuntimeMetadataFailureReason(reason) ||
|
||||
isOpenCodeBridgeLaunchFailureReason(reason)
|
||||
);
|
||||
}
|
||||
|
|
@ -4179,6 +4184,7 @@ export class TeamProvisioningService {
|
|||
private inFlightResponses = new Set<string>();
|
||||
private runtimeAdapterRegistry: TeamRuntimeAdapterRegistry | null = null;
|
||||
private controlApiBaseUrlResolver: (() => Promise<string | null>) | null = null;
|
||||
private readonly stoppedTeamOpenCodeRuntimeCleanupInFlight = new Map<string, Promise<number>>();
|
||||
private crossTeamSender:
|
||||
| ((request: {
|
||||
fromTeam: string;
|
||||
|
|
@ -4728,6 +4734,246 @@ export class TeamProvisioningService {
|
|||
return null;
|
||||
}
|
||||
|
||||
private canDeliverToOpenCodeRuntimeForTeam(teamName: string): boolean {
|
||||
if (this.isTeamAlive(teamName)) {
|
||||
return true;
|
||||
}
|
||||
return this.hasAlivePersistedTeamProcess(teamName);
|
||||
}
|
||||
|
||||
private hasAlivePersistedTeamProcess(teamName: string): boolean {
|
||||
const processesPath = path.join(getTeamsBasePath(), teamName, 'processes.json');
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(fs.readFileSync(processesPath, 'utf8')) as unknown;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
if (!Array.isArray(parsed)) {
|
||||
return false;
|
||||
}
|
||||
return parsed.some((row) => {
|
||||
if (!row || typeof row !== 'object') {
|
||||
return false;
|
||||
}
|
||||
const processRow = row as { pid?: unknown; stoppedAt?: unknown };
|
||||
return (
|
||||
typeof processRow.pid === 'number' &&
|
||||
Number.isFinite(processRow.pid) &&
|
||||
processRow.stoppedAt == null &&
|
||||
isProcessAlive(processRow.pid)
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
private cleanupStoppedTeamOpenCodeRuntimeLanesInBackground(teamName: string): void {
|
||||
void this.stopOpenCodeRuntimeLanesForStoppedTeam(teamName).catch((error) => {
|
||||
logger.warn(
|
||||
`[${teamName}] Failed to clean up stopped-team OpenCode runtime lanes: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}`
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
private stopOpenCodeRuntimeLanesForStoppedTeam(teamName: string): Promise<number> {
|
||||
const existing = this.stoppedTeamOpenCodeRuntimeCleanupInFlight.get(teamName);
|
||||
if (existing) {
|
||||
return existing;
|
||||
}
|
||||
let cleanup!: Promise<number>;
|
||||
cleanup = this.stopOpenCodeRuntimeLanesForStoppedTeamInternal(teamName).finally(() => {
|
||||
if (this.stoppedTeamOpenCodeRuntimeCleanupInFlight.get(teamName) === cleanup) {
|
||||
this.stoppedTeamOpenCodeRuntimeCleanupInFlight.delete(teamName);
|
||||
}
|
||||
});
|
||||
this.stoppedTeamOpenCodeRuntimeCleanupInFlight.set(teamName, cleanup);
|
||||
return cleanup;
|
||||
}
|
||||
|
||||
private async stopOpenCodeRuntimeLanesForStoppedTeamInternal(teamName: string): Promise<number> {
|
||||
if (this.canDeliverToOpenCodeRuntimeForTeam(teamName)) {
|
||||
return 0;
|
||||
}
|
||||
const laneIndex = await readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName).catch(
|
||||
() => null
|
||||
);
|
||||
const activeLaneIds = Object.entries(laneIndex?.lanes ?? {})
|
||||
.filter(([, entry]) => entry.state === 'active')
|
||||
.map(([laneId]) => laneId)
|
||||
.sort((left, right) => left.localeCompare(right));
|
||||
if (activeLaneIds.length === 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const adapter = this.getOpenCodeRuntimeAdapter();
|
||||
const previousLaunchState = await this.launchStateStore.read(teamName).catch(() => null);
|
||||
const [config, metaMembers] = await Promise.all([
|
||||
this.configReader.getConfig(teamName).catch(() => null),
|
||||
this.membersMetaStore.getMembers(teamName).catch(() => []),
|
||||
]);
|
||||
const evidenceReader = new OpenCodeRuntimeManifestEvidenceReader({
|
||||
teamsBasePath: getTeamsBasePath(),
|
||||
});
|
||||
let stopped = 0;
|
||||
for (const laneId of activeLaneIds) {
|
||||
const evidence = await evidenceReader.read(teamName, laneId).catch(() => null);
|
||||
const runId = evidence?.activeRunId?.trim() || null;
|
||||
if (adapter && runId) {
|
||||
try {
|
||||
await adapter.stop({
|
||||
runId,
|
||||
laneId,
|
||||
teamName,
|
||||
cwd: this.resolveOpenCodeRuntimeLaneCleanupCwd(teamName, laneId, config, metaMembers),
|
||||
providerId: 'opencode',
|
||||
reason: 'cleanup',
|
||||
previousLaunchState,
|
||||
force: true,
|
||||
});
|
||||
stopped += 1;
|
||||
} catch (error) {
|
||||
logger.warn(
|
||||
`[${teamName}] Failed to stop orphaned OpenCode lane ${laneId}: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}`
|
||||
);
|
||||
continue;
|
||||
}
|
||||
} else if (runId) {
|
||||
logger.warn(
|
||||
`[${teamName}] OpenCode lane ${laneId} belongs to stopped team, but runtime adapter is unavailable.`
|
||||
);
|
||||
continue;
|
||||
} else if (!runId) {
|
||||
const pidStopResult = this.tryStopPersistedOpenCodeRuntimePidForStoppedLane({
|
||||
teamName,
|
||||
laneId,
|
||||
previousLaunchState,
|
||||
});
|
||||
if (pidStopResult === 'unsafe') {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
await clearOpenCodeRuntimeLaneStorage({
|
||||
teamsBasePath: getTeamsBasePath(),
|
||||
teamName,
|
||||
laneId,
|
||||
}).catch(() => undefined);
|
||||
this.deleteSecondaryRuntimeRun(teamName, laneId);
|
||||
if (laneId === 'primary') {
|
||||
this.runtimeAdapterRunByTeam.delete(teamName);
|
||||
this.aliveRunByTeam.delete(teamName);
|
||||
this.provisioningRunByTeam.delete(teamName);
|
||||
}
|
||||
}
|
||||
return stopped;
|
||||
}
|
||||
|
||||
private tryStopPersistedOpenCodeRuntimePidForStoppedLane(input: {
|
||||
teamName: string;
|
||||
laneId: string;
|
||||
previousLaunchState: PersistedTeamLaunchSnapshot | null;
|
||||
}): 'stopped' | 'no_pid' | 'unsafe' {
|
||||
const persistedMember = Object.values(input.previousLaunchState?.members ?? {}).find(
|
||||
(member) => member.providerId === 'opencode' && member.laneId === input.laneId
|
||||
);
|
||||
if (!persistedMember) {
|
||||
return 'no_pid';
|
||||
}
|
||||
const pid = persistedMember.runtimePid;
|
||||
if (typeof pid !== 'number' || !Number.isFinite(pid) || pid <= 0) {
|
||||
return 'no_pid';
|
||||
}
|
||||
const command = this.readProcessCommandByPid(pid);
|
||||
if (!command) {
|
||||
return 'no_pid';
|
||||
}
|
||||
const persistedProcessCommand = (persistedMember as { processCommand?: unknown })
|
||||
.processCommand;
|
||||
const expectedCommand =
|
||||
typeof persistedProcessCommand === 'string' ? persistedProcessCommand.trim() : '';
|
||||
if (expectedCommand && command !== expectedCommand) {
|
||||
logger.warn(
|
||||
`[${input.teamName}] Refusing to stop persisted OpenCode pid ${pid} for lane ${input.laneId}: process command changed.`
|
||||
);
|
||||
return 'unsafe';
|
||||
}
|
||||
if (!this.isOpenCodeServeCommand(command)) {
|
||||
logger.warn(
|
||||
`[${input.teamName}] Refusing to stop persisted OpenCode pid ${pid} for lane ${input.laneId}: process is not opencode serve.`
|
||||
);
|
||||
return 'unsafe';
|
||||
}
|
||||
try {
|
||||
killProcessByPid(pid);
|
||||
logger.info(
|
||||
`[${input.teamName}] Killed orphaned OpenCode runtime pid=${pid} for stopped lane ${input.laneId}`
|
||||
);
|
||||
return 'stopped';
|
||||
} catch (error) {
|
||||
logger.warn(
|
||||
`[${input.teamName}] Failed to kill orphaned OpenCode runtime pid=${pid} for stopped lane ${
|
||||
input.laneId
|
||||
}: ${error instanceof Error ? error.message : String(error)}`
|
||||
);
|
||||
return 'unsafe';
|
||||
}
|
||||
}
|
||||
|
||||
private readProcessCommandByPid(pid: number): string | null {
|
||||
if (process.platform === 'win32') {
|
||||
try {
|
||||
return (
|
||||
listWindowsProcessTableSync()
|
||||
.find((row) => row.pid === pid)
|
||||
?.command?.trim() || null
|
||||
);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
try {
|
||||
return execFileSync('ps', ['-p', String(pid), '-o', 'command='], {
|
||||
encoding: 'utf8',
|
||||
stdio: ['ignore', 'pipe', 'ignore'],
|
||||
}).trim();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private isOpenCodeServeCommand(command: string): boolean {
|
||||
return /(^|[/\\\s])opencode(?:\.exe)?(\s|$)/i.test(command) && /\sserve(\s|$)/i.test(command);
|
||||
}
|
||||
|
||||
private resolveOpenCodeRuntimeLaneCleanupCwd(
|
||||
teamName: string,
|
||||
laneId: string,
|
||||
config: TeamConfig | null,
|
||||
metaMembers: readonly TeamMember[]
|
||||
): string | undefined {
|
||||
const projectPath = config?.projectPath?.trim() || this.readPersistedTeamProjectPath(teamName);
|
||||
const memberName = this.extractOpenCodeRuntimeLaneMemberName(laneId);
|
||||
if (!memberName) {
|
||||
return projectPath || undefined;
|
||||
}
|
||||
const normalized = memberName.toLowerCase();
|
||||
const configMember = config?.members?.find(
|
||||
(member) => member.name?.trim().toLowerCase() === normalized
|
||||
);
|
||||
const metaMember = metaMembers.find(
|
||||
(member) => member.name?.trim().toLowerCase() === normalized
|
||||
);
|
||||
return metaMember?.cwd?.trim() || configMember?.cwd?.trim() || projectPath || undefined;
|
||||
}
|
||||
|
||||
private extractOpenCodeRuntimeLaneMemberName(laneId: string): string | null {
|
||||
const match = /^secondary:opencode:(.+)$/i.exec(laneId.trim());
|
||||
return match?.[1]?.trim() || null;
|
||||
}
|
||||
|
||||
private getOpenCodeRuntimeAdapter(): TeamLaunchRuntimeAdapter | null {
|
||||
if (!this.runtimeAdapterRegistry?.has('opencode')) {
|
||||
return null;
|
||||
|
|
@ -5332,6 +5578,10 @@ export class TeamProvisioningService {
|
|||
if (!this.isOpenCodePromptDeliveryWatchdogEnabled()) {
|
||||
return 0;
|
||||
}
|
||||
if (!this.canDeliverToOpenCodeRuntimeForTeam(teamName)) {
|
||||
await this.stopOpenCodeRuntimeLanesForStoppedTeam(teamName);
|
||||
return 0;
|
||||
}
|
||||
const laneIndex = await readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName).catch(
|
||||
() => null
|
||||
);
|
||||
|
|
@ -5484,6 +5734,10 @@ export class TeamProvisioningService {
|
|||
if (!adapter) {
|
||||
return { delivered: false, reason: 'opencode_runtime_message_bridge_unavailable' };
|
||||
}
|
||||
if (!this.canDeliverToOpenCodeRuntimeForTeam(teamName)) {
|
||||
this.cleanupStoppedTeamOpenCodeRuntimeLanesInBackground(teamName);
|
||||
return { delivered: false, reason: 'opencode_runtime_not_active' };
|
||||
}
|
||||
|
||||
const [config, teamMeta, metaMembers] = await Promise.all([
|
||||
this.configReader.getConfig(teamName).catch(() => null),
|
||||
|
|
@ -6472,6 +6726,10 @@ export class TeamProvisioningService {
|
|||
member: TeamMember;
|
||||
projectPath: string | null;
|
||||
}): Promise<boolean> {
|
||||
if (!this.canDeliverToOpenCodeRuntimeForTeam(input.teamName)) {
|
||||
this.cleanupStoppedTeamOpenCodeRuntimeLanesInBackground(input.teamName);
|
||||
return false;
|
||||
}
|
||||
const snapshot = await this.launchStateStore.read(input.teamName).catch(() => null);
|
||||
const persistedMember =
|
||||
snapshot?.members?.[input.member.name] ??
|
||||
|
|
@ -6499,6 +6757,10 @@ export class TeamProvisioningService {
|
|||
private async tryRecoverOpenCodeRuntimeLanesForDeliveryWatchdog(
|
||||
teamName: string
|
||||
): Promise<string[]> {
|
||||
if (!this.canDeliverToOpenCodeRuntimeForTeam(teamName)) {
|
||||
this.cleanupStoppedTeamOpenCodeRuntimeLanesInBackground(teamName);
|
||||
return [];
|
||||
}
|
||||
const snapshot = await this.launchStateStore.read(teamName).catch(() => null);
|
||||
const candidates = Object.values(snapshot?.members ?? {}).filter(
|
||||
isRecoverablePersistedOpenCodeRuntimeCandidate
|
||||
|
|
@ -14114,6 +14376,17 @@ export class TeamProvisioningService {
|
|||
memberName: string,
|
||||
options: OpenCodeMemberInboxRelayOptions = {}
|
||||
): Promise<OpenCodeMemberInboxRelayResult> {
|
||||
if (!this.canDeliverToOpenCodeRuntimeForTeam(teamName)) {
|
||||
this.cleanupStoppedTeamOpenCodeRuntimeLanesInBackground(teamName);
|
||||
return {
|
||||
relayed: 0,
|
||||
attempted: 0,
|
||||
delivered: 0,
|
||||
failed: 1,
|
||||
lastDelivery: { delivered: false, reason: 'opencode_runtime_not_active' },
|
||||
diagnostics: ['opencode_runtime_not_active'],
|
||||
};
|
||||
}
|
||||
const relayKey = this.getOpenCodeMemberRelayKey(teamName, memberName);
|
||||
const existing = this.openCodeMemberInboxRelayInFlight.get(relayKey);
|
||||
if (existing) {
|
||||
|
|
@ -17874,48 +18147,113 @@ export class TeamProvisioningService {
|
|||
} catch {
|
||||
return [];
|
||||
}
|
||||
const projectPath = config?.projectPath?.trim();
|
||||
if (!projectPath) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const projectDir = path.join(getProjectsBasePath(), extractBaseDir(encodePath(projectPath)));
|
||||
let entries: fs.Dirent[];
|
||||
try {
|
||||
entries = await fs.promises.readdir(projectDir, { withFileTypes: true });
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
|
||||
const outcomes: BootstrapTranscriptOutcome[] = [];
|
||||
const jsonlFiles = entries
|
||||
.filter((entry) => entry.isFile() && entry.name.endsWith('.jsonl'))
|
||||
.sort((left, right) => right.name.localeCompare(left.name));
|
||||
const projectDirs = await this.collectBootstrapTranscriptProjectDirs(
|
||||
teamName,
|
||||
memberName,
|
||||
config
|
||||
);
|
||||
const contextMemberNames = [
|
||||
memberName,
|
||||
...((config?.members ?? [])
|
||||
.map((member) => member.name?.trim())
|
||||
.filter((name): name is string => Boolean(name)) ?? []),
|
||||
];
|
||||
for (const entry of jsonlFiles) {
|
||||
if (config?.leadSessionId && entry.name === `${config.leadSessionId}.jsonl`) {
|
||||
for (const projectDir of projectDirs) {
|
||||
let entries: fs.Dirent[];
|
||||
try {
|
||||
entries = await fs.promises.readdir(projectDir, { withFileTypes: true });
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
const outcome = await this.readRecentBootstrapTranscriptOutcome(
|
||||
path.join(projectDir, entry.name),
|
||||
sinceMs,
|
||||
memberName,
|
||||
teamName,
|
||||
{ contextMemberNames }
|
||||
);
|
||||
if (outcome) {
|
||||
outcomes.push(outcome);
|
||||
|
||||
const jsonlFiles = entries
|
||||
.filter((entry) => entry.isFile() && entry.name.endsWith('.jsonl'))
|
||||
.sort((left, right) => right.name.localeCompare(left.name));
|
||||
for (const entry of jsonlFiles) {
|
||||
if (config?.leadSessionId && entry.name === `${config.leadSessionId}.jsonl`) {
|
||||
continue;
|
||||
}
|
||||
const outcome = await this.readRecentBootstrapTranscriptOutcome(
|
||||
path.join(projectDir, entry.name),
|
||||
sinceMs,
|
||||
memberName,
|
||||
teamName,
|
||||
{ contextMemberNames }
|
||||
);
|
||||
if (outcome) {
|
||||
outcomes.push(outcome);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return outcomes;
|
||||
}
|
||||
|
||||
private async collectBootstrapTranscriptProjectDirs(
|
||||
teamName: string,
|
||||
memberName: string,
|
||||
config: Awaited<ReturnType<TeamConfigReader['getConfig']>>
|
||||
): Promise<string[]> {
|
||||
const pathCandidates: string[] = [];
|
||||
const pathSeen = new Set<string>();
|
||||
const pushPath = (value: unknown): void => {
|
||||
if (typeof value !== 'string') {
|
||||
return;
|
||||
}
|
||||
let trimmed = value.trim();
|
||||
while (trimmed.endsWith('/') || trimmed.endsWith('\\')) {
|
||||
trimmed = trimmed.slice(0, -1);
|
||||
}
|
||||
if (!trimmed || pathSeen.has(trimmed)) {
|
||||
return;
|
||||
}
|
||||
pathSeen.add(trimmed);
|
||||
pathCandidates.push(trimmed);
|
||||
};
|
||||
|
||||
pushPath(config?.projectPath);
|
||||
if (Array.isArray(config?.projectPathHistory)) {
|
||||
for (let index = config.projectPathHistory.length - 1; index >= 0; index -= 1) {
|
||||
pushPath(config.projectPathHistory[index]);
|
||||
}
|
||||
}
|
||||
|
||||
const normalizedMemberName = memberName.trim().toLowerCase();
|
||||
const pushMatchingMemberCwd = (member: { name?: unknown; cwd?: unknown }): void => {
|
||||
const candidateName = typeof member.name === 'string' ? member.name.trim().toLowerCase() : '';
|
||||
if (candidateName && matchesTeamMemberIdentity(candidateName, normalizedMemberName)) {
|
||||
pushPath(member.cwd);
|
||||
}
|
||||
};
|
||||
for (const member of config?.members ?? []) {
|
||||
pushMatchingMemberCwd(member);
|
||||
}
|
||||
|
||||
const metaMembers = await this.membersMetaStore.getMembers(teamName).catch(() => []);
|
||||
for (const member of metaMembers) {
|
||||
pushMatchingMemberCwd(member);
|
||||
}
|
||||
|
||||
const dirs: string[] = [];
|
||||
const dirSeen = new Set<string>();
|
||||
const pushDir = (dir: string): void => {
|
||||
if (!dir || dirSeen.has(dir)) {
|
||||
return;
|
||||
}
|
||||
dirSeen.add(dir);
|
||||
dirs.push(dir);
|
||||
};
|
||||
for (const projectPath of pathCandidates) {
|
||||
const projectId = extractBaseDir(encodePath(projectPath));
|
||||
pushDir(path.join(getProjectsBasePath(), projectId));
|
||||
if (projectId.includes('_')) {
|
||||
pushDir(path.join(getProjectsBasePath(), projectId.replace(/_/g, '-')));
|
||||
}
|
||||
}
|
||||
return dirs;
|
||||
}
|
||||
|
||||
private selectLatestBootstrapTranscriptOutcome(
|
||||
outcomes: readonly BootstrapTranscriptOutcome[]
|
||||
): BootstrapTranscriptOutcome | null {
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ import { createPersistedLaunchSnapshot } from '../../../../src/main/services/tea
|
|||
import {
|
||||
getOpenCodeRuntimeLaneIndexPath,
|
||||
readOpenCodeRuntimeLaneIndex,
|
||||
setOpenCodeRuntimeActiveRunManifest,
|
||||
upsertOpenCodeRuntimeLaneIndexEntry,
|
||||
} from '../../../../src/main/services/team/opencode/store/OpenCodeRuntimeManifestEvidenceReader';
|
||||
|
||||
|
|
@ -11020,6 +11021,166 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
expect(adapter.messageInputs).toEqual([]);
|
||||
});
|
||||
|
||||
it('does not scan or deliver orphaned mixed OpenCode lanes after app restart when team is stopped', async () => {
|
||||
const teamName = 'mixed-opencode-stopped-orphaned-lane-safe-e2e';
|
||||
await writeMixedTeamConfig({ teamName, projectPath });
|
||||
await writeTeamMeta(teamName, projectPath);
|
||||
await writeMembersMeta(teamName);
|
||||
await upsertOpenCodeRuntimeLaneIndexEntry({
|
||||
teamsBasePath: getTeamsBasePath(),
|
||||
teamName,
|
||||
laneId: 'secondary:opencode:bob',
|
||||
state: 'active',
|
||||
diagnostics: ['orphaned lane from previous app session'],
|
||||
});
|
||||
await setOpenCodeRuntimeActiveRunManifest({
|
||||
teamsBasePath: getTeamsBasePath(),
|
||||
teamName,
|
||||
laneId: 'secondary:opencode:bob',
|
||||
runId: 'orphaned-opencode-run-bob',
|
||||
});
|
||||
const inboxDir = path.join(getTeamsBasePath(), teamName, 'inboxes');
|
||||
await fs.mkdir(inboxDir, { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(inboxDir, 'bob.json'),
|
||||
`${JSON.stringify(
|
||||
[
|
||||
{
|
||||
from: 'user',
|
||||
to: 'bob',
|
||||
text: 'must not be delivered while parent team is stopped',
|
||||
timestamp: '2026-04-23T10:01:00.000Z',
|
||||
read: false,
|
||||
messageId: 'msg-stopped-orphaned-lane-bob',
|
||||
},
|
||||
],
|
||||
null,
|
||||
2
|
||||
)}\n`,
|
||||
'utf8'
|
||||
);
|
||||
const adapter = new FakeOpenCodeRuntimeAdapter('clean_success', {
|
||||
bob: 'confirmed',
|
||||
});
|
||||
const restartedService = new TeamProvisioningService();
|
||||
restartedService.setRuntimeAdapterRegistry(new TeamRuntimeAdapterRegistry([adapter]));
|
||||
|
||||
await expect(restartedService.scanOpenCodePromptDeliveryWatchdog(teamName)).resolves.toBe(0);
|
||||
expect(adapter.stopInputs).toHaveLength(1);
|
||||
expect(adapter.stopInputs[0]).toMatchObject({
|
||||
runId: 'orphaned-opencode-run-bob',
|
||||
teamName,
|
||||
laneId: 'secondary:opencode:bob',
|
||||
providerId: 'opencode',
|
||||
reason: 'cleanup',
|
||||
force: true,
|
||||
});
|
||||
await expect(readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName)).resolves.toMatchObject({
|
||||
lanes: {},
|
||||
});
|
||||
await expect(
|
||||
restartedService.relayOpenCodeMemberInboxMessages(teamName, 'bob')
|
||||
).resolves.toMatchObject({
|
||||
relayed: 0,
|
||||
failed: 1,
|
||||
lastDelivery: { delivered: false, reason: 'opencode_runtime_not_active' },
|
||||
});
|
||||
await expect(
|
||||
restartedService.deliverOpenCodeMemberMessage(teamName, {
|
||||
memberName: 'bob',
|
||||
text: 'direct message must not reach orphaned lane',
|
||||
messageId: 'msg-stopped-orphaned-direct-bob',
|
||||
})
|
||||
).resolves.toEqual({
|
||||
delivered: false,
|
||||
reason: 'opencode_runtime_not_active',
|
||||
});
|
||||
|
||||
expect(adapter.reconcileInputs).toEqual([]);
|
||||
expect(adapter.messageInputs).toEqual([]);
|
||||
});
|
||||
|
||||
it('does not recover missing mixed OpenCode lanes from persisted runtime evidence when parent team is stopped', async () => {
|
||||
const teamName = 'mixed-opencode-stopped-missing-lane-recovery-safe-e2e';
|
||||
await writeMixedTeamConfig({ teamName, projectPath });
|
||||
await writeTeamMeta(teamName, projectPath);
|
||||
await writeMembersMeta(teamName);
|
||||
await fs.writeFile(
|
||||
path.join(getTeamsBasePath(), teamName, 'launch-state.json'),
|
||||
`${JSON.stringify(
|
||||
createPersistedLaunchSnapshot({
|
||||
teamName,
|
||||
expectedMembers: ['alice', 'bob'],
|
||||
leadSessionId: 'lead-session',
|
||||
launchPhase: 'reconciled',
|
||||
members: {
|
||||
alice: {
|
||||
name: 'alice',
|
||||
providerId: 'codex',
|
||||
laneId: 'primary',
|
||||
laneKind: 'primary',
|
||||
laneOwnerProviderId: 'codex',
|
||||
launchState: 'confirmed_alive',
|
||||
agentToolAccepted: true,
|
||||
runtimeAlive: true,
|
||||
bootstrapConfirmed: true,
|
||||
hardFailure: false,
|
||||
lastEvaluatedAt: '2026-04-23T10:00:00.000Z',
|
||||
},
|
||||
bob: {
|
||||
name: 'bob',
|
||||
providerId: 'opencode',
|
||||
model: 'opencode/minimax-m2.5-free',
|
||||
laneId: 'secondary:opencode:bob',
|
||||
laneKind: 'secondary',
|
||||
laneOwnerProviderId: 'opencode',
|
||||
launchState: 'failed_to_start',
|
||||
agentToolAccepted: true,
|
||||
runtimeAlive: false,
|
||||
bootstrapConfirmed: false,
|
||||
hardFailure: true,
|
||||
hardFailureReason: 'OpenCode bridge reported member launch failure',
|
||||
runtimePid: 7743,
|
||||
runtimeSessionId: 'ses_bob_materialized',
|
||||
livenessKind: 'runtime_process_candidate',
|
||||
pidSource: 'opencode_bridge',
|
||||
lastEvaluatedAt: '2026-04-23T10:00:00.000Z',
|
||||
},
|
||||
},
|
||||
updatedAt: '2026-04-23T10:00:00.000Z',
|
||||
}),
|
||||
null,
|
||||
2
|
||||
)}\n`,
|
||||
'utf8'
|
||||
);
|
||||
const adapter = new FakeOpenCodeRuntimeAdapter('partial_pending', {
|
||||
bob: 'launching',
|
||||
});
|
||||
const restartedService = new TeamProvisioningService();
|
||||
restartedService.setRuntimeAdapterRegistry(new TeamRuntimeAdapterRegistry([adapter]));
|
||||
|
||||
await expect(
|
||||
readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName)
|
||||
).resolves.toMatchObject({ lanes: {} });
|
||||
await expect(
|
||||
restartedService.deliverOpenCodeMemberMessage(teamName, {
|
||||
memberName: 'bob',
|
||||
text: 'must not recover stopped parent team',
|
||||
messageId: 'msg-stopped-missing-lane-bob',
|
||||
})
|
||||
).resolves.toEqual({
|
||||
delivered: false,
|
||||
reason: 'opencode_runtime_not_active',
|
||||
});
|
||||
|
||||
expect(adapter.reconcileInputs).toEqual([]);
|
||||
expect(adapter.messageInputs).toEqual([]);
|
||||
await expect(readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName)).resolves.toMatchObject({
|
||||
lanes: {},
|
||||
});
|
||||
});
|
||||
|
||||
it('recovers a missing mixed OpenCode lane index from materialized persisted runtime evidence before direct delivery', async () => {
|
||||
const teamName = 'mixed-opencode-direct-message-recovers-missing-lane-safe-e2e';
|
||||
await writeMixedTeamConfig({ teamName, projectPath });
|
||||
|
|
@ -11093,6 +11254,7 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
bob: 'launching',
|
||||
tom: 'failed',
|
||||
});
|
||||
await writeAliveProcessRegistry(teamName);
|
||||
const restartedService = new TeamProvisioningService();
|
||||
restartedService.setRuntimeAdapterRegistry(new TeamRuntimeAdapterRegistry([adapter]));
|
||||
|
||||
|
|
@ -11192,6 +11354,7 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
const adapter = new FakeOpenCodeRuntimeAdapter('clean_success', {
|
||||
bob: 'confirmed',
|
||||
});
|
||||
await writeAliveProcessRegistry(teamName);
|
||||
const restartedService = new TeamProvisioningService();
|
||||
restartedService.setRuntimeAdapterRegistry(new TeamRuntimeAdapterRegistry([adapter]));
|
||||
|
||||
|
|
@ -11326,6 +11489,7 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
bob: 'launching',
|
||||
tom: 'failed',
|
||||
});
|
||||
await writeAliveProcessRegistry(teamName);
|
||||
const restartedService = new TeamProvisioningService();
|
||||
restartedService.setRuntimeAdapterRegistry(new TeamRuntimeAdapterRegistry([adapter]));
|
||||
const scheduledWatchdogJobs: unknown[] = [];
|
||||
|
|
@ -11466,6 +11630,7 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
bob: 'launching',
|
||||
tom: 'confirmed',
|
||||
});
|
||||
await writeAliveProcessRegistry(teamName);
|
||||
const restartedService = new TeamProvisioningService();
|
||||
restartedService.setRuntimeAdapterRegistry(new TeamRuntimeAdapterRegistry([adapter]));
|
||||
const scheduledWatchdogJobs: unknown[] = [];
|
||||
|
|
@ -17231,6 +17396,27 @@ function trackLiveRun(svc: TeamProvisioningService, run: any): void {
|
|||
(svc as any).aliveRunByTeam.set(run.teamName, run.runId);
|
||||
}
|
||||
|
||||
async function writeAliveProcessRegistry(teamName: string): Promise<void> {
|
||||
const teamDir = path.join(getTeamsBasePath(), teamName);
|
||||
await fs.mkdir(teamDir, { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(teamDir, 'processes.json'),
|
||||
`${JSON.stringify(
|
||||
[
|
||||
{
|
||||
id: 'lead-process',
|
||||
label: 'Team Lead',
|
||||
pid: process.pid,
|
||||
registeredAt: '2026-04-23T10:00:00.000Z',
|
||||
},
|
||||
],
|
||||
null,
|
||||
2
|
||||
)}\n`,
|
||||
'utf8'
|
||||
);
|
||||
}
|
||||
|
||||
function expectDirectChildKillCount(actual: number, expected: number): void {
|
||||
// Windows uses taskkill.exe for process-tree termination, so fake child.kill is not called.
|
||||
expect(actual).toBe(process.platform === 'win32' ? 0 : expected);
|
||||
|
|
|
|||
|
|
@ -275,6 +275,27 @@ function writeBootstrapState(
|
|||
);
|
||||
}
|
||||
|
||||
function writeAliveProcessRegistry(teamName: string): void {
|
||||
const teamDir = path.join(tempTeamsBase, teamName);
|
||||
fs.mkdirSync(teamDir, { recursive: true });
|
||||
fs.writeFileSync(
|
||||
path.join(teamDir, 'processes.json'),
|
||||
`${JSON.stringify(
|
||||
[
|
||||
{
|
||||
id: 'lead-process',
|
||||
label: 'Team Lead',
|
||||
pid: process.pid,
|
||||
registeredAt: '2026-04-23T10:00:00.000Z',
|
||||
},
|
||||
],
|
||||
null,
|
||||
2
|
||||
)}\n`,
|
||||
'utf8'
|
||||
);
|
||||
}
|
||||
|
||||
function writeTeamMeta(teamName: string, overrides: Record<string, unknown> = {}): void {
|
||||
const teamDir = path.join(tempTeamsBase, teamName);
|
||||
fs.mkdirSync(teamDir, { recursive: true });
|
||||
|
|
@ -434,6 +455,7 @@ describe('TeamProvisioningService', () => {
|
|||
fs.mkdirSync(tempTeamsBase, { recursive: true });
|
||||
fs.mkdirSync(tempTasksBase, { recursive: true });
|
||||
fs.mkdirSync(tempProjectsBase, { recursive: true });
|
||||
writeAliveProcessRegistry('team-a');
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
|
|
@ -3134,6 +3156,7 @@ describe('TeamProvisioningService', () => {
|
|||
);
|
||||
|
||||
(svc as any).getTrackedRunId = vi.fn(() => null);
|
||||
(svc as any).canDeliverToOpenCodeRuntimeForTeam = vi.fn(() => true);
|
||||
(svc as any).resolveCurrentOpenCodeRuntimeRunId = vi.fn(async () => 'opencode-run-bob');
|
||||
(svc as any).isOpenCodeRuntimeLaneIndexActive = vi.fn(async () => true);
|
||||
(svc as any).configReader = {
|
||||
|
|
@ -4416,9 +4439,11 @@ describe('TeamProvisioningService', () => {
|
|||
sessionId: 'oc-session-bob',
|
||||
prePromptCursor: 'cursor-before',
|
||||
responseObservation: {
|
||||
state: sendMessageToMember.mock.calls.length === 1 ? 'responded_non_visible_tool' : 'pending',
|
||||
state:
|
||||
sendMessageToMember.mock.calls.length === 1 ? 'responded_non_visible_tool' : 'pending',
|
||||
deliveredUserMessageId: 'oc-user-ask',
|
||||
assistantMessageId: sendMessageToMember.mock.calls.length === 1 ? 'oc-assistant-read' : null,
|
||||
assistantMessageId:
|
||||
sendMessageToMember.mock.calls.length === 1 ? 'oc-assistant-read' : null,
|
||||
toolCallNames: sendMessageToMember.mock.calls.length === 1 ? ['read'] : [],
|
||||
visibleMessageToolCallId: null,
|
||||
visibleReplyMessageId: null,
|
||||
|
|
@ -7909,9 +7934,7 @@ describe('TeamProvisioningService', () => {
|
|||
}
|
||||
);
|
||||
|
||||
const config = JSON.parse(
|
||||
fs.readFileSync(path.join(teamDir, 'config.json'), 'utf8')
|
||||
) as {
|
||||
const config = JSON.parse(fs.readFileSync(path.join(teamDir, 'config.json'), 'utf8')) as {
|
||||
leadSessionId?: string;
|
||||
projectPath?: string;
|
||||
members: Array<{
|
||||
|
|
@ -7996,7 +8019,9 @@ describe('TeamProvisioningService', () => {
|
|||
};
|
||||
});
|
||||
|
||||
const { svc, membersMetaStore } = createSafeLaunchService({ memberWorktreeManager: worktreeManager });
|
||||
const { svc, membersMetaStore } = createSafeLaunchService({
|
||||
memberWorktreeManager: worktreeManager,
|
||||
});
|
||||
svc.setRuntimeAdapterRegistry(
|
||||
new TeamRuntimeAdapterRegistry([
|
||||
{
|
||||
|
|
@ -9934,6 +9959,49 @@ describe('TeamProvisioningService', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('clears registered-only stale failure when a verified runtime process appears later', async () => {
|
||||
const svc = new TeamProvisioningService();
|
||||
(svc as any).getLiveTeamAgentRuntimeMetadata = vi.fn(
|
||||
async () =>
|
||||
new Map([
|
||||
[
|
||||
'tom',
|
||||
{
|
||||
alive: true,
|
||||
model: 'gpt-5.4',
|
||||
livenessKind: 'runtime_process',
|
||||
runtimeDiagnostic: 'verified runtime process detected',
|
||||
},
|
||||
],
|
||||
])
|
||||
);
|
||||
|
||||
const result = await (svc as any).attachLiveRuntimeMetadataToStatuses('forge-labs-10', {
|
||||
tom: createMemberSpawnStatusEntry({
|
||||
status: 'error',
|
||||
launchState: 'failed_to_start',
|
||||
error: 'registered runtime metadata without live process',
|
||||
hardFailure: true,
|
||||
hardFailureReason: 'registered runtime metadata without live process',
|
||||
livenessKind: 'registered_only',
|
||||
runtimeDiagnostic: 'registered runtime metadata without live process',
|
||||
}),
|
||||
});
|
||||
|
||||
expect(result.tom).toMatchObject({
|
||||
status: 'online',
|
||||
launchState: 'runtime_pending_bootstrap',
|
||||
runtimeAlive: true,
|
||||
hardFailure: false,
|
||||
hardFailureReason: undefined,
|
||||
error: undefined,
|
||||
runtimeModel: 'gpt-5.4',
|
||||
livenessKind: 'runtime_process',
|
||||
runtimeDiagnostic: 'verified runtime process detected',
|
||||
livenessSource: 'process',
|
||||
});
|
||||
});
|
||||
|
||||
it('does not clear OpenCode bridge launch failure from process-only liveness', async () => {
|
||||
const svc = new TeamProvisioningService();
|
||||
(svc as any).getLiveTeamAgentRuntimeMetadata = vi.fn(
|
||||
|
|
@ -10622,6 +10690,85 @@ describe('TeamProvisioningService', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('confirms a teammate from bootstrap transcript stored under its worktree cwd', async () => {
|
||||
const teamName = 'worktree-bootstrap-transcript-team';
|
||||
const leadSessionId = 'lead-session';
|
||||
const projectPath = '/Users/test/proj';
|
||||
const worktreePath = `${projectPath}/.claude/worktrees/team-${teamName}-tom-12345678`;
|
||||
const acceptedAt = new Date(Date.now() - 90_000).toISOString();
|
||||
const observedAt = new Date(Date.now() - 30_000).toISOString();
|
||||
const teamDir = path.join(tempTeamsBase, teamName);
|
||||
fs.mkdirSync(teamDir, { recursive: true });
|
||||
fs.writeFileSync(
|
||||
path.join(teamDir, 'config.json'),
|
||||
JSON.stringify({
|
||||
name: teamName,
|
||||
projectPath,
|
||||
leadSessionId,
|
||||
members: [
|
||||
{ name: 'team-lead', agentType: 'team-lead', cwd: projectPath },
|
||||
{ name: 'tom', providerId: 'codex', model: 'gpt-5.4', cwd: worktreePath },
|
||||
{ name: 'bob', providerId: 'codex', model: 'gpt-5.4-mini', cwd: projectPath },
|
||||
],
|
||||
}),
|
||||
'utf8'
|
||||
);
|
||||
writeLaunchState(teamName, leadSessionId, {
|
||||
tom: {
|
||||
providerId: 'codex',
|
||||
laneId: 'primary',
|
||||
laneKind: 'primary',
|
||||
laneOwnerProviderId: 'codex',
|
||||
launchState: 'failed_to_start',
|
||||
agentToolAccepted: true,
|
||||
runtimeAlive: false,
|
||||
bootstrapConfirmed: false,
|
||||
hardFailure: true,
|
||||
hardFailureReason: 'registered runtime metadata without live process',
|
||||
firstSpawnAcceptedAt: acceptedAt,
|
||||
lastEvaluatedAt: acceptedAt,
|
||||
},
|
||||
});
|
||||
const worktreeProjectDir = path.join(tempProjectsBase, encodePath(worktreePath));
|
||||
fs.mkdirSync(worktreeProjectDir, { recursive: true });
|
||||
fs.writeFileSync(
|
||||
path.join(worktreeProjectDir, 'tom-session.jsonl'),
|
||||
`${JSON.stringify({
|
||||
type: 'user',
|
||||
teamName,
|
||||
agentName: 'tom',
|
||||
timestamp: observedAt,
|
||||
cwd: worktreePath,
|
||||
message: {
|
||||
role: 'user',
|
||||
content: [
|
||||
{
|
||||
type: 'tool_result',
|
||||
tool_use_id: 'item_0',
|
||||
content: `Member briefing for tom on team "${teamName}" (${teamName}).\nRole: developer.`,
|
||||
},
|
||||
],
|
||||
},
|
||||
})}\n`,
|
||||
'utf8'
|
||||
);
|
||||
|
||||
const svc = new TeamProvisioningService();
|
||||
(svc as any).getLiveTeamAgentRuntimeMetadata = vi.fn(async () => new Map());
|
||||
const result = await svc.getMemberSpawnStatuses(teamName);
|
||||
|
||||
expect(result.statuses.tom).toMatchObject({
|
||||
status: 'online',
|
||||
launchState: 'confirmed_alive',
|
||||
agentToolAccepted: true,
|
||||
runtimeAlive: false,
|
||||
bootstrapConfirmed: true,
|
||||
hardFailure: false,
|
||||
hardFailureReason: undefined,
|
||||
lastHeartbeatAt: observedAt,
|
||||
});
|
||||
});
|
||||
|
||||
it('treats suffixed persisted heartbeat senders as the expected member during reconcile', async () => {
|
||||
const teamName = 'suffixed-heartbeat-reconcile-team';
|
||||
const svc = new TeamProvisioningService();
|
||||
|
|
|
|||
Loading…
Reference in a new issue