fix(team): recover stale mixed opencode lane state
This commit is contained in:
parent
7cdee429ec
commit
6211fd95ab
4 changed files with 569 additions and 31 deletions
|
|
@ -166,6 +166,7 @@ import {
|
||||||
getOpenCodeTeamRuntimeDirectory,
|
getOpenCodeTeamRuntimeDirectory,
|
||||||
migrateLegacyOpenCodeRuntimeState,
|
migrateLegacyOpenCodeRuntimeState,
|
||||||
readOpenCodeRuntimeLaneIndex,
|
readOpenCodeRuntimeLaneIndex,
|
||||||
|
recoverStaleOpenCodeRuntimeLaneIndexEntry,
|
||||||
removeOpenCodeRuntimeLaneIndexEntry,
|
removeOpenCodeRuntimeLaneIndexEntry,
|
||||||
upsertOpenCodeRuntimeLaneIndexEntry,
|
upsertOpenCodeRuntimeLaneIndexEntry,
|
||||||
} from './opencode/store/OpenCodeRuntimeManifestEvidenceReader';
|
} from './opencode/store/OpenCodeRuntimeManifestEvidenceReader';
|
||||||
|
|
@ -6063,12 +6064,7 @@ export class TeamProvisioningService {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
if (!this.isCurrentTrackedRun(run)) return;
|
if (!this.isCurrentTrackedRun(run)) return;
|
||||||
this.teamChangeEmitter?.({
|
this.emitMemberSpawnChange(run, memberName);
|
||||||
type: 'member-spawn',
|
|
||||||
teamName: run.teamName,
|
|
||||||
runId: run.runId,
|
|
||||||
detail: memberName,
|
|
||||||
});
|
|
||||||
if (run.isLaunch) {
|
if (run.isLaunch) {
|
||||||
void this.persistLaunchStateSnapshot(run, run.provisioningComplete ? 'finished' : 'active');
|
void this.persistLaunchStateSnapshot(run, run.provisioningComplete ? 'finished' : 'active');
|
||||||
}
|
}
|
||||||
|
|
@ -6115,13 +6111,15 @@ export class TeamProvisioningService {
|
||||||
await this.persistLaunchStateSnapshot(run, run.provisioningComplete ? 'finished' : 'active');
|
await this.persistLaunchStateSnapshot(run, run.provisioningComplete ? 'finished' : 'active');
|
||||||
|
|
||||||
const persisted = await this.launchStateStore.read(teamName);
|
const persisted = await this.launchStateStore.read(teamName);
|
||||||
const liveSnapshot = snapshotFromRuntimeMemberStatuses({
|
const liveSnapshot =
|
||||||
teamName: run.teamName,
|
this.buildLiveLaunchSnapshotForRun(run, run.provisioningComplete ? 'finished' : 'active') ??
|
||||||
expectedMembers: run.expectedMembers,
|
snapshotFromRuntimeMemberStatuses({
|
||||||
leadSessionId: run.detectedSessionId ?? undefined,
|
teamName: run.teamName,
|
||||||
launchPhase: run.provisioningComplete ? 'finished' : 'active',
|
expectedMembers: run.expectedMembers,
|
||||||
statuses: this.buildRuntimeSpawnStatusRecord(run),
|
leadSessionId: run.detectedSessionId ?? undefined,
|
||||||
});
|
launchPhase: run.provisioningComplete ? 'finished' : 'active',
|
||||||
|
statuses: this.buildRuntimeSpawnStatusRecord(run),
|
||||||
|
});
|
||||||
const snapshot = persisted ?? liveSnapshot;
|
const snapshot = persisted ?? liveSnapshot;
|
||||||
const statuses = await this.attachLiveRuntimeMetadataToStatuses(
|
const statuses = await this.attachLiveRuntimeMetadataToStatuses(
|
||||||
teamName,
|
teamName,
|
||||||
|
|
@ -9689,6 +9687,9 @@ export class TeamProvisioningService {
|
||||||
this.provisioningRunByTeam.set(request.teamName, runId);
|
this.provisioningRunByTeam.set(request.teamName, runId);
|
||||||
run.onProgress(run.progress);
|
run.onProgress(run.progress);
|
||||||
await this.clearPersistedLaunchState(request.teamName);
|
await this.clearPersistedLaunchState(request.teamName);
|
||||||
|
for (const lane of run.mixedSecondaryLanes ?? []) {
|
||||||
|
await this.publishMixedSecondaryLaneStatusChange(run, lane);
|
||||||
|
}
|
||||||
|
|
||||||
// Read existing tasks to include in teammate prompts for work resumption
|
// Read existing tasks to include in teammate prompts for work resumption
|
||||||
const taskReader = new TeamTaskReader();
|
const taskReader = new TeamTaskReader();
|
||||||
|
|
@ -11813,6 +11814,53 @@ export class TeamProvisioningService {
|
||||||
return statuses;
|
return statuses;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private buildLiveLaunchSnapshotForRun(
|
||||||
|
run: ProvisioningRun,
|
||||||
|
launchPhase: PersistedTeamLaunchPhase = run.provisioningComplete ? 'finished' : 'active'
|
||||||
|
): PersistedTeamLaunchSnapshot | null {
|
||||||
|
const mixedSnapshot = this.buildMixedPersistedLaunchSnapshotForRun(run, launchPhase);
|
||||||
|
if (mixedSnapshot) {
|
||||||
|
return mixedSnapshot;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!run.isLaunch || !run.expectedMembers || run.expectedMembers.length === 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return snapshotFromRuntimeMemberStatuses({
|
||||||
|
teamName: run.teamName,
|
||||||
|
expectedMembers: run.expectedMembers,
|
||||||
|
leadSessionId: run.detectedSessionId ?? undefined,
|
||||||
|
launchPhase,
|
||||||
|
statuses: this.buildRuntimeSpawnStatusRecord(run),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private emitMemberSpawnChange(
|
||||||
|
run: Pick<ProvisioningRun, 'teamName' | 'runId'>,
|
||||||
|
memberName: string
|
||||||
|
) {
|
||||||
|
this.teamChangeEmitter?.({
|
||||||
|
type: 'member-spawn',
|
||||||
|
teamName: run.teamName,
|
||||||
|
runId: run.runId,
|
||||||
|
detail: memberName,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private async publishMixedSecondaryLaneStatusChange(
|
||||||
|
run: ProvisioningRun,
|
||||||
|
lane: MixedSecondaryRuntimeLaneState
|
||||||
|
): Promise<void> {
|
||||||
|
if (run.isLaunch) {
|
||||||
|
await this.persistLaunchStateSnapshot(run, this.getMixedSecondaryLaunchPhase(run));
|
||||||
|
}
|
||||||
|
if (!this.isCurrentTrackedRun(run)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
this.emitMemberSpawnChange(run, lane.member.name);
|
||||||
|
}
|
||||||
|
|
||||||
private buildMixedPersistedLaunchSnapshotForRun(
|
private buildMixedPersistedLaunchSnapshotForRun(
|
||||||
run: ProvisioningRun,
|
run: ProvisioningRun,
|
||||||
launchPhase: PersistedTeamLaunchPhase
|
launchPhase: PersistedTeamLaunchPhase
|
||||||
|
|
@ -11888,27 +11936,14 @@ export class TeamProvisioningService {
|
||||||
? 'finished'
|
? 'finished'
|
||||||
: 'active'
|
: 'active'
|
||||||
): Promise<PersistedTeamLaunchSnapshot | null> {
|
): Promise<PersistedTeamLaunchSnapshot | null> {
|
||||||
const mixedSnapshot = this.buildMixedPersistedLaunchSnapshotForRun(run, launchPhase);
|
const snapshot = this.buildLiveLaunchSnapshotForRun(run, launchPhase);
|
||||||
if (mixedSnapshot) {
|
if (!snapshot) {
|
||||||
await this.launchStateStore.write(run.teamName, mixedSnapshot);
|
|
||||||
return mixedSnapshot;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!run.isLaunch || !run.expectedMembers || run.expectedMembers.length === 0) {
|
|
||||||
if (run.isLaunch) {
|
if (run.isLaunch) {
|
||||||
await this.clearPersistedLaunchState(run.teamName);
|
await this.clearPersistedLaunchState(run.teamName);
|
||||||
}
|
}
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
const snapshot = snapshotFromRuntimeMemberStatuses({
|
|
||||||
teamName: run.teamName,
|
|
||||||
expectedMembers: run.expectedMembers,
|
|
||||||
leadSessionId: run.detectedSessionId ?? undefined,
|
|
||||||
launchPhase,
|
|
||||||
statuses: this.buildRuntimeSpawnStatusRecord(run),
|
|
||||||
});
|
|
||||||
|
|
||||||
if (snapshot.teamLaunchState === 'clean_success' && launchPhase !== 'active') {
|
if (snapshot.teamLaunchState === 'clean_success' && launchPhase !== 'active') {
|
||||||
await this.clearPersistedLaunchState(run.teamName);
|
await this.clearPersistedLaunchState(run.teamName);
|
||||||
return null;
|
return null;
|
||||||
|
|
@ -11949,6 +11984,7 @@ export class TeamProvisioningService {
|
||||||
};
|
};
|
||||||
lane.warnings = [];
|
lane.warnings = [];
|
||||||
lane.diagnostics = [message];
|
lane.diagnostics = [message];
|
||||||
|
await this.publishMixedSecondaryLaneStatusChange(run, lane);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -11977,8 +12013,7 @@ export class TeamProvisioningService {
|
||||||
memberName: lane.member.name,
|
memberName: lane.member.name,
|
||||||
cwd: run.request.cwd,
|
cwd: run.request.cwd,
|
||||||
});
|
});
|
||||||
|
await this.publishMixedSecondaryLaneStatusChange(run, lane);
|
||||||
await this.persistLaunchStateSnapshot(run, this.getMixedSecondaryLaunchPhase(run));
|
|
||||||
const previousLaunchState = await this.launchStateStore.read(run.teamName);
|
const previousLaunchState = await this.launchStateStore.read(run.teamName);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
|
@ -12050,7 +12085,7 @@ export class TeamProvisioningService {
|
||||||
this.deleteSecondaryRuntimeRun(run.teamName, lane.laneId);
|
this.deleteSecondaryRuntimeRun(run.teamName, lane.laneId);
|
||||||
}
|
}
|
||||||
|
|
||||||
await this.persistLaunchStateSnapshot(run, this.getMixedSecondaryLaunchPhase(run));
|
await this.publishMixedSecondaryLaneStatusChange(run, lane);
|
||||||
}
|
}
|
||||||
|
|
||||||
private async stopSingleMixedSecondaryRuntimeLane(
|
private async stopSingleMixedSecondaryRuntimeLane(
|
||||||
|
|
@ -12129,6 +12164,7 @@ export class TeamProvisioningService {
|
||||||
diagnostics: ['OpenCode runtime adapter is not registered for mixed team launch.'],
|
diagnostics: ['OpenCode runtime adapter is not registered for mixed team launch.'],
|
||||||
};
|
};
|
||||||
lane.diagnostics = lane.result.diagnostics;
|
lane.diagnostics = lane.result.diagnostics;
|
||||||
|
await this.publishMixedSecondaryLaneStatusChange(run, lane);
|
||||||
}
|
}
|
||||||
return this.persistLaunchStateSnapshot(run, 'finished');
|
return this.persistLaunchStateSnapshot(run, 'finished');
|
||||||
}
|
}
|
||||||
|
|
@ -12140,12 +12176,187 @@ export class TeamProvisioningService {
|
||||||
return this.persistLaunchStateSnapshot(run, this.getMixedSecondaryLaunchPhase(run));
|
return this.persistLaunchStateSnapshot(run, this.getMixedSecondaryLaunchPhase(run));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async recoverStaleMixedSecondaryLaunchSnapshot(
|
||||||
|
teamName: string,
|
||||||
|
bootstrapSnapshot: PersistedTeamLaunchSnapshot | null,
|
||||||
|
persistedSnapshot: PersistedTeamLaunchSnapshot | null
|
||||||
|
): Promise<PersistedTeamLaunchSnapshot | null> {
|
||||||
|
if (persistedSnapshot && this.hasMixedLaunchMetadata(persistedSnapshot)) {
|
||||||
|
return persistedSnapshot;
|
||||||
|
}
|
||||||
|
|
||||||
|
const teamMeta = await this.teamMetaStore.getMeta(teamName).catch(() => null);
|
||||||
|
const leadProviderId = normalizeOptionalTeamProviderId(teamMeta?.providerId);
|
||||||
|
if (!leadProviderId || leadProviderId === 'opencode') {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const membersMeta = await this.membersMetaStore.getMeta(teamName).catch(() => null);
|
||||||
|
const activeMembers = (membersMeta?.members ?? []).filter(
|
||||||
|
(member) => !member.removedAt && !isLeadMember({ name: member.name })
|
||||||
|
);
|
||||||
|
if (activeMembers.length === 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const laneIndex = await readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName).catch(
|
||||||
|
() => ({
|
||||||
|
version: 1 as const,
|
||||||
|
updatedAt: nowIso(),
|
||||||
|
lanes: {} as Record<
|
||||||
|
string,
|
||||||
|
{
|
||||||
|
laneId: string;
|
||||||
|
state: 'active' | 'stopped' | 'degraded';
|
||||||
|
updatedAt: string;
|
||||||
|
diagnostics?: string[];
|
||||||
|
}
|
||||||
|
>,
|
||||||
|
})
|
||||||
|
);
|
||||||
|
const bootstrapStatuses = snapshotToMemberSpawnStatuses(bootstrapSnapshot);
|
||||||
|
const leadDefaults = {
|
||||||
|
providerId: leadProviderId,
|
||||||
|
providerBackendId:
|
||||||
|
migrateProviderBackendId(
|
||||||
|
leadProviderId,
|
||||||
|
teamMeta?.providerBackendId ?? membersMeta?.providerBackendId
|
||||||
|
) ?? null,
|
||||||
|
selectedFastMode: teamMeta?.fastMode,
|
||||||
|
resolvedFastMode:
|
||||||
|
typeof teamMeta?.launchIdentity?.resolvedFastMode === 'boolean'
|
||||||
|
? teamMeta.launchIdentity.resolvedFastMode
|
||||||
|
: null,
|
||||||
|
launchIdentity: teamMeta?.launchIdentity ?? null,
|
||||||
|
};
|
||||||
|
const primaryMembers: TeamMember[] = [];
|
||||||
|
const secondaryMembers: Array<{
|
||||||
|
laneId: string;
|
||||||
|
member: TeamMember;
|
||||||
|
leadDefaults: typeof leadDefaults;
|
||||||
|
evidence?: {
|
||||||
|
launchState?: MemberLaunchState;
|
||||||
|
agentToolAccepted?: boolean;
|
||||||
|
runtimeAlive?: boolean;
|
||||||
|
bootstrapConfirmed?: boolean;
|
||||||
|
hardFailure?: boolean;
|
||||||
|
hardFailureReason?: string;
|
||||||
|
diagnostics?: string[];
|
||||||
|
};
|
||||||
|
pendingReason?: string;
|
||||||
|
}> = [];
|
||||||
|
let recoveredAny = false;
|
||||||
|
|
||||||
|
for (const member of activeMembers) {
|
||||||
|
const laneIdentity = buildPlannedMemberLaneIdentity({
|
||||||
|
leadProviderId,
|
||||||
|
member: {
|
||||||
|
name: member.name,
|
||||||
|
providerId: normalizeOptionalTeamProviderId(member.providerId),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
if (
|
||||||
|
laneIdentity.laneKind !== 'secondary' ||
|
||||||
|
laneIdentity.laneOwnerProviderId !== 'opencode'
|
||||||
|
) {
|
||||||
|
primaryMembers.push(member);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let laneEntry = laneIndex.lanes[laneIdentity.laneId];
|
||||||
|
if (laneEntry?.state === 'active') {
|
||||||
|
const recovery = await recoverStaleOpenCodeRuntimeLaneIndexEntry({
|
||||||
|
teamsBasePath: getTeamsBasePath(),
|
||||||
|
teamName,
|
||||||
|
laneId: laneIdentity.laneId,
|
||||||
|
});
|
||||||
|
if (recovery.stale) {
|
||||||
|
recoveredAny = true;
|
||||||
|
laneEntry = {
|
||||||
|
laneId: laneIdentity.laneId,
|
||||||
|
state: 'degraded',
|
||||||
|
updatedAt: nowIso(),
|
||||||
|
diagnostics: recovery.diagnostics,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (laneEntry?.state === 'degraded') {
|
||||||
|
recoveredAny = true;
|
||||||
|
const diagnostics = laneEntry.diagnostics?.length
|
||||||
|
? [...laneEntry.diagnostics]
|
||||||
|
: [`OpenCode lane ${laneIdentity.laneId} is degraded and requires stop + relaunch.`];
|
||||||
|
secondaryMembers.push({
|
||||||
|
laneId: laneIdentity.laneId,
|
||||||
|
member,
|
||||||
|
leadDefaults,
|
||||||
|
evidence: {
|
||||||
|
launchState: 'failed_to_start',
|
||||||
|
agentToolAccepted: false,
|
||||||
|
runtimeAlive: false,
|
||||||
|
bootstrapConfirmed: false,
|
||||||
|
hardFailure: true,
|
||||||
|
hardFailureReason: diagnostics[0],
|
||||||
|
diagnostics,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
secondaryMembers.push({
|
||||||
|
laneId: laneIdentity.laneId,
|
||||||
|
member,
|
||||||
|
leadDefaults,
|
||||||
|
pendingReason: 'Waiting for OpenCode secondary lane recovery.',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!recoveredAny) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const primaryStatuses = Object.fromEntries(
|
||||||
|
primaryMembers.map((member) => [
|
||||||
|
member.name,
|
||||||
|
bootstrapStatuses[member.name] ?? createInitialMemberSpawnStatusEntry(),
|
||||||
|
])
|
||||||
|
);
|
||||||
|
const recoveredSnapshot = this.runtimeLaneCoordinator.buildAggregateLaunchSnapshot({
|
||||||
|
teamName,
|
||||||
|
leadSessionId: persistedSnapshot?.leadSessionId ?? bootstrapSnapshot?.leadSessionId,
|
||||||
|
launchPhase:
|
||||||
|
persistedSnapshot?.launchPhase === 'active'
|
||||||
|
? 'active'
|
||||||
|
: bootstrapSnapshot?.launchPhase === 'active'
|
||||||
|
? 'active'
|
||||||
|
: 'reconciled',
|
||||||
|
leadDefaults,
|
||||||
|
primaryMembers,
|
||||||
|
primaryStatuses,
|
||||||
|
secondaryMembers,
|
||||||
|
});
|
||||||
|
await this.launchStateStore.write(teamName, recoveredSnapshot);
|
||||||
|
return recoveredSnapshot;
|
||||||
|
}
|
||||||
|
|
||||||
private async reconcilePersistedLaunchState(teamName: string): Promise<{
|
private async reconcilePersistedLaunchState(teamName: string): Promise<{
|
||||||
snapshot: ReturnType<typeof createPersistedLaunchSnapshot> | null;
|
snapshot: ReturnType<typeof createPersistedLaunchSnapshot> | null;
|
||||||
statuses: Record<string, MemberSpawnStatusEntry>;
|
statuses: Record<string, MemberSpawnStatusEntry>;
|
||||||
}> {
|
}> {
|
||||||
const bootstrapSnapshot = await readBootstrapLaunchSnapshot(teamName);
|
const bootstrapSnapshot = await readBootstrapLaunchSnapshot(teamName);
|
||||||
const persisted = await this.launchStateStore.read(teamName);
|
const persisted = await this.launchStateStore.read(teamName);
|
||||||
|
const recoveredMixedSnapshot = await this.recoverStaleMixedSecondaryLaunchSnapshot(
|
||||||
|
teamName,
|
||||||
|
bootstrapSnapshot,
|
||||||
|
persisted
|
||||||
|
);
|
||||||
|
if (recoveredMixedSnapshot) {
|
||||||
|
return {
|
||||||
|
snapshot: recoveredMixedSnapshot,
|
||||||
|
statuses: snapshotToMemberSpawnStatuses(recoveredMixedSnapshot),
|
||||||
|
};
|
||||||
|
}
|
||||||
const preferredSnapshot = choosePreferredLaunchSnapshot(bootstrapSnapshot, persisted);
|
const preferredSnapshot = choosePreferredLaunchSnapshot(bootstrapSnapshot, persisted);
|
||||||
if (preferredSnapshot && preferredSnapshot === bootstrapSnapshot) {
|
if (preferredSnapshot && preferredSnapshot === bootstrapSnapshot) {
|
||||||
return {
|
return {
|
||||||
|
|
|
||||||
|
|
@ -153,6 +153,37 @@ export function getOpenCodeRuntimeManifestPath(
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function inspectOpenCodeRuntimeLaneStorage(params: {
|
||||||
|
teamsBasePath: string;
|
||||||
|
teamName: string;
|
||||||
|
laneId: string;
|
||||||
|
}): Promise<{
|
||||||
|
laneDirectoryExists: boolean;
|
||||||
|
hasStateOnDisk: boolean;
|
||||||
|
fileNames: string[];
|
||||||
|
}> {
|
||||||
|
const laneDir = getOpenCodeTeamRuntimeLaneDirectory(
|
||||||
|
params.teamsBasePath,
|
||||||
|
params.teamName,
|
||||||
|
params.laneId
|
||||||
|
);
|
||||||
|
const laneDirectoryExists = await fileExists(laneDir);
|
||||||
|
if (!laneDirectoryExists) {
|
||||||
|
return {
|
||||||
|
laneDirectoryExists: false,
|
||||||
|
hasStateOnDisk: false,
|
||||||
|
fileNames: [],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const fileNames = (await readdir(laneDir).catch(() => [] as string[])).sort();
|
||||||
|
return {
|
||||||
|
laneDirectoryExists: true,
|
||||||
|
hasStateOnDisk: fileNames.length > 0,
|
||||||
|
fileNames,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export function getOpenCodeLaneScopedRuntimeFilePath(params: {
|
export function getOpenCodeLaneScopedRuntimeFilePath(params: {
|
||||||
teamsBasePath: string;
|
teamsBasePath: string;
|
||||||
teamName: string;
|
teamName: string;
|
||||||
|
|
@ -284,6 +315,51 @@ export async function clearOpenCodeRuntimeLaneStorage(params: {
|
||||||
await removeOpenCodeRuntimeLaneIndexEntry(params);
|
await removeOpenCodeRuntimeLaneIndexEntry(params);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function recoverStaleOpenCodeRuntimeLaneIndexEntry(params: {
|
||||||
|
teamsBasePath: string;
|
||||||
|
teamName: string;
|
||||||
|
laneId: string;
|
||||||
|
}): Promise<{
|
||||||
|
stale: boolean;
|
||||||
|
degraded: boolean;
|
||||||
|
diagnostics: string[];
|
||||||
|
}> {
|
||||||
|
const index = await readOpenCodeRuntimeLaneIndex(params.teamsBasePath, params.teamName);
|
||||||
|
const entry = index.lanes[params.laneId];
|
||||||
|
if (!entry || entry.state !== 'active') {
|
||||||
|
return {
|
||||||
|
stale: false,
|
||||||
|
degraded: false,
|
||||||
|
diagnostics: [],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const storage = await inspectOpenCodeRuntimeLaneStorage(params);
|
||||||
|
if (storage.hasStateOnDisk) {
|
||||||
|
return {
|
||||||
|
stale: false,
|
||||||
|
degraded: false,
|
||||||
|
diagnostics: [],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const diagnostics = [
|
||||||
|
`OpenCode lane ${params.laneId} is marked active in lanes.json, but no lane state exists on disk.`,
|
||||||
|
];
|
||||||
|
await upsertOpenCodeRuntimeLaneIndexEntry({
|
||||||
|
teamsBasePath: params.teamsBasePath,
|
||||||
|
teamName: params.teamName,
|
||||||
|
laneId: params.laneId,
|
||||||
|
state: 'degraded',
|
||||||
|
diagnostics,
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
stale: true,
|
||||||
|
degraded: true,
|
||||||
|
diagnostics,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export async function migrateLegacyOpenCodeRuntimeState(params: {
|
export async function migrateLegacyOpenCodeRuntimeState(params: {
|
||||||
teamsBasePath: string;
|
teamsBasePath: string;
|
||||||
teamName: string;
|
teamName: string;
|
||||||
|
|
|
||||||
|
|
@ -9,8 +9,10 @@ import {
|
||||||
getOpenCodeLaneScopedRuntimeFilePath,
|
getOpenCodeLaneScopedRuntimeFilePath,
|
||||||
getOpenCodeRuntimeLaneIndexPath,
|
getOpenCodeRuntimeLaneIndexPath,
|
||||||
getOpenCodeTeamRuntimeDirectory,
|
getOpenCodeTeamRuntimeDirectory,
|
||||||
|
inspectOpenCodeRuntimeLaneStorage,
|
||||||
migrateLegacyOpenCodeRuntimeState,
|
migrateLegacyOpenCodeRuntimeState,
|
||||||
readOpenCodeRuntimeLaneIndex,
|
readOpenCodeRuntimeLaneIndex,
|
||||||
|
recoverStaleOpenCodeRuntimeLaneIndexEntry,
|
||||||
upsertOpenCodeRuntimeLaneIndexEntry,
|
upsertOpenCodeRuntimeLaneIndexEntry,
|
||||||
} from '../../../../src/main/services/team/opencode/store/OpenCodeRuntimeManifestEvidenceReader';
|
} from '../../../../src/main/services/team/opencode/store/OpenCodeRuntimeManifestEvidenceReader';
|
||||||
|
|
||||||
|
|
@ -238,4 +240,53 @@ describe('OpenCodeRuntimeManifestEvidenceReader migration', () => {
|
||||||
capabilitySnapshotId: 'cap-1',
|
capabilitySnapshotId: 'cap-1',
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('reports missing lane storage when an active lane index entry has no lane dir or state', async () => {
|
||||||
|
const teamName = 'team-epsilon';
|
||||||
|
const laneId = 'secondary:opencode:alice';
|
||||||
|
|
||||||
|
await upsertOpenCodeRuntimeLaneIndexEntry({
|
||||||
|
teamsBasePath: tempDir,
|
||||||
|
teamName,
|
||||||
|
laneId,
|
||||||
|
state: 'active',
|
||||||
|
});
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
inspectOpenCodeRuntimeLaneStorage({
|
||||||
|
teamsBasePath: tempDir,
|
||||||
|
teamName,
|
||||||
|
laneId,
|
||||||
|
})
|
||||||
|
).resolves.toEqual({
|
||||||
|
laneDirectoryExists: false,
|
||||||
|
hasStateOnDisk: false,
|
||||||
|
fileNames: [],
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await recoverStaleOpenCodeRuntimeLaneIndexEntry({
|
||||||
|
teamsBasePath: tempDir,
|
||||||
|
teamName,
|
||||||
|
laneId,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toEqual({
|
||||||
|
stale: true,
|
||||||
|
degraded: true,
|
||||||
|
diagnostics: [
|
||||||
|
`OpenCode lane ${laneId} is marked active in lanes.json, but no lane state exists on disk.`,
|
||||||
|
],
|
||||||
|
});
|
||||||
|
await expect(readOpenCodeRuntimeLaneIndex(tempDir, teamName)).resolves.toMatchObject({
|
||||||
|
lanes: {
|
||||||
|
[laneId]: {
|
||||||
|
laneId,
|
||||||
|
state: 'degraded',
|
||||||
|
diagnostics: [
|
||||||
|
`OpenCode lane ${laneId} is marked active in lanes.json, but no lane state exists on disk.`,
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -268,6 +268,54 @@ function writeBootstrapState(
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function writeTeamMeta(
|
||||||
|
teamName: string,
|
||||||
|
overrides: Record<string, unknown> = {}
|
||||||
|
): void {
|
||||||
|
const teamDir = path.join(tempTeamsBase, teamName);
|
||||||
|
fs.mkdirSync(teamDir, { recursive: true });
|
||||||
|
fs.writeFileSync(
|
||||||
|
path.join(teamDir, 'team.meta.json'),
|
||||||
|
`${JSON.stringify(
|
||||||
|
{
|
||||||
|
version: 1,
|
||||||
|
cwd: '/Users/test/proj',
|
||||||
|
providerId: 'codex',
|
||||||
|
providerBackendId: 'codex-native',
|
||||||
|
model: 'gpt-5.4',
|
||||||
|
effort: 'medium',
|
||||||
|
createdAt: Date.now(),
|
||||||
|
...overrides,
|
||||||
|
},
|
||||||
|
null,
|
||||||
|
2
|
||||||
|
)}\n`,
|
||||||
|
'utf8'
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function writeMembersMeta(
|
||||||
|
teamName: string,
|
||||||
|
members: Record<string, unknown>[],
|
||||||
|
providerBackendId = 'codex-native'
|
||||||
|
): void {
|
||||||
|
const teamDir = path.join(tempTeamsBase, teamName);
|
||||||
|
fs.mkdirSync(teamDir, { recursive: true });
|
||||||
|
fs.writeFileSync(
|
||||||
|
path.join(teamDir, 'members.meta.json'),
|
||||||
|
`${JSON.stringify(
|
||||||
|
{
|
||||||
|
version: 1,
|
||||||
|
providerBackendId,
|
||||||
|
members,
|
||||||
|
},
|
||||||
|
null,
|
||||||
|
2
|
||||||
|
)}\n`,
|
||||||
|
'utf8'
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
function createMemberSpawnStatusEntry(
|
function createMemberSpawnStatusEntry(
|
||||||
overrides: Record<string, unknown> = {}
|
overrides: Record<string, unknown> = {}
|
||||||
): Record<string, unknown> {
|
): Record<string, unknown> {
|
||||||
|
|
@ -1733,6 +1781,7 @@ describe('TeamProvisioningService', () => {
|
||||||
(svc as any).launchStateStore = {
|
(svc as any).launchStateStore = {
|
||||||
read: vi.fn(async () => null),
|
read: vi.fn(async () => null),
|
||||||
write: vi.fn(async () => {}),
|
write: vi.fn(async () => {}),
|
||||||
|
clear: vi.fn(async () => {}),
|
||||||
};
|
};
|
||||||
|
|
||||||
const run = createMemberSpawnRun({
|
const run = createMemberSpawnRun({
|
||||||
|
|
@ -4932,4 +4981,155 @@ describe('TeamProvisioningService', () => {
|
||||||
agentToolAccepted: true,
|
agentToolAccepted: true,
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('recovers stale mixed secondary lanes when lanes.json says active but lane state is missing', async () => {
|
||||||
|
const teamName = 'signal-ops-6212';
|
||||||
|
writeTeamMeta(teamName, {
|
||||||
|
providerId: 'codex',
|
||||||
|
providerBackendId: 'codex-native',
|
||||||
|
model: 'gpt-5.4',
|
||||||
|
});
|
||||||
|
writeMembersMeta(teamName, [
|
||||||
|
{
|
||||||
|
name: 'atlas',
|
||||||
|
providerId: 'opencode',
|
||||||
|
model: 'opencode/nemotron-3-super-free',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'bob',
|
||||||
|
providerId: 'codex',
|
||||||
|
model: 'gpt-5.4',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'nova',
|
||||||
|
providerId: 'codex',
|
||||||
|
model: 'gpt-5.4',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'tom',
|
||||||
|
providerId: 'opencode',
|
||||||
|
model: 'opencode/minimax-m2.5-free',
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
writeLaunchConfig(teamName, '/Users/test/proj', 'lead-session', ['bob', 'nova']);
|
||||||
|
writeBootstrapState(teamName, [
|
||||||
|
{ name: 'bob', status: 'registered' },
|
||||||
|
{ name: 'nova', status: 'registered' },
|
||||||
|
]);
|
||||||
|
await upsertOpenCodeRuntimeLaneIndexEntry({
|
||||||
|
teamsBasePath: tempTeamsBase,
|
||||||
|
teamName,
|
||||||
|
laneId: 'secondary:opencode:atlas',
|
||||||
|
state: 'active',
|
||||||
|
});
|
||||||
|
await upsertOpenCodeRuntimeLaneIndexEntry({
|
||||||
|
teamsBasePath: tempTeamsBase,
|
||||||
|
teamName,
|
||||||
|
laneId: 'secondary:opencode:tom',
|
||||||
|
state: 'active',
|
||||||
|
});
|
||||||
|
|
||||||
|
const svc = new TeamProvisioningService();
|
||||||
|
const result = await svc.getMemberSpawnStatuses(teamName);
|
||||||
|
|
||||||
|
expect(result.teamLaunchState).toBe('partial_failure');
|
||||||
|
expect(result.launchPhase).toBe('reconciled');
|
||||||
|
expect(result.expectedMembers).toEqual(expect.arrayContaining(['atlas', 'bob', 'nova', 'tom']));
|
||||||
|
expect(result.statuses.atlas).toMatchObject({
|
||||||
|
status: 'error',
|
||||||
|
launchState: 'failed_to_start',
|
||||||
|
error: expect.stringContaining('no lane state exists on disk'),
|
||||||
|
});
|
||||||
|
expect(result.statuses.tom).toMatchObject({
|
||||||
|
status: 'error',
|
||||||
|
launchState: 'failed_to_start',
|
||||||
|
error: expect.stringContaining('no lane state exists on disk'),
|
||||||
|
});
|
||||||
|
await expect(readOpenCodeRuntimeLaneIndex(tempTeamsBase, teamName)).resolves.toMatchObject({
|
||||||
|
lanes: {
|
||||||
|
'secondary:opencode:atlas': {
|
||||||
|
state: 'degraded',
|
||||||
|
},
|
||||||
|
'secondary:opencode:tom': {
|
||||||
|
state: 'degraded',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await expect(fsPromises.readFile(getTeamLaunchStatePath(teamName), 'utf8')).resolves.toContain(
|
||||||
|
'"secondary:opencode:atlas"'
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('includes queued OpenCode secondary lanes in live spawn statuses before the final mixed snapshot settles', async () => {
|
||||||
|
const svc = new TeamProvisioningService();
|
||||||
|
vi.spyOn(svc as any, 'refreshMemberSpawnStatusesFromLeadInbox').mockResolvedValue(undefined);
|
||||||
|
vi.spyOn(svc as any, 'maybeAuditMemberSpawnStatuses').mockResolvedValue(undefined);
|
||||||
|
|
||||||
|
const run = createMemberSpawnRun({
|
||||||
|
teamName: 'mixed-live-team',
|
||||||
|
runId: 'run-mixed-live-1',
|
||||||
|
expectedMembers: ['bob'],
|
||||||
|
memberSpawnStatuses: new Map([
|
||||||
|
[
|
||||||
|
'bob',
|
||||||
|
createMemberSpawnStatusEntry({
|
||||||
|
status: 'online',
|
||||||
|
launchState: 'confirmed_alive',
|
||||||
|
runtimeAlive: true,
|
||||||
|
bootstrapConfirmed: true,
|
||||||
|
livenessSource: 'heartbeat',
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
]),
|
||||||
|
});
|
||||||
|
run.isLaunch = true;
|
||||||
|
run.request = {
|
||||||
|
teamName: 'mixed-live-team',
|
||||||
|
cwd: '/tmp/mixed-live-team',
|
||||||
|
providerId: 'codex',
|
||||||
|
providerBackendId: 'codex-native',
|
||||||
|
model: 'gpt-5.4',
|
||||||
|
members: [],
|
||||||
|
};
|
||||||
|
run.effectiveMembers = [
|
||||||
|
{
|
||||||
|
name: 'bob',
|
||||||
|
providerId: 'codex',
|
||||||
|
model: 'gpt-5.4',
|
||||||
|
},
|
||||||
|
];
|
||||||
|
run.mixedSecondaryLanes = [
|
||||||
|
{
|
||||||
|
laneId: 'secondary:opencode:atlas',
|
||||||
|
providerId: 'opencode',
|
||||||
|
member: {
|
||||||
|
name: 'atlas',
|
||||||
|
providerId: 'opencode',
|
||||||
|
model: 'opencode/nemotron-3-super-free',
|
||||||
|
},
|
||||||
|
runId: null,
|
||||||
|
state: 'queued',
|
||||||
|
result: null,
|
||||||
|
warnings: [],
|
||||||
|
diagnostics: [],
|
||||||
|
},
|
||||||
|
];
|
||||||
|
run.detectedSessionId = 'lead-session';
|
||||||
|
|
||||||
|
(svc as any).runs.set(run.runId, run);
|
||||||
|
(svc as any).provisioningRunByTeam.set(run.teamName, run.runId);
|
||||||
|
|
||||||
|
const result = await svc.getMemberSpawnStatuses(run.teamName);
|
||||||
|
|
||||||
|
expect(result.teamLaunchState).toBe('partial_pending');
|
||||||
|
expect(result.expectedMembers).toEqual(expect.arrayContaining(['bob', 'atlas']));
|
||||||
|
expect(result.statuses.bob).toMatchObject({
|
||||||
|
status: 'online',
|
||||||
|
launchState: 'confirmed_alive',
|
||||||
|
});
|
||||||
|
expect(result.statuses.atlas).toMatchObject({
|
||||||
|
status: 'spawning',
|
||||||
|
launchState: 'starting',
|
||||||
|
});
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue