From 9fb9e5f66a556db78452098efbeed3837c8b500d Mon Sep 17 00:00:00 2001 From: 777genius Date: Thu, 30 Apr 2026 19:53:33 +0300 Subject: [PATCH] chore: snapshot dev work sync state --- .../src/internal/runtime.js | 111 +- .../test/controller.test.js | 149 ++ .../member-work-sync-debugging.md | 40 + .../core/application/MemberWorkSyncAudit.ts | 44 + .../MemberWorkSyncNudgeDispatcher.ts | 40 +- .../MemberWorkSyncNudgeOutboxPlanner.ts | 30 +- .../application/MemberWorkSyncReconciler.ts | 29 + .../application/MemberWorkSyncReporter.ts | 38 + .../application/RuntimeTurnSettledIngestor.ts | 59 +- .../core/application/index.ts | 1 + .../core/application/ports.ts | 50 + .../input/MemberWorkSyncTeamChangeRouter.ts | 14 +- .../createMemberWorkSyncFeature.ts | 15 +- .../FileMemberWorkSyncAuditJournal.ts | 157 ++ .../infrastructure/JsonMemberWorkSyncStore.ts | 1275 +++++++++++++---- .../MemberWorkSyncEventQueue.ts | 71 +- .../MemberWorkSyncStorePaths.ts | 72 +- .../infrastructure/EventLoopLagMonitor.ts | 19 +- src/main/services/team/TeamBackupService.ts | 6 +- .../services/team/TeamLaunchStateEvaluator.ts | 1 + .../services/team/TeamMemberStoragePaths.ts | 109 ++ .../services/team/TeamProvisioningService.ts | 230 ++- .../OpenCodeRuntimeManifestEvidenceReader.ts | 63 +- .../runtime/OpenCodeTeamRuntimeAdapter.ts | 32 +- .../components/team/CliLogsRichView.tsx | 8 +- .../team/members/MemberDetailDialog.tsx | 4 +- src/shared/types/team.ts | 2 + .../core/MemberWorkSyncUseCases.test.ts | 39 +- .../FileMemberWorkSyncAuditJournal.test.ts | 113 ++ .../main/JsonMemberWorkSyncStore.test.ts | 459 +++++- .../main/MemberWorkSyncEventQueue.test.ts | 7 + .../main/RuntimeTurnSettledIngestor.test.ts | 17 + ...nCodeRuntimeManifestEvidenceReader.test.ts | 114 ++ .../team/OpenCodeTeamRuntimeAdapter.test.ts | 8 +- .../TeamAgentLaunchMatrix.safe-e2e.test.ts | 214 ++- .../services/team/TeamBackupService.test.ts | 80 ++ .../team/TeamMemberStoragePaths.test.ts | 76 + .../team/TeamProvisioningService.test.ts | 853 ++++++++++- 38 files changed, 4195 insertions(+), 454 deletions(-) create mode 100644 docs/team-management/member-work-sync-debugging.md create mode 100644 src/features/member-work-sync/core/application/MemberWorkSyncAudit.ts create mode 100644 src/features/member-work-sync/main/infrastructure/FileMemberWorkSyncAuditJournal.ts create mode 100644 src/main/services/team/TeamMemberStoragePaths.ts create mode 100644 test/features/member-work-sync/main/FileMemberWorkSyncAuditJournal.test.ts create mode 100644 test/main/services/team/TeamMemberStoragePaths.test.ts diff --git a/agent-teams-controller/src/internal/runtime.js b/agent-teams-controller/src/internal/runtime.js index a798b6bc..ed328839 100644 --- a/agent-teams-controller/src/internal/runtime.js +++ b/agent-teams-controller/src/internal/runtime.js @@ -8,6 +8,9 @@ const MAX_WAIT_TIMEOUT_MS = 10 * 60 * 1000; const POLL_INTERVAL_MS = 1000; const TEAM_CONTROL_API_STATE_FILE = 'team-control-api.json'; const RETRYABLE_CONTROL_ERROR = 'retryableControlError'; +const BOOTSTRAP_CHECKIN_MAX_ATTEMPTS = 3; +const BOOTSTRAP_CHECKIN_ATTEMPT_TIMEOUT_MS = 4000; +const BOOTSTRAP_CHECKIN_RETRY_DELAYS_MS = [300, 900]; function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); @@ -67,9 +70,15 @@ function resolveControlBaseUrls(context, flags = {}) { return candidates; } -function makeRetryableControlError(message, cause) { +function makeRetryableControlError(message, cause, metadata = {}) { const error = new Error(message); error[RETRYABLE_CONTROL_ERROR] = true; + if (metadata.kind) { + error.retryableKind = metadata.kind; + } + if (metadata.statusCode) { + error.statusCode = metadata.statusCode; + } if (cause) { error.cause = cause; } @@ -114,7 +123,9 @@ async function requestJson(baseUrl, pathname, options = {}) { : `${response.status} ${response.statusText}`.trim(); if (isRetryableStatusCode(response.status)) { throw makeRetryableControlError( - `Team control API ${response.status} at ${baseUrl}${pathname}: ${detail || 'request failed'}` + `Team control API ${response.status} at ${baseUrl}${pathname}: ${detail || 'request failed'}`, + undefined, + { kind: 'status', statusCode: response.status } ); } throw new Error(detail || 'Team control API request failed'); @@ -122,19 +133,24 @@ async function requestJson(baseUrl, pathname, options = {}) { if (payload == null) { throw makeRetryableControlError( - `Team control API returned empty or non-JSON response at ${baseUrl}${pathname}` + `Team control API returned empty or non-JSON response at ${baseUrl}${pathname}`, + undefined, + { kind: 'empty' } ); } return payload; } catch (error) { if (error && error.name === 'AbortError') { - throw makeRetryableControlError(`Timed out calling team control API: ${pathname}`, error); + throw makeRetryableControlError(`Timed out calling team control API: ${pathname}`, error, { + kind: 'timeout', + }); } if (error && error.name === 'TypeError') { throw makeRetryableControlError( `Failed to reach team control API at ${baseUrl}: ${error.message || 'fetch failed'}`, - error + error, + { kind: 'network' } ); } throw error; @@ -161,6 +177,54 @@ async function requestJsonWithFallback(baseUrls, pathname, options = {}) { throw lastError || new Error('Team control API request failed'); } +function isBootstrapCheckinRetryableControlError(error) { + if (!isRetryableControlError(error)) { + return false; + } + + if (error.retryableKind === 'timeout' || error.retryableKind === 'network') { + return true; + } + + if (error.retryableKind === 'status') { + return typeof error.statusCode === 'number' && error.statusCode >= 500; + } + + return false; +} + +async function requestJsonWithBoundedRetry(baseUrls, pathname, options = {}, retryOptions = {}) { + const maxAttempts = Math.max(1, retryOptions.maxAttempts || 1); + let lastError = null; + + for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { + try { + const result = await requestJsonWithFallback(baseUrls, pathname, options); + if (attempt > 1 && result && typeof result === 'object' && !Array.isArray(result)) { + return { + ...result, + diagnostics: uniqueNonEmpty([ + ...(Array.isArray(result.diagnostics) ? result.diagnostics : []), + 'opencode_bootstrap_checkin_retry', + ]), + }; + } + return result; + } catch (error) { + lastError = error; + if (attempt >= maxAttempts || !isBootstrapCheckinRetryableControlError(error)) { + throw error; + } + const delayMs = retryOptions.delaysMs?.[attempt - 1] || 0; + if (delayMs > 0) { + await sleep(delayMs); + } + } + } + + throw lastError || new Error('Team control API request failed'); +} + function buildLaunchRequest(flags = {}) { const cwd = typeof flags.cwd === 'string' ? flags.cwd.trim() : ''; if (!cwd) { @@ -412,18 +476,31 @@ async function getRuntimeState(context, flags = {}) { } async function runtimeBootstrapCheckin(context, flags = {}) { - return postRuntimeTool( - context, - flags, - 'bootstrap-checkin', - compactRuntimeToolBody(context, flags, [ - 'runId', - 'memberName', - 'runtimeSessionId', - 'observedAt', - 'diagnostics', - 'metadata', - ]) + const baseUrls = resolveControlBaseUrls(context, flags); + const explicitTimeoutMs = flags.waitTimeoutMs || flags['wait-timeout-ms']; + const timeoutMs = Math.min( + normalizeTimeoutMs(explicitTimeoutMs || BOOTSTRAP_CHECKIN_ATTEMPT_TIMEOUT_MS), + BOOTSTRAP_CHECKIN_ATTEMPT_TIMEOUT_MS + ); + return requestJsonWithBoundedRetry( + baseUrls, + `/api/teams/${encodeURIComponent(context.teamName)}/opencode/runtime/bootstrap-checkin`, + { + method: 'POST', + body: compactRuntimeToolBody(context, flags, [ + 'runId', + 'memberName', + 'runtimeSessionId', + 'observedAt', + 'diagnostics', + 'metadata', + ]), + timeoutMs, + }, + { + maxAttempts: BOOTSTRAP_CHECKIN_MAX_ATTEMPTS, + delaysMs: BOOTSTRAP_CHECKIN_RETRY_DELAYS_MS, + } ); } diff --git a/agent-teams-controller/test/controller.test.js b/agent-teams-controller/test/controller.test.js index 6a6a32b7..1cfb6144 100644 --- a/agent-teams-controller/test/controller.test.js +++ b/agent-teams-controller/test/controller.test.js @@ -2251,6 +2251,155 @@ describe('agent-teams-controller API', () => { } }); + it('retries OpenCode bootstrap check-in on retryable control API failures', async () => { + const claudeDir = makeClaudeDir(); + const controller = createController({ teamName: 'my-team', claudeDir }); + const calls = []; + + const server = await startControlServer(async ({ method, url, body }) => { + calls.push({ method, url, body }); + if (calls.length < 3) { + return { statusCode: 500, body: { error: 'temporary bootstrap failure' } }; + } + return { body: { ok: true, state: 'accepted', diagnostics: [] } }; + }); + + try { + const result = await controller.runtime.runtimeBootstrapCheckin({ + controlUrl: server.baseUrl, + runId: 'run-oc', + memberName: 'bob', + runtimeSessionId: 'ses-1', + }); + + expect(result).toMatchObject({ + ok: true, + state: 'accepted', + diagnostics: expect.arrayContaining(['opencode_bootstrap_checkin_retry']), + }); + expect(calls).toHaveLength(3); + expect(calls.map((call) => call.body)).toEqual([ + { + teamName: 'my-team', + runId: 'run-oc', + memberName: 'bob', + runtimeSessionId: 'ses-1', + }, + { + teamName: 'my-team', + runId: 'run-oc', + memberName: 'bob', + runtimeSessionId: 'ses-1', + }, + { + teamName: 'my-team', + runId: 'run-oc', + memberName: 'bob', + runtimeSessionId: 'ses-1', + }, + ]); + } finally { + await server.close(); + } + }); + + it('accepts idempotent OpenCode bootstrap check-in after a timed-out committed request', async () => { + const claudeDir = makeClaudeDir(); + const controller = createController({ teamName: 'my-team', claudeDir }); + const calls = []; + let committed = false; + + const server = await startControlServer(async ({ method, url, body }) => { + calls.push({ method, url, body }); + if (!committed) { + committed = true; + await new Promise((resolve) => setTimeout(resolve, 1200)); + return { body: { ok: true, state: 'accepted', diagnostics: [] } }; + } + return { + body: { + ok: true, + state: 'accepted', + diagnostics: ['opencode_bootstrap_checkin_duplicate_accepted'], + }, + }; + }); + + try { + const result = await controller.runtime.runtimeBootstrapCheckin({ + controlUrl: server.baseUrl, + waitTimeoutMs: 1000, + runId: 'run-oc', + memberName: 'bob', + runtimeSessionId: 'ses-1', + }); + + expect(result).toMatchObject({ + ok: true, + state: 'accepted', + diagnostics: expect.arrayContaining([ + 'opencode_bootstrap_checkin_duplicate_accepted', + 'opencode_bootstrap_checkin_retry', + ]), + }); + expect(calls).toHaveLength(2); + } finally { + await server.close(); + } + }); + + it('does not retry OpenCode bootstrap check-in on validation failures', async () => { + const claudeDir = makeClaudeDir(); + const controller = createController({ teamName: 'my-team', claudeDir }); + const calls = []; + + const server = await startControlServer(async ({ method, url, body }) => { + calls.push({ method, url, body }); + return { statusCode: 400, body: { error: 'invalid bootstrap payload' } }; + }); + + try { + await expect( + controller.runtime.runtimeBootstrapCheckin({ + controlUrl: server.baseUrl, + runId: 'run-oc', + memberName: 'bob', + runtimeSessionId: 'ses-1', + }) + ).rejects.toThrow('invalid bootstrap payload'); + expect(calls).toHaveLength(1); + } finally { + await server.close(); + } + }); + + it('fails OpenCode bootstrap check-in clearly after bounded timeout retries', async () => { + const claudeDir = makeClaudeDir(); + const controller = createController({ teamName: 'my-team', claudeDir }); + const calls = []; + + const server = await startControlServer(async ({ method, url, body }) => { + calls.push({ method, url, body }); + await new Promise((resolve) => setTimeout(resolve, 1200)); + return { body: { ok: true, state: 'accepted' } }; + }); + + try { + await expect( + controller.runtime.runtimeBootstrapCheckin({ + controlUrl: server.baseUrl, + waitTimeoutMs: 1000, + runId: 'run-oc', + memberName: 'bob', + runtimeSessionId: 'ses-1', + }) + ).rejects.toThrow('Timed out calling team control API'); + expect(calls).toHaveLength(3); + } finally { + await server.close(); + } + }); + it('forwards member work sync status and reports to the app validator', async () => { const claudeDir = makeClaudeDir(); const controller = createController({ teamName: 'my-team', claudeDir }); diff --git a/docs/team-management/member-work-sync-debugging.md b/docs/team-management/member-work-sync-debugging.md new file mode 100644 index 00000000..9f3d5f95 --- /dev/null +++ b/docs/team-management/member-work-sync-debugging.md @@ -0,0 +1,40 @@ +# Member Work Sync Debugging + +`member-work-sync` stores member-scoped control-plane state under each team member: + +```text +~/.claude/teams//members//.member-work-sync/ + status.json + reports.json + outbox.json + journal.jsonl +``` + +`member-key` is the normalized, percent-encoded member name. The canonical name is stored in: + +```text +~/.claude/teams//members//member.meta.json +``` + +Use the journal for local debugging: + +```bash +tail -f ~/.claude/teams//members//.member-work-sync/journal.jsonl +``` + +The journal is append-only JSONL and records sync decisions, not raw agent transcripts. Useful events: + +- `reconcile_started`, `agenda_loaded`, `decision_made`, `status_written` +- `report_received`, `report_accepted`, `report_rejected` +- `nudge_planned`, `nudge_delivered`, `nudge_skipped`, `nudge_retryable`, `nudge_superseded` +- `member_busy`, `watchdog_cooldown_active`, `team_inactive`, `legacy_fallback_used` + +Team-level shared/index state remains under: + +```text +~/.claude/teams//.member-work-sync/ + indexes/ + report-token-secret.json +``` + +The indexes are implementation details used to avoid scanning every member directory on the hot path. diff --git a/src/features/member-work-sync/core/application/MemberWorkSyncAudit.ts b/src/features/member-work-sync/core/application/MemberWorkSyncAudit.ts new file mode 100644 index 00000000..f2a52b09 --- /dev/null +++ b/src/features/member-work-sync/core/application/MemberWorkSyncAudit.ts @@ -0,0 +1,44 @@ +import type { + MemberWorkSyncAuditEvent, + MemberWorkSyncAuditEventName, + MemberWorkSyncUseCaseDeps, +} from './ports'; + +export type MemberWorkSyncAuditEventInput = Omit & { + timestamp?: string; +}; + +export async function appendMemberWorkSyncAudit( + deps: Pick, + input: MemberWorkSyncAuditEventInput +): Promise { + if (!deps.auditJournal) { + return; + } + try { + await deps.auditJournal.append({ + ...input, + timestamp: input.timestamp ?? deps.clock.now().toISOString(), + }); + } catch (error) { + deps.logger?.warn('member work sync audit event failed', { + teamName: input.teamName, + memberName: input.memberName, + event: input.event, + error: String(error), + }); + } +} + +export function reasonToAuditEvent(reason: string): MemberWorkSyncAuditEventName { + if (reason.startsWith('member_busy:')) { + return 'member_busy'; + } + if (reason === 'watchdog_cooldown_active') { + return 'watchdog_cooldown_active'; + } + if (reason === 'team_inactive') { + return 'team_inactive'; + } + return 'nudge_skipped'; +} diff --git a/src/features/member-work-sync/core/application/MemberWorkSyncNudgeDispatcher.ts b/src/features/member-work-sync/core/application/MemberWorkSyncNudgeDispatcher.ts index ebcbe89f..81902547 100644 --- a/src/features/member-work-sync/core/application/MemberWorkSyncNudgeDispatcher.ts +++ b/src/features/member-work-sync/core/application/MemberWorkSyncNudgeDispatcher.ts @@ -1,5 +1,6 @@ import type { MemberWorkSyncOutboxItem } from '../../contracts'; -import type { MemberWorkSyncUseCaseDeps } from './ports'; +import { appendMemberWorkSyncAudit, reasonToAuditEvent } from './MemberWorkSyncAudit'; +import type { MemberWorkSyncAuditEventName, MemberWorkSyncUseCaseDeps } from './ports'; const MEMBER_WORK_SYNC_MAX_NUDGES_PER_MEMBER_PER_HOUR = 2; const MEMBER_WORK_SYNC_RETRY_BASE_MINUTES = 10; @@ -50,7 +51,9 @@ function nextRetryAt(item: MemberWorkSyncOutboxItem, nowIso: string): string { export class MemberWorkSyncNudgeDispatcher { constructor(private readonly deps: MemberWorkSyncUseCaseDeps) {} - async dispatchDue(options: MemberWorkSyncNudgeDispatchOptions): Promise { + async dispatchDue( + options: MemberWorkSyncNudgeDispatchOptions + ): Promise { const outbox = this.deps.outboxStore; const inbox = this.deps.inboxNudge; if (!outbox || !inbox) { @@ -59,7 +62,9 @@ export class MemberWorkSyncNudgeDispatcher { const nowIso = this.deps.clock.now().toISOString(); const summary = emptySummary(); - for (const teamName of [...new Set(options.teamNames.map((name) => name.trim()).filter(Boolean))]) { + for (const teamName of [ + ...new Set(options.teamNames.map((name) => name.trim()).filter(Boolean)), + ]) { const claimed = await outbox.claimDue({ teamName, claimedBy: options.claimedBy, @@ -97,6 +102,11 @@ export class MemberWorkSyncNudgeDispatcher { nowIso, nextAttemptAt: revalidation.nextAttemptAt ?? nextRetryAt(item, nowIso), }); + await this.appendDispatchAudit( + item, + reasonToAuditEvent(revalidation.reason), + revalidation.reason + ); return 'retryable'; } await outbox.markSuperseded({ @@ -105,6 +115,7 @@ export class MemberWorkSyncNudgeDispatcher { reason: revalidation.reason, nowIso, }); + await this.appendDispatchAudit(item, 'nudge_superseded', revalidation.reason); return 'superseded'; } @@ -126,6 +137,7 @@ export class MemberWorkSyncNudgeDispatcher { retryable: false, nowIso, }); + await this.appendDispatchAudit(item, 'nudge_skipped', 'inbox_payload_conflict'); return 'terminal'; } await outbox.markDelivered({ @@ -135,6 +147,7 @@ export class MemberWorkSyncNudgeDispatcher { deliveredMessageId: inserted.messageId, nowIso, }); + await this.appendDispatchAudit(item, 'nudge_delivered', 'inbox_inserted'); return 'delivered'; } catch (error) { await outbox.markFailed({ @@ -146,16 +159,33 @@ export class MemberWorkSyncNudgeDispatcher { nowIso, nextAttemptAt: nextRetryAt(item, nowIso), }); + await this.appendDispatchAudit(item, 'nudge_retryable', String(error)); return 'retryable'; } } + private async appendDispatchAudit( + item: MemberWorkSyncOutboxItem, + event: MemberWorkSyncAuditEventName, + reason: string + ): Promise { + await appendMemberWorkSyncAudit(this.deps, { + teamName: item.teamName, + memberName: item.memberName, + event, + source: 'nudge_dispatcher', + agendaFingerprint: item.agendaFingerprint, + reason, + taskRefs: item.payload.taskRefs, + messagePreview: item.payload.text, + }); + } + private async revalidate( item: MemberWorkSyncOutboxItem, nowIso: string ): Promise< - | { ok: true } - | { ok: false; reason: string; retryable: boolean; nextAttemptAt?: string } + { ok: true } | { ok: false; reason: string; retryable: boolean; nextAttemptAt?: string } > { if (this.deps.lifecycle && !(await this.deps.lifecycle.isTeamActive(item.teamName))) { return { ok: false, reason: 'team_inactive', retryable: false }; diff --git a/src/features/member-work-sync/core/application/MemberWorkSyncNudgeOutboxPlanner.ts b/src/features/member-work-sync/core/application/MemberWorkSyncNudgeOutboxPlanner.ts index a042d0bf..ab01079e 100644 --- a/src/features/member-work-sync/core/application/MemberWorkSyncNudgeOutboxPlanner.ts +++ b/src/features/member-work-sync/core/application/MemberWorkSyncNudgeOutboxPlanner.ts @@ -1,5 +1,7 @@ import { buildMemberWorkSyncOutboxEnsureInput } from '../domain'; +import { appendMemberWorkSyncAudit } from './MemberWorkSyncAudit'; + import type { MemberWorkSyncStatus } from '../../contracts'; import type { MemberWorkSyncUseCaseDeps } from './ports'; @@ -37,6 +39,7 @@ export class MemberWorkSyncNudgeOutboxPlanner { const metrics = await this.deps.statusStore.readTeamMetrics(status.teamName); if (metrics.phase2Readiness.state !== 'shadow_ready') { + await this.appendPlanAudit(status, { planned: false, code: 'phase2_not_ready' }); return { planned: false, code: 'phase2_not_ready' }; } @@ -49,9 +52,34 @@ export class MemberWorkSyncNudgeOutboxPlanner { existingPayloadHash: result.existingPayloadHash, requestedPayloadHash: result.requestedPayloadHash, }); + await this.appendPlanAudit(status, { planned: false, code: 'payload_conflict' }); return { planned: false, code: 'payload_conflict' }; } - return { planned: true, code: result.outcome }; + const planResult = { planned: true, code: result.outcome } as const; + await this.appendPlanAudit(status, planResult); + return planResult; + } + + private async appendPlanAudit( + status: MemberWorkSyncStatus, + result: MemberWorkSyncNudgeOutboxPlanResult + ): Promise { + await appendMemberWorkSyncAudit(this.deps, { + teamName: status.teamName, + memberName: status.memberName, + event: result.planned ? 'nudge_planned' : 'nudge_skipped', + source: 'nudge_planner', + agendaFingerprint: status.agenda.fingerprint, + state: status.state, + actionableCount: status.agenda.items.length, + reason: result.code, + ...(status.providerId ? { providerId: status.providerId } : {}), + taskRefs: status.agenda.items.map((item) => ({ + taskId: item.taskId, + displayId: item.displayId, + teamName: status.teamName, + })), + }); } } diff --git a/src/features/member-work-sync/core/application/MemberWorkSyncReconciler.ts b/src/features/member-work-sync/core/application/MemberWorkSyncReconciler.ts index 3043f90b..0ac4f068 100644 --- a/src/features/member-work-sync/core/application/MemberWorkSyncReconciler.ts +++ b/src/features/member-work-sync/core/application/MemberWorkSyncReconciler.ts @@ -5,6 +5,7 @@ import { formatAgendaFingerprint, } from '../domain'; +import { appendMemberWorkSyncAudit } from './MemberWorkSyncAudit'; import { MemberWorkSyncNudgeOutboxPlanner } from './MemberWorkSyncNudgeOutboxPlanner'; import type { MemberWorkSyncStatus, MemberWorkSyncStatusRequest } from '../../contracts'; @@ -46,8 +47,25 @@ export class MemberWorkSyncReconciler { request: MemberWorkSyncStatusRequest, context: MemberWorkSyncReconcileContext = {} ): Promise { + await appendMemberWorkSyncAudit(this.deps, { + teamName: request.teamName, + memberName: request.memberName, + event: 'reconcile_started', + source: 'reconciler', + ...(context.triggerReasons?.length ? { triggerReasons: context.triggerReasons } : {}), + }); const source = await this.deps.agendaSource.loadAgenda(request); const agenda = finalizeMemberWorkSyncAgenda(this.deps, source); + await appendMemberWorkSyncAudit(this.deps, { + teamName: agenda.teamName, + memberName: agenda.memberName, + event: 'agenda_loaded', + source: 'reconciler', + agendaFingerprint: agenda.fingerprint, + actionableCount: agenda.items.length, + ...(source.providerId ? { providerId: source.providerId } : {}), + diagnostics: agenda.diagnostics, + }); const previous = await this.deps.statusStore.read(request); const nowIso = this.deps.clock.now().toISOString(); const teamActive = this.deps.lifecycle @@ -59,6 +77,17 @@ export class MemberWorkSyncReconciler { nowIso, inactive: source.inactive || !teamActive, }); + await appendMemberWorkSyncAudit(this.deps, { + teamName: agenda.teamName, + memberName: agenda.memberName, + event: source.inactive || !teamActive ? 'team_inactive' : 'decision_made', + source: 'reconciler', + agendaFingerprint: agenda.fingerprint, + state: decision.state, + actionableCount: agenda.items.length, + ...(source.providerId ? { providerId: source.providerId } : {}), + diagnostics: decision.diagnostics, + }); const status = await attachMemberWorkSyncReportToken(this.deps, { teamName: agenda.teamName, diff --git a/src/features/member-work-sync/core/application/MemberWorkSyncReporter.ts b/src/features/member-work-sync/core/application/MemberWorkSyncReporter.ts index 2076adf4..8899468f 100644 --- a/src/features/member-work-sync/core/application/MemberWorkSyncReporter.ts +++ b/src/features/member-work-sync/core/application/MemberWorkSyncReporter.ts @@ -1,5 +1,6 @@ import { validateMemberWorkSyncReport } from '../domain'; +import { appendMemberWorkSyncAudit } from './MemberWorkSyncAudit'; import { attachMemberWorkSyncReportToken, finalizeMemberWorkSyncAgenda, @@ -22,6 +23,22 @@ export class MemberWorkSyncReporter { } async execute(request: MemberWorkSyncReportRequest): Promise { + await appendMemberWorkSyncAudit(this.deps, { + teamName: request.teamName, + memberName: request.memberName, + event: 'report_received', + source: 'reporter', + agendaFingerprint: request.agendaFingerprint, + state: request.state, + ...(request.taskIds?.length + ? { + taskRefs: request.taskIds.map((taskId) => ({ + taskId, + teamName: request.teamName, + })), + } + : {}), + }); const source = await this.deps.agendaSource.loadAgenda(request); const agenda = finalizeMemberWorkSyncAgenda(this.deps, source); const nowIso = this.deps.clock.now().toISOString(); @@ -105,6 +122,16 @@ export class MemberWorkSyncReporter { }); await this.deps.statusStore.write(status); + await appendMemberWorkSyncAudit(this.deps, { + teamName: status.teamName, + memberName: status.memberName, + event: 'report_accepted', + source: 'reporter', + agendaFingerprint: agenda.fingerprint, + state: status.state, + actionableCount: agenda.items.length, + ...(source.providerId ? { providerId: source.providerId } : {}), + }); return { accepted: true, code: 'accepted', @@ -135,6 +162,17 @@ export class MemberWorkSyncReporter { diagnostics: [...status.diagnostics, `report_rejected:${rejectionCode}`], }; await this.deps.statusStore.write(rejectedStatus); + await appendMemberWorkSyncAudit(this.deps, { + teamName: status.teamName, + memberName: status.memberName, + event: 'report_rejected', + source: 'reporter', + agendaFingerprint: request.agendaFingerprint, + state: request.state, + actionableCount: status.agenda.items.length, + reason: rejectionCode, + ...(status.providerId ? { providerId: status.providerId } : {}), + }); return rejectedStatus; } } diff --git a/src/features/member-work-sync/core/application/RuntimeTurnSettledIngestor.ts b/src/features/member-work-sync/core/application/RuntimeTurnSettledIngestor.ts index a745557d..39f66af3 100644 --- a/src/features/member-work-sync/core/application/RuntimeTurnSettledIngestor.ts +++ b/src/features/member-work-sync/core/application/RuntimeTurnSettledIngestor.ts @@ -1,4 +1,9 @@ -import type { MemberWorkSyncClockPort, MemberWorkSyncLoggerPort } from './ports'; +import { appendMemberWorkSyncAudit } from './MemberWorkSyncAudit'; +import type { + MemberWorkSyncAuditJournalPort, + MemberWorkSyncClockPort, + MemberWorkSyncLoggerPort, +} from './ports'; import type { RuntimeTurnSettledEvent } from '../domain'; import type { RuntimeTurnSettledEventStorePort, @@ -13,6 +18,7 @@ export interface RuntimeTurnSettledIngestorDeps { targetResolver: RuntimeTurnSettledTargetResolverPort; reconcileQueue: RuntimeTurnSettledReconcileQueuePort; clock: MemberWorkSyncClockPort; + auditJournal?: MemberWorkSyncAuditJournalPort; logger?: MemberWorkSyncLoggerPort; } @@ -78,6 +84,20 @@ export class RuntimeTurnSettledIngestor { continue; } + if (normalized.event.teamName && normalized.event.memberName) { + await appendMemberWorkSyncAudit(this.deps, { + teamName: normalized.event.teamName, + memberName: normalized.event.memberName, + event: 'turn_settled_claimed', + source: 'runtime_turn_settled_ingestor', + reason: normalized.event.provider, + metadata: { + sourceId: normalized.event.sourceId, + provider: normalized.event.provider, + }, + }); + } + const ignoredReason = getIgnoredReason(normalized.event); if (ignoredReason) { summary.ignored += 1; @@ -87,6 +107,19 @@ export class RuntimeTurnSettledIngestor { reason: ignoredReason, processedAt, }); + if (normalized.event.teamName && normalized.event.memberName) { + await appendMemberWorkSyncAudit(this.deps, { + teamName: normalized.event.teamName, + memberName: normalized.event.memberName, + event: 'turn_settled_ignored', + source: 'runtime_turn_settled_ingestor', + reason: ignoredReason, + metadata: { + sourceId: normalized.event.sourceId, + provider: normalized.event.provider, + }, + }); + } continue; } @@ -99,6 +132,19 @@ export class RuntimeTurnSettledIngestor { reason: resolution.reason, processedAt, }); + if (normalized.event.teamName && normalized.event.memberName) { + await appendMemberWorkSyncAudit(this.deps, { + teamName: normalized.event.teamName, + memberName: normalized.event.memberName, + event: 'turn_settled_unresolved', + source: 'runtime_turn_settled_ingestor', + reason: resolution.reason, + metadata: { + sourceId: normalized.event.sourceId, + provider: normalized.event.provider, + }, + }); + } continue; } @@ -115,6 +161,17 @@ export class RuntimeTurnSettledIngestor { outcome: 'enqueued', processedAt, }); + await appendMemberWorkSyncAudit(this.deps, { + teamName: resolution.teamName, + memberName: resolution.memberName, + event: 'turn_settled_resolved', + source: 'runtime_turn_settled_ingestor', + reason: normalized.event.provider, + metadata: { + sourceId: normalized.event.sourceId, + provider: normalized.event.provider, + }, + }); } catch (error) { summary.failed += 1; this.deps.logger?.warn('runtime turn settled ingest failed', { diff --git a/src/features/member-work-sync/core/application/index.ts b/src/features/member-work-sync/core/application/index.ts index 74021af2..704debbc 100644 --- a/src/features/member-work-sync/core/application/index.ts +++ b/src/features/member-work-sync/core/application/index.ts @@ -1,5 +1,6 @@ export * from './MemberWorkSyncDiagnosticsReader'; export * from './MemberWorkSyncMetricsReader'; +export * from './MemberWorkSyncAudit'; export * from './MemberWorkSyncNudgeDispatcher'; export * from './MemberWorkSyncNudgeOutboxPlanner'; export * from './MemberWorkSyncPendingReportIntentReplayer'; diff --git a/src/features/member-work-sync/core/application/ports.ts b/src/features/member-work-sync/core/application/ports.ts index 4ac83fec..886cb178 100644 --- a/src/features/member-work-sync/core/application/ports.ts +++ b/src/features/member-work-sync/core/application/ports.ts @@ -64,6 +64,55 @@ export interface MemberWorkSyncLoggerPort { error(message: string, metadata?: Record): void; } +export type MemberWorkSyncAuditEventName = + | 'turn_settled_claimed' + | 'turn_settled_resolved' + | 'turn_settled_unresolved' + | 'turn_settled_ignored' + | 'queue_enqueued' + | 'queue_coalesced' + | 'queue_reconciled' + | 'queue_dropped' + | 'reconcile_started' + | 'agenda_loaded' + | 'decision_made' + | 'status_written' + | 'report_received' + | 'report_accepted' + | 'report_rejected' + | 'nudge_planned' + | 'nudge_delivered' + | 'nudge_skipped' + | 'nudge_retryable' + | 'nudge_superseded' + | 'watchdog_cooldown_active' + | 'member_busy' + | 'team_inactive' + | 'index_repaired' + | 'legacy_fallback_used'; + +export interface MemberWorkSyncAuditEvent { + timestamp: string; + teamName: string; + memberName: string; + event: MemberWorkSyncAuditEventName; + source: string; + agendaFingerprint?: string; + state?: string; + actionableCount?: number; + reason?: string; + triggerReasons?: string[]; + providerId?: string; + taskRefs?: { taskId: string; displayId?: string; teamName?: string }[]; + diagnostics?: string[]; + messagePreview?: string; + metadata?: Record; +} + +export interface MemberWorkSyncAuditJournalPort { + append(event: MemberWorkSyncAuditEvent): Promise; +} + export interface MemberWorkSyncAgendaSourceResult { agenda: Omit; activeMemberNames: string[]; @@ -143,6 +192,7 @@ export interface MemberWorkSyncUseCaseDeps { watchdogCooldown?: MemberWorkSyncWatchdogCooldownPort; busySignal?: MemberWorkSyncBusySignalPort; reportToken?: MemberWorkSyncReportTokenPort; + auditJournal?: MemberWorkSyncAuditJournalPort; lifecycle?: MemberWorkSyncLifecyclePort; logger?: MemberWorkSyncLoggerPort; } diff --git a/src/features/member-work-sync/main/adapters/input/MemberWorkSyncTeamChangeRouter.ts b/src/features/member-work-sync/main/adapters/input/MemberWorkSyncTeamChangeRouter.ts index 477a8995..edb54e8a 100644 --- a/src/features/member-work-sync/main/adapters/input/MemberWorkSyncTeamChangeRouter.ts +++ b/src/features/member-work-sync/main/adapters/input/MemberWorkSyncTeamChangeRouter.ts @@ -14,6 +14,10 @@ interface MemberWorkSyncRosterSource { loadActiveMemberNames(teamName: string): Promise; } +interface MemberWorkSyncMemberStorageMaterializer { + materializeMember(teamName: string, memberName: string): Promise; +} + const TEAM_WIDE_REASONS: Partial> = { config: 'config_changed', task: 'task_changed', @@ -58,7 +62,8 @@ function parseMemberTurnSettled(detail: string | undefined): MemberTurnSettledEv export class MemberWorkSyncTeamChangeRouter { constructor( private readonly rosterSource: MemberWorkSyncRosterSource, - private readonly queue: MemberWorkSyncEventQueue + private readonly queue: MemberWorkSyncEventQueue, + private readonly materializer?: MemberWorkSyncMemberStorageMaterializer ) {} async enqueueStartupScan(teamNames: string[]): Promise { @@ -137,6 +142,13 @@ export class MemberWorkSyncTeamChangeRouter { runAfterMs?: number ): Promise { const activeMembers = await this.rosterSource.loadActiveMemberNames(teamName); + if (this.materializer) { + await Promise.allSettled( + activeMembers.map((memberName) => + this.materializer?.materializeMember(teamName, memberName) + ) + ); + } for (const memberName of activeMembers) { this.queue.enqueue({ teamName, memberName, triggerReason, runAfterMs }); } diff --git a/src/features/member-work-sync/main/composition/createMemberWorkSyncFeature.ts b/src/features/member-work-sync/main/composition/createMemberWorkSyncFeature.ts index 17ca5ac3..11c4fef4 100644 --- a/src/features/member-work-sync/main/composition/createMemberWorkSyncFeature.ts +++ b/src/features/member-work-sync/main/composition/createMemberWorkSyncFeature.ts @@ -21,6 +21,7 @@ import { ClaudeStopHookPayloadNormalizer } from '../infrastructure/ClaudeStopHoo import { CodexNativeTurnSettledPayloadNormalizer } from '../infrastructure/CodexNativeTurnSettledPayloadNormalizer'; import { CompositeRuntimeTurnSettledPayloadNormalizer } from '../infrastructure/CompositeRuntimeTurnSettledPayloadNormalizer'; import { FileRuntimeTurnSettledEventStore } from '../infrastructure/FileRuntimeTurnSettledEventStore'; +import { FileMemberWorkSyncAuditJournal } from '../infrastructure/FileMemberWorkSyncAuditJournal'; import { HmacMemberWorkSyncReportTokenAdapter } from '../infrastructure/HmacMemberWorkSyncReportTokenAdapter'; import { JsonMemberWorkSyncStore } from '../infrastructure/JsonMemberWorkSyncStore'; import { @@ -132,7 +133,11 @@ export function createMemberWorkSyncFeature(deps: { clock, }); const storePaths = new MemberWorkSyncStorePaths(deps.teamsBasePath); - const store = new JsonMemberWorkSyncStore(storePaths); + const auditJournal = new FileMemberWorkSyncAuditJournal(storePaths, deps.logger); + const store = new JsonMemberWorkSyncStore(storePaths, { + auditJournal, + logger: deps.logger, + }); const runtimeTurnSettledSpool = new RuntimeTurnSettledSpoolInitializer(deps.teamsBasePath); const runtimeTurnSettledStore = new FileRuntimeTurnSettledEventStore({ paths: runtimeTurnSettledSpool.getPaths(), @@ -165,6 +170,7 @@ export function createMemberWorkSyncFeature(deps: { watchdogCooldown, busySignal, reportToken, + auditJournal, ...(deps.isTeamActive ? { lifecycle: { isTeamActive: deps.isTeamActive } } : {}), logger: deps.logger, }; @@ -186,9 +192,13 @@ export function createMemberWorkSyncFeature(deps: { }, isTeamActive: deps.isTeamActive ?? (() => true), ...(deps.queueQuietWindowMs != null ? { quietWindowMs: deps.queueQuietWindowMs } : {}), + auditJournal, logger: deps.logger, }); - const router = new MemberWorkSyncTeamChangeRouter(agendaSource, queue); + const router = new MemberWorkSyncTeamChangeRouter(agendaSource, queue, { + materializeMember: (teamName, memberName) => + storePaths.ensureMemberWorkSyncDir(teamName, memberName), + }); const runtimeTurnSettledIngestor = new RuntimeTurnSettledIngestor({ eventStore: runtimeTurnSettledStore, normalizer: runtimeTurnSettledNormalizer, @@ -207,6 +217,7 @@ export function createMemberWorkSyncFeature(deps: { }, }, clock, + auditJournal, logger: deps.logger, }); const runtimeTurnSettledDrainScheduler = new RuntimeTurnSettledDrainScheduler({ diff --git a/src/features/member-work-sync/main/infrastructure/FileMemberWorkSyncAuditJournal.ts b/src/features/member-work-sync/main/infrastructure/FileMemberWorkSyncAuditJournal.ts new file mode 100644 index 00000000..9b47e604 --- /dev/null +++ b/src/features/member-work-sync/main/infrastructure/FileMemberWorkSyncAuditJournal.ts @@ -0,0 +1,157 @@ +import { appendFile, mkdir, rename, rm, stat } from 'fs/promises'; +import { dirname } from 'path'; + +import { withFileLock } from '@main/services/team/fileLock'; + +import type { + MemberWorkSyncAuditEvent, + MemberWorkSyncAuditJournalPort, + MemberWorkSyncLoggerPort, +} from '../../core/application'; +import type { MemberWorkSyncStorePaths } from './MemberWorkSyncStorePaths'; + +const DEFAULT_MAX_BYTES = 5 * 1024 * 1024; +const DEFAULT_ROTATED_FILE_COUNT = 5; +const MAX_PREVIEW_CHARS = 240; +const MAX_DIAGNOSTICS = 20; +const MAX_TRIGGER_REASONS = 20; +const MAX_TASK_REFS = 20; +const MAX_SHORT_FIELD_CHARS = 240; + +interface PersistedAuditEvent extends MemberWorkSyncAuditEvent { + schemaVersion: 1; +} + +export interface FileMemberWorkSyncAuditJournalOptions { + maxBytes?: number; + rotatedFileCount?: number; +} + +function truncateText(value: string, maxChars: number): string { + return value.length <= maxChars ? value : `${value.slice(0, maxChars)}...`; +} + +function sanitizeMetadata( + metadata: MemberWorkSyncAuditEvent['metadata'] +): MemberWorkSyncAuditEvent['metadata'] { + if (!metadata) { + return undefined; + } + const sanitized = Object.create(null) as NonNullable; + for (const [key, value] of Object.entries(metadata)) { + sanitized[truncateText(key, MAX_SHORT_FIELD_CHARS)] = + typeof value === 'string' ? truncateText(value, MAX_SHORT_FIELD_CHARS) : value; + } + return sanitized; +} + +function sanitizeTaskRefs( + taskRefs: MemberWorkSyncAuditEvent['taskRefs'] +): MemberWorkSyncAuditEvent['taskRefs'] { + return taskRefs?.slice(0, MAX_TASK_REFS).map((taskRef) => ({ + taskId: truncateText(taskRef.taskId, MAX_SHORT_FIELD_CHARS), + ...(taskRef.displayId + ? { displayId: truncateText(taskRef.displayId, MAX_SHORT_FIELD_CHARS) } + : {}), + ...(taskRef.teamName + ? { teamName: truncateText(taskRef.teamName, MAX_SHORT_FIELD_CHARS) } + : {}), + })); +} + +function sanitizeEvent(event: MemberWorkSyncAuditEvent): PersistedAuditEvent { + return { + ...event, + schemaVersion: 1, + source: truncateText(event.source, MAX_SHORT_FIELD_CHARS), + ...(event.reason ? { reason: truncateText(event.reason, MAX_SHORT_FIELD_CHARS) } : {}), + ...(event.providerId + ? { providerId: truncateText(event.providerId, MAX_SHORT_FIELD_CHARS) } + : {}), + ...(event.state ? { state: truncateText(event.state, MAX_SHORT_FIELD_CHARS) } : {}), + ...(event.agendaFingerprint + ? { agendaFingerprint: truncateText(event.agendaFingerprint, MAX_SHORT_FIELD_CHARS) } + : {}), + ...(typeof event.messagePreview === 'string' + ? { messagePreview: truncateText(event.messagePreview, MAX_PREVIEW_CHARS) } + : {}), + ...(event.diagnostics + ? { + diagnostics: event.diagnostics + .slice(0, MAX_DIAGNOSTICS) + .map((diagnostic) => truncateText(diagnostic, MAX_SHORT_FIELD_CHARS)), + } + : {}), + ...(event.triggerReasons + ? { + triggerReasons: event.triggerReasons + .slice(0, MAX_TRIGGER_REASONS) + .map((reason) => truncateText(reason, MAX_SHORT_FIELD_CHARS)), + } + : {}), + ...(event.taskRefs ? { taskRefs: sanitizeTaskRefs(event.taskRefs) } : {}), + ...(event.metadata ? { metadata: sanitizeMetadata(event.metadata) } : {}), + }; +} + +function rotatedPath(filePath: string, index: number): string { + return `${filePath}.${index}`; +} + +async function rotateIfNeeded( + filePath: string, + maxBytes: number, + rotatedFileCount: number +): Promise { + const current = await stat(filePath).catch(() => null); + if (!current?.isFile() || current.size < maxBytes) { + return; + } + + await rm(rotatedPath(filePath, rotatedFileCount), { force: true }).catch(() => undefined); + for (let index = rotatedFileCount - 1; index >= 1; index -= 1) { + await rename(rotatedPath(filePath, index), rotatedPath(filePath, index + 1)).catch( + () => undefined + ); + } + await rename(filePath, rotatedPath(filePath, 1)).catch(() => undefined); +} + +export class NoopMemberWorkSyncAuditJournal implements MemberWorkSyncAuditJournalPort { + async append(): Promise { + // Intentionally empty. + } +} + +export class FileMemberWorkSyncAuditJournal implements MemberWorkSyncAuditJournalPort { + private readonly maxBytes: number; + private readonly rotatedFileCount: number; + + constructor( + private readonly paths: MemberWorkSyncStorePaths, + private readonly logger?: MemberWorkSyncLoggerPort, + options: FileMemberWorkSyncAuditJournalOptions = {} + ) { + this.maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES; + this.rotatedFileCount = options.rotatedFileCount ?? DEFAULT_ROTATED_FILE_COUNT; + } + + async append(event: MemberWorkSyncAuditEvent): Promise { + try { + await this.paths.ensureMemberWorkSyncDir(event.teamName, event.memberName); + const filePath = this.paths.getMemberJournalPath(event.teamName, event.memberName); + await mkdir(dirname(filePath), { recursive: true }); + await withFileLock(filePath, async () => { + await rotateIfNeeded(filePath, this.maxBytes, this.rotatedFileCount); + await appendFile(filePath, `${JSON.stringify(sanitizeEvent(event))}\n`, 'utf8'); + }); + } catch (error) { + this.logger?.warn('member work sync audit journal append failed', { + teamName: event.teamName, + memberName: event.memberName, + event: event.event, + error: String(error), + }); + } + } +} diff --git a/src/features/member-work-sync/main/infrastructure/JsonMemberWorkSyncStore.ts b/src/features/member-work-sync/main/infrastructure/JsonMemberWorkSyncStore.ts index 1b702de8..7fb1901c 100644 --- a/src/features/member-work-sync/main/infrastructure/JsonMemberWorkSyncStore.ts +++ b/src/features/member-work-sync/main/infrastructure/JsonMemberWorkSyncStore.ts @@ -1,7 +1,8 @@ import { withFileLock } from '@main/services/team/fileLock'; import { atomicWriteAsync } from '@main/utils/atomicWrite'; import { createHash } from 'crypto'; -import { mkdir, readFile, rename } from 'fs/promises'; +import { mkdir, readdir, readFile, rename } from 'fs/promises'; +import { dirname, join } from 'path'; import { assessMemberWorkSyncPhase2Readiness } from '../../core/domain'; @@ -22,13 +23,15 @@ import type { MemberWorkSyncTeamMetrics, } from '../../contracts'; import type { + MemberWorkSyncAuditJournalPort, + MemberWorkSyncLoggerPort, MemberWorkSyncOutboxStorePort, MemberWorkSyncReportStorePort, MemberWorkSyncStatusStorePort, } from '../../core/application'; import type { MemberWorkSyncStorePaths } from './MemberWorkSyncStorePaths'; -interface StoreFile { +interface LegacyStatusFile { schemaVersion: 1; members: Record; metrics?: { @@ -36,29 +39,90 @@ interface StoreFile { }; } -interface PendingReportFile { +interface MemberStatusFile { + schemaVersion: 2; + status: MemberWorkSyncStatus; +} + +interface MetricsIndexMember { + memberName: string; + state: MemberWorkSyncStatusState; + agendaFingerprint: string; + actionableCount: number; + evaluatedAt: string; + providerId?: string; +} + +interface MetricsIndexFile { + schemaVersion: 2; + members: Record; + recentEvents: MemberWorkSyncMetricEvent[]; +} + +interface LegacyPendingReportFile { schemaVersion: 1; intents: Record; } -interface OutboxFile { +interface MemberReportsFile { + schemaVersion: 2; + intents: Record; +} + +interface PendingReportsIndexFile { + schemaVersion: 2; + items: Record< + string, + { + memberKey: string; + memberName: string; + status: MemberWorkSyncReportIntent['status']; + recordedAt: string; + processedAt?: string; + } + >; +} + +interface LegacyOutboxFile { schemaVersion: 1; items: Record; } +interface MemberOutboxFile { + schemaVersion: 2; + items: Record; +} + +interface OutboxIndexFile { + schemaVersion: 2; + items: Record< + string, + { + memberKey: string; + memberName: string; + status: MemberWorkSyncOutboxItem['status']; + nextAttemptAt?: string; + updatedAt: string; + createdAt: string; + } + >; +} + +type OutboxIndexRoute = OutboxIndexFile['items'][string]; +type OutboxDueRoute = [string, OutboxIndexRoute]; + +export interface JsonMemberWorkSyncStoreDeps { + auditJournal?: MemberWorkSyncAuditJournalPort; + logger?: MemberWorkSyncLoggerPort; + now?: () => Date; +} + function normalizeMemberKey(memberName: string): string { return memberName.trim().toLowerCase(); } -function isStoreFile(value: unknown): value is StoreFile { - return ( - value != null && - typeof value === 'object' && - (value as StoreFile).schemaVersion === 1 && - (value as StoreFile).members != null && - typeof (value as StoreFile).members === 'object' && - !Array.isArray((value as StoreFile).members) - ); +function emptyMetricsIndex(): MetricsIndexFile { + return { schemaVersion: 2, members: {}, recentEvents: [] }; } function emptyStateCounts(): Record { @@ -72,25 +136,101 @@ function emptyStateCounts(): Record { }; } -function isPendingReportFile(value: unknown): value is PendingReportFile { +function isLegacyStatusFile(value: unknown): value is LegacyStatusFile { return ( value != null && typeof value === 'object' && - (value as PendingReportFile).schemaVersion === 1 && - (value as PendingReportFile).intents != null && - typeof (value as PendingReportFile).intents === 'object' && - !Array.isArray((value as PendingReportFile).intents) + (value as LegacyStatusFile).schemaVersion === 1 && + (value as LegacyStatusFile).members != null && + typeof (value as LegacyStatusFile).members === 'object' && + !Array.isArray((value as LegacyStatusFile).members) ); } -function isOutboxFile(value: unknown): value is OutboxFile { +function isMemberStatusFile(value: unknown): value is MemberStatusFile { return ( value != null && typeof value === 'object' && - (value as OutboxFile).schemaVersion === 1 && - (value as OutboxFile).items != null && - typeof (value as OutboxFile).items === 'object' && - !Array.isArray((value as OutboxFile).items) + (value as MemberStatusFile).schemaVersion === 2 && + (value as MemberStatusFile).status != null && + typeof (value as MemberStatusFile).status === 'object' + ); +} + +function isMetricsIndexFile(value: unknown): value is MetricsIndexFile { + return ( + value != null && + typeof value === 'object' && + (value as MetricsIndexFile).schemaVersion === 2 && + (value as MetricsIndexFile).members != null && + typeof (value as MetricsIndexFile).members === 'object' && + Array.isArray((value as MetricsIndexFile).recentEvents) + ); +} + +function isLegacyPendingReportFile(value: unknown): value is LegacyPendingReportFile { + return ( + value != null && + typeof value === 'object' && + (value as LegacyPendingReportFile).schemaVersion === 1 && + (value as LegacyPendingReportFile).intents != null && + typeof (value as LegacyPendingReportFile).intents === 'object' && + !Array.isArray((value as LegacyPendingReportFile).intents) + ); +} + +function isMemberReportsFile(value: unknown): value is MemberReportsFile { + return ( + value != null && + typeof value === 'object' && + (value as MemberReportsFile).schemaVersion === 2 && + (value as MemberReportsFile).intents != null && + typeof (value as MemberReportsFile).intents === 'object' && + !Array.isArray((value as MemberReportsFile).intents) + ); +} + +function isPendingReportsIndexFile(value: unknown): value is PendingReportsIndexFile { + return ( + value != null && + typeof value === 'object' && + (value as PendingReportsIndexFile).schemaVersion === 2 && + (value as PendingReportsIndexFile).items != null && + typeof (value as PendingReportsIndexFile).items === 'object' && + !Array.isArray((value as PendingReportsIndexFile).items) + ); +} + +function isLegacyOutboxFile(value: unknown): value is LegacyOutboxFile { + return ( + value != null && + typeof value === 'object' && + (value as LegacyOutboxFile).schemaVersion === 1 && + (value as LegacyOutboxFile).items != null && + typeof (value as LegacyOutboxFile).items === 'object' && + !Array.isArray((value as LegacyOutboxFile).items) + ); +} + +function isMemberOutboxFile(value: unknown): value is MemberOutboxFile { + return ( + value != null && + typeof value === 'object' && + (value as MemberOutboxFile).schemaVersion === 2 && + (value as MemberOutboxFile).items != null && + typeof (value as MemberOutboxFile).items === 'object' && + !Array.isArray((value as MemberOutboxFile).items) + ); +} + +function isOutboxIndexFile(value: unknown): value is OutboxIndexFile { + return ( + value != null && + typeof value === 'object' && + (value as OutboxIndexFile).schemaVersion === 2 && + (value as OutboxIndexFile).items != null && + typeof (value as OutboxIndexFile).items === 'object' && + !Array.isArray((value as OutboxIndexFile).items) ); } @@ -112,6 +252,22 @@ function canClaimOutboxItem(item: MemberWorkSyncOutboxItem, nowIso: string): boo return item.nextAttemptAt <= nowIso; } +function getDueOutboxRoutes( + index: OutboxIndexFile, + nowIso: string, + limit: number +): OutboxDueRoute[] { + return Object.entries(index.items) + .filter(([, route]) => route.status === 'pending' || route.status === 'failed_retryable') + .filter(([, route]) => !route.nextAttemptAt || route.nextAttemptAt <= nowIso) + .sort((left, right) => { + const leftTime = left[1].nextAttemptAt ?? left[1].updatedAt; + const rightTime = right[1].nextAttemptAt ?? right[1].updatedAt; + return leftTime.localeCompare(rightTime); + }) + .slice(0, Math.max(0, limit)); +} + function stableStringify(value: unknown): string { if (value == null || typeof value !== 'object') { return JSON.stringify(value); @@ -216,16 +372,62 @@ function buildMetricEvents(status: MemberWorkSyncStatus): MemberWorkSyncMetricEv return events; } -function appendMetricEvents(file: StoreFile, status: MemberWorkSyncStatus): void { - const current = file.metrics?.recentEvents ?? []; - const byId = new Map(current.map((event) => [event.id, event])); +function appendMetricEvents(file: MetricsIndexFile, status: MemberWorkSyncStatus): void { + const byId = new Map(file.recentEvents.map((event) => [event.id, event])); for (const event of buildMetricEvents(status)) { byId.set(event.id, event); } - file.metrics = { - recentEvents: [...byId.values()] - .sort((left, right) => left.recordedAt.localeCompare(right.recordedAt)) - .slice(-200), + file.recentEvents = [...byId.values()] + .sort((left, right) => left.recordedAt.localeCompare(right.recordedAt)) + .slice(-200); +} + +function updateMetricsMember( + file: MetricsIndexFile, + status: MemberWorkSyncStatus, + memberKey: string +): void { + file.members[memberKey] = { + memberName: status.memberName, + state: status.state, + agendaFingerprint: status.agenda.fingerprint, + actionableCount: status.agenda.items.length, + evaluatedAt: status.evaluatedAt, + ...(status.providerId ? { providerId: status.providerId } : {}), + }; + appendMetricEvents(file, status); +} + +function toMetrics(teamName: string, file: MetricsIndexFile): MemberWorkSyncTeamMetrics { + const stateCounts = emptyStateCounts(); + const members = Object.values(file.members); + let actionableItemCount = 0; + for (const member of members) { + stateCounts[member.state] += 1; + actionableItemCount += member.actionableCount; + } + const recentEvents = [...file.recentEvents].sort((left, right) => + left.recordedAt.localeCompare(right.recordedAt) + ); + const metrics = { + teamName, + generatedAt: new Date().toISOString(), + memberCount: members.length, + stateCounts, + actionableItemCount, + wouldNudgeCount: recentEvents.filter((event) => event.kind === 'would_nudge').length, + fingerprintChangeCount: recentEvents.filter((event) => event.kind === 'fingerprint_changed') + .length, + reportAcceptedCount: recentEvents.filter((event) => event.kind === 'report_accepted').length, + reportRejectedCount: recentEvents.filter((event) => event.kind === 'report_rejected').length, + recentEvents, + }; + return { + ...metrics, + phase2Readiness: assessMemberWorkSyncPhase2Readiness({ + memberCount: metrics.memberCount, + recentEvents: metrics.recentEvents, + }), }; } @@ -237,6 +439,34 @@ async function quarantineFile(filePath: string): Promise { } } +async function readJsonFile( + filePath: string, + guard: (value: unknown) => value is T, + fallback: T, + options: { quarantineInvalid?: boolean } = {} +): Promise { + try { + const raw = await readFile(filePath, 'utf8'); + const parsed = JSON.parse(raw); + if (guard(parsed)) { + return parsed; + } + if (options.quarantineInvalid) { + await quarantineFile(filePath); + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT' && options.quarantineInvalid) { + await quarantineFile(filePath); + } + } + return fallback; +} + +async function writeJsonFile(filePath: string, value: unknown): Promise { + await mkdir(dirname(filePath), { recursive: true }); + await atomicWriteAsync(filePath, `${JSON.stringify(value, null, 2)}\n`); +} + export class JsonMemberWorkSyncStore implements MemberWorkSyncStatusStorePort, @@ -244,94 +474,166 @@ export class JsonMemberWorkSyncStore MemberWorkSyncOutboxStorePort { private readonly writeQueues = new Map>(); + private readonly now: () => Date; - constructor(private readonly paths: MemberWorkSyncStorePaths) {} + constructor( + private readonly paths: MemberWorkSyncStorePaths, + private readonly deps: JsonMemberWorkSyncStoreDeps = {} + ) { + this.now = deps.now ?? (() => new Date()); + } async read(input: { teamName: string; memberName: string; }): Promise { - const file = await this.readFile(input.teamName); - return file.members[normalizeMemberKey(input.memberName)] ?? null; + const memberFile = await this.readMemberStatusFile(input.teamName, input.memberName); + if (memberFile) { + return memberFile.status; + } + + const legacy = await this.readLegacyStatusFile(input.teamName); + const legacyStatus = legacy.members[normalizeMemberKey(input.memberName)] ?? null; + if (legacyStatus) { + await this.appendAudit({ + teamName: input.teamName, + memberName: input.memberName, + event: 'legacy_fallback_used', + source: 'json_store', + reason: 'status_v1', + }); + } + return legacyStatus; } async write(status: MemberWorkSyncStatus): Promise { + const memberKey = this.paths.getMemberKey(status.memberName); + await this.paths.ensureMemberWorkSyncDir(status.teamName, status.memberName); await this.enqueue(status.teamName, async () => { - await withFileLock(this.paths.getStatusPath(status.teamName), async () => { - const existing = await this.readFile(status.teamName); - existing.members[normalizeMemberKey(status.memberName)] = status; - appendMetricEvents(existing, status); - await mkdir(this.paths.getTeamDir(status.teamName), { recursive: true }); - await atomicWriteAsync( - this.paths.getStatusPath(status.teamName), - JSON.stringify(existing, null, 2) + await withFileLock(this.paths.getMetricsIndexPath(status.teamName), async () => { + await withFileLock( + this.paths.getMemberStatusPath(status.teamName, status.memberName), + async () => { + const metrics = await this.readMetricsIndexFile(status.teamName); + updateMetricsMember(metrics, status, memberKey); + await this.writeMemberStatusFile(status); + await this.writeMetricsIndexFile(status.teamName, metrics); + } + ); + }); + }); + await this.appendAudit({ + teamName: status.teamName, + memberName: status.memberName, + event: 'status_written', + source: 'json_store', + agendaFingerprint: status.agenda.fingerprint, + state: status.state, + actionableCount: status.agenda.items.length, + ...(status.shadow?.triggerReasons ? { triggerReasons: status.shadow.triggerReasons } : {}), + ...(status.providerId ? { providerId: status.providerId } : {}), + }); + } + + async readTeamMetrics(teamName: string): Promise { + let file = await this.readMetricsIndexFile(teamName); + if (Object.keys(file.members).length === 0 && file.recentEvents.length === 0) { + const repaired = await this.repairMetricsIndex(teamName); + if (repaired) { + file = repaired; + } + } + return toMetrics(teamName, file); + } + + async appendPendingReport(request: MemberWorkSyncReportRequest, reason: string): Promise { + const id = buildPendingReportIntentId(request); + const memberKey = this.paths.getMemberKey(request.memberName); + await this.paths.ensureMemberWorkSyncDir(request.teamName, request.memberName); + await this.enqueue(request.teamName, async () => { + await withFileLock(this.paths.getPendingReportsIndexPath(request.teamName), async () => { + await withFileLock( + this.paths.getMemberReportsPath(request.teamName, request.memberName), + async () => { + const reports = await this.readMemberReportsFile(request.teamName, request.memberName); + const current = reports.intents[id]; + if (current && current.status !== 'pending') { + return; + } + const intent: MemberWorkSyncReportIntent = { + id, + teamName: request.teamName, + memberName: request.memberName, + request, + reason: current?.reason ?? reason, + status: 'pending', + recordedAt: current?.recordedAt ?? this.now().toISOString(), + }; + reports.intents[id] = intent; + await this.writeMemberReportsFile(request.teamName, request.memberName, reports); + + const index = await this.readPendingReportsIndexFile(request.teamName); + index.items[id] = { + memberKey, + memberName: request.memberName, + status: intent.status, + recordedAt: intent.recordedAt, + }; + await this.writePendingReportsIndexFile(request.teamName, index); + } ); }); }); } - async readTeamMetrics(teamName: string): Promise { - const file = await this.readFile(teamName); - const stateCounts = emptyStateCounts(); - const members = Object.values(file.members); - let actionableItemCount = 0; - for (const status of members) { - stateCounts[status.state] += 1; - actionableItemCount += status.agenda.items.length; - } - const recentEvents = [...(file.metrics?.recentEvents ?? [])].sort((left, right) => - left.recordedAt.localeCompare(right.recordedAt) - ); - const metrics = { - teamName, - generatedAt: new Date().toISOString(), - memberCount: members.length, - stateCounts, - actionableItemCount, - wouldNudgeCount: recentEvents.filter((event) => event.kind === 'would_nudge').length, - fingerprintChangeCount: recentEvents.filter((event) => event.kind === 'fingerprint_changed') - .length, - reportAcceptedCount: recentEvents.filter((event) => event.kind === 'report_accepted').length, - reportRejectedCount: recentEvents.filter((event) => event.kind === 'report_rejected').length, - recentEvents, - }; - return { - ...metrics, - phase2Readiness: assessMemberWorkSyncPhase2Readiness({ - memberCount: metrics.memberCount, - recentEvents: metrics.recentEvents, - }), - }; - } - - async appendPendingReport(request: MemberWorkSyncReportRequest, reason: string): Promise { - const id = buildPendingReportIntentId(request); - await this.enqueue(request.teamName, async () => { - await withFileLock(this.paths.getPendingReportsPath(request.teamName), async () => { - const existing = await this.readPendingFile(request.teamName); - const current = existing.intents[id]; - if (current && current.status !== 'pending') { - return; - } - existing.intents[id] = { - id, - teamName: request.teamName, - memberName: request.memberName, - request, - reason: current?.reason ?? reason, - status: 'pending', - recordedAt: current?.recordedAt ?? new Date().toISOString(), - }; - await this.writePendingFile(request.teamName, existing); - }); - }); - } - async listPendingReports(teamName: string): Promise { - const file = await this.readPendingFile(teamName); - return Object.values(file.intents) - .filter((intent) => intent.status === 'pending') - .sort((left, right) => left.recordedAt.localeCompare(right.recordedAt)); + let index = await this.readPendingReportsIndexFile(teamName); + if (Object.keys(index.items).length === 0) { + await this.enqueue(teamName, async () => { + await withFileLock(this.paths.getPendingReportsIndexPath(teamName), async () => { + index = await this.readPendingReportsIndexFile(teamName); + if (Object.keys(index.items).length === 0) { + index = await this.repairPendingReportsIndex(teamName); + } + }); + }); + } + let staleIndex = false; + const pending: MemberWorkSyncReportIntent[] = []; + for (const [id, route] of Object.entries(index.items)) { + if (route.status !== 'pending') { + continue; + } + const file = await this.readMemberReportsFile(teamName, route.memberName); + const intent = file.intents[id]; + if (intent?.status === 'pending') { + pending.push(intent); + } else { + staleIndex = true; + } + } + const missingIndexedPending = staleIndex + ? false + : await this.hasMissingIndexedPendingReport(teamName, index); + if (staleIndex || missingIndexedPending) { + await this.enqueue(teamName, async () => { + await withFileLock(this.paths.getPendingReportsIndexPath(teamName), async () => { + index = await this.repairPendingReportsIndex(teamName); + }); + }); + pending.length = 0; + for (const [id, route] of Object.entries(index.items)) { + if (route.status !== 'pending') { + continue; + } + const file = await this.readMemberReportsFile(teamName, route.memberName); + const intent = file.intents[id]; + if (intent?.status === 'pending') { + pending.push(intent); + } + } + } + return pending.sort((left, right) => left.recordedAt.localeCompare(right.recordedAt)); } async markPendingReportProcessed( @@ -344,19 +646,38 @@ export class JsonMemberWorkSyncStore } ): Promise { await this.enqueue(teamName, async () => { - await withFileLock(this.paths.getPendingReportsPath(teamName), async () => { - const existing = await this.readPendingFile(teamName); - const current = existing.intents[id]; - if (current?.status !== 'pending') { + await withFileLock(this.paths.getPendingReportsIndexPath(teamName), async () => { + let index = await this.readPendingReportsIndexFile(teamName); + if (!index.items[id]) { + index = await this.repairPendingReportsIndex(teamName); + } + const route = index.items[id]; + if (!route) { return; } - existing.intents[id] = { - ...current, - status: result.status, - resultCode: result.resultCode, - processedAt: result.processedAt, - }; - await this.writePendingFile(teamName, existing); + await withFileLock( + this.paths.getMemberReportsPath(teamName, route.memberName), + async () => { + const reports = await this.readMemberReportsFile(teamName, route.memberName); + const current = reports.intents[id]; + if (current?.status !== 'pending') { + return; + } + reports.intents[id] = { + ...current, + status: result.status, + resultCode: result.resultCode, + processedAt: result.processedAt, + }; + await this.writeMemberReportsFile(teamName, route.memberName, reports); + index.items[id] = { + ...route, + status: result.status, + processedAt: result.processedAt, + }; + await this.writePendingReportsIndexFile(teamName, index); + } + ); }); }); } @@ -365,63 +686,73 @@ export class JsonMemberWorkSyncStore input: MemberWorkSyncOutboxEnsureInput ): Promise { let result: MemberWorkSyncOutboxEnsureResult | null = null; + const memberKey = this.paths.getMemberKey(input.memberName); + await this.paths.ensureMemberWorkSyncDir(input.teamName, input.memberName); await this.enqueue(input.teamName, async () => { - await withFileLock(this.paths.getOutboxPath(input.teamName), async () => { - const existing = await this.readOutboxFile(input.teamName); - const current = existing.items[input.id]; - if (current) { - if (current.payloadHash !== input.payloadHash) { - result = { - ok: false, - outcome: 'payload_conflict', - item: current, - existingPayloadHash: current.payloadHash, - requestedPayloadHash: input.payloadHash, - }; - return; - } + await withFileLock(this.paths.getOutboxIndexPath(input.teamName), async () => { + await withFileLock( + this.paths.getMemberOutboxPath(input.teamName, input.memberName), + async () => { + const outbox = await this.readMemberOutboxFile(input.teamName, input.memberName); + const current = outbox.items[input.id]; + if (current) { + if (current.payloadHash !== input.payloadHash) { + result = { + ok: false, + outcome: 'payload_conflict', + item: current, + existingPayloadHash: current.payloadHash, + requestedPayloadHash: input.payloadHash, + }; + return; + } - if (canReviveOutboxItem(current.status)) { - const next: MemberWorkSyncOutboxItem = { - ...current, + if (canReviveOutboxItem(current.status)) { + const next: MemberWorkSyncOutboxItem = { + ...current, + status: 'pending', + updatedAt: input.nowIso, + }; + const nextAttemptAt = input.nextAttemptAt ?? current.nextAttemptAt; + if (nextAttemptAt) { + next.nextAttemptAt = nextAttemptAt; + } else { + delete next.nextAttemptAt; + } + delete next.claimedBy; + delete next.claimedAt; + delete next.lastError; + outbox.items[input.id] = next; + await this.writeMemberOutboxFile(input.teamName, input.memberName, outbox); + await this.upsertOutboxIndexItem(input.teamName, next, memberKey); + result = { ok: true, outcome: 'existing', item: next }; + return; + } + + await this.upsertOutboxIndexItem(input.teamName, current, memberKey); + result = { ok: true, outcome: 'existing', item: current }; + return; + } + + const item: MemberWorkSyncOutboxItem = { + id: input.id, + teamName: input.teamName, + memberName: input.memberName, + agendaFingerprint: input.agendaFingerprint, + payloadHash: input.payloadHash, + payload: input.payload, status: 'pending', + attemptGeneration: 0, + ...(input.nextAttemptAt ? { nextAttemptAt: input.nextAttemptAt } : {}), + createdAt: input.nowIso, updatedAt: input.nowIso, }; - const nextAttemptAt = input.nextAttemptAt ?? current.nextAttemptAt; - if (nextAttemptAt) { - next.nextAttemptAt = nextAttemptAt; - } else { - delete next.nextAttemptAt; - } - delete next.claimedBy; - delete next.claimedAt; - delete next.lastError; - existing.items[input.id] = next; - await this.writeOutboxFile(input.teamName, existing); - result = { ok: true, outcome: 'existing', item: existing.items[input.id] }; - return; + outbox.items[input.id] = item; + await this.writeMemberOutboxFile(input.teamName, input.memberName, outbox); + await this.upsertOutboxIndexItem(input.teamName, item, memberKey); + result = { ok: true, outcome: 'created', item }; } - - result = { ok: true, outcome: 'existing', item: current }; - return; - } - - const item: MemberWorkSyncOutboxItem = { - id: input.id, - teamName: input.teamName, - memberName: input.memberName, - agendaFingerprint: input.agendaFingerprint, - payloadHash: input.payloadHash, - payload: input.payload, - status: 'pending', - attemptGeneration: 0, - ...(input.nextAttemptAt ? { nextAttemptAt: input.nextAttemptAt } : {}), - createdAt: input.nowIso, - updatedAt: input.nowIso, - }; - existing.items[input.id] = item; - await this.writeOutboxFile(input.teamName, existing); - result = { ok: true, outcome: 'created', item }; + ); }); }); @@ -434,33 +765,54 @@ export class JsonMemberWorkSyncStore async claimDue(input: MemberWorkSyncOutboxClaimInput): Promise { const claimed: MemberWorkSyncOutboxItem[] = []; await this.enqueue(input.teamName, async () => { - await withFileLock(this.paths.getOutboxPath(input.teamName), async () => { - const existing = await this.readOutboxFile(input.teamName); - const due = Object.values(existing.items) - .filter((item) => canClaimOutboxItem(item, input.nowIso)) - .sort((left, right) => { - const leftTime = left.nextAttemptAt ?? left.updatedAt; - const rightTime = right.nextAttemptAt ?? right.updatedAt; - return leftTime.localeCompare(rightTime); - }) - .slice(0, Math.max(0, input.limit)); - - for (const item of due) { - const next: MemberWorkSyncOutboxItem = { - ...item, - status: 'claimed', - attemptGeneration: item.attemptGeneration + 1, - claimedBy: input.claimedBy, - claimedAt: input.nowIso, - updatedAt: input.nowIso, - }; - delete next.lastError; - existing.items[item.id] = next; - claimed.push(next); + await withFileLock(this.paths.getOutboxIndexPath(input.teamName), async () => { + let index = await this.readOutboxIndexFile(input.teamName); + if (Object.keys(index.items).length === 0) { + index = await this.repairOutboxIndex(input.teamName); + } + let dueRoutes = getDueOutboxRoutes(index, input.nowIso, input.limit); + if ( + dueRoutes.length > 0 && + dueRoutes.length < Math.max(0, input.limit) && + (await this.hasMissingIndexedDueOutboxItem(input.teamName, index, input.nowIso)) + ) { + index = await this.repairOutboxIndex(input.teamName); + dueRoutes = getDueOutboxRoutes(index, input.nowIso, input.limit); } - if (due.length > 0) { - await this.writeOutboxFile(input.teamName, existing); + let staleIndex = false; + for (const [id, route] of dueRoutes) { + await withFileLock( + this.paths.getMemberOutboxPath(input.teamName, route.memberName), + async () => { + const outbox = await this.readMemberOutboxFile(input.teamName, route.memberName); + const item = outbox.items[id]; + if (!item || !canClaimOutboxItem(item, input.nowIso)) { + delete index.items[id]; + staleIndex = true; + return; + } + const next: MemberWorkSyncOutboxItem = { + ...item, + status: 'claimed', + attemptGeneration: item.attemptGeneration + 1, + claimedBy: input.claimedBy, + claimedAt: input.nowIso, + updatedAt: input.nowIso, + }; + delete next.lastError; + outbox.items[id] = next; + await this.writeMemberOutboxFile(input.teamName, route.memberName, outbox); + index.items[id] = toOutboxIndexItem(next, route.memberKey); + claimed.push(next); + } + ); + } + + if (staleIndex) { + index = await this.repairOutboxIndex(input.teamName); + } else if (dueRoutes.length > 0) { + await this.writeOutboxIndexFile(input.teamName, index); } }); }); @@ -519,69 +871,178 @@ export class JsonMemberWorkSyncStore async countRecentDelivered( input: MemberWorkSyncOutboxCountRecentDeliveredInput ): Promise { - const file = await this.readOutboxFile(input.teamName); - return Object.values(file.items).filter( + let index = await this.readOutboxIndexFile(input.teamName); + if (Object.keys(index.items).length === 0) { + await this.enqueue(input.teamName, async () => { + await withFileLock(this.paths.getOutboxIndexPath(input.teamName), async () => { + index = await this.readOutboxIndexFile(input.teamName); + if (Object.keys(index.items).length === 0) { + index = await this.repairOutboxIndex(input.teamName); + } + }); + }); + } + const indexedCount = Object.values(index.items).filter( (item) => - item.memberName.trim().toLowerCase() === input.memberName.trim().toLowerCase() && + normalizeMemberKey(item.memberName) === normalizeMemberKey(input.memberName) && item.status === 'delivered' && item.updatedAt >= input.sinceIso ).length; - } - - private async readFile(teamName: string): Promise { - const filePath = this.paths.getStatusPath(teamName); - try { - const raw = await readFile(filePath, 'utf8'); - const parsed = JSON.parse(raw); - if (isStoreFile(parsed)) { - return parsed; - } - await quarantineFile(filePath); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { - await quarantineFile(filePath); - } + const memberOutbox = await this.readMemberOutboxFile(input.teamName, input.memberName); + const memberFileCount = Object.values(memberOutbox.items).filter( + (item) => item.status === 'delivered' && item.updatedAt >= input.sinceIso + ).length; + if (memberFileCount > indexedCount) { + await this.enqueue(input.teamName, async () => { + await withFileLock(this.paths.getOutboxIndexPath(input.teamName), async () => { + await this.repairOutboxIndex(input.teamName); + }); + }); } - return { schemaVersion: 1, members: {}, metrics: { recentEvents: [] } }; + return Math.max(indexedCount, memberFileCount); } - private async readPendingFile(teamName: string): Promise { - const filePath = this.paths.getPendingReportsPath(teamName); - try { - const raw = await readFile(filePath, 'utf8'); - const parsed = JSON.parse(raw); - if (isPendingReportFile(parsed)) { - return parsed; - } - await quarantineFile(filePath); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { - await quarantineFile(filePath); - } - } - return { schemaVersion: 1, intents: {} }; + private async readLegacyStatusFile(teamName: string): Promise { + return readJsonFile( + this.paths.getLegacyStatusPath(teamName), + isLegacyStatusFile, + { schemaVersion: 1, members: {}, metrics: { recentEvents: [] } }, + { quarantineInvalid: true } + ); } - private async readOutboxFile(teamName: string): Promise { - const filePath = this.paths.getOutboxPath(teamName); - try { - const raw = await readFile(filePath, 'utf8'); - const parsed = JSON.parse(raw); - if (isOutboxFile(parsed)) { - return parsed; - } - await quarantineFile(filePath); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { - await quarantineFile(filePath); - } - } - return { schemaVersion: 1, items: {} }; + private async readMemberStatusFile( + teamName: string, + memberName: string + ): Promise { + const file = await readJsonFile( + this.paths.getMemberStatusPath(teamName, memberName), + (value): value is MemberStatusFile | null => value === null || isMemberStatusFile(value), + null, + { quarantineInvalid: true } + ); + return file; } - private async writeOutboxFile(teamName: string, file: OutboxFile): Promise { - await mkdir(this.paths.getTeamDir(teamName), { recursive: true }); - await atomicWriteAsync(this.paths.getOutboxPath(teamName), JSON.stringify(file, null, 2)); + private async writeMemberStatusFile(status: MemberWorkSyncStatus): Promise { + await writeJsonFile(this.paths.getMemberStatusPath(status.teamName, status.memberName), { + schemaVersion: 2, + status, + } satisfies MemberStatusFile); + } + + private async readMetricsIndexFile(teamName: string): Promise { + return readJsonFile( + this.paths.getMetricsIndexPath(teamName), + isMetricsIndexFile, + emptyMetricsIndex(), + { + quarantineInvalid: true, + } + ); + } + + private async writeMetricsIndexFile(teamName: string, file: MetricsIndexFile): Promise { + await writeJsonFile(this.paths.getMetricsIndexPath(teamName), file); + } + + private async readLegacyPendingFile(teamName: string): Promise { + return readJsonFile( + this.paths.getLegacyPendingReportsPath(teamName), + isLegacyPendingReportFile, + { schemaVersion: 1, intents: {} }, + { quarantineInvalid: true } + ); + } + + private async readMemberReportsFile( + teamName: string, + memberName: string + ): Promise { + return readJsonFile( + this.paths.getMemberReportsPath(teamName, memberName), + isMemberReportsFile, + { schemaVersion: 2, intents: {} }, + { quarantineInvalid: true } + ); + } + + private async writeMemberReportsFile( + teamName: string, + memberName: string, + file: MemberReportsFile + ): Promise { + await this.paths.ensureMemberWorkSyncDir(teamName, memberName); + await writeJsonFile(this.paths.getMemberReportsPath(teamName, memberName), file); + } + + private async readPendingReportsIndexFile(teamName: string): Promise { + return readJsonFile( + this.paths.getPendingReportsIndexPath(teamName), + isPendingReportsIndexFile, + { schemaVersion: 2, items: {} }, + { quarantineInvalid: true } + ); + } + + private async writePendingReportsIndexFile( + teamName: string, + file: PendingReportsIndexFile + ): Promise { + await writeJsonFile(this.paths.getPendingReportsIndexPath(teamName), file); + } + + private async readLegacyOutboxFile(teamName: string): Promise { + return readJsonFile( + this.paths.getLegacyOutboxPath(teamName), + isLegacyOutboxFile, + { schemaVersion: 1, items: {} }, + { quarantineInvalid: true } + ); + } + + private async readMemberOutboxFile( + teamName: string, + memberName: string + ): Promise { + return readJsonFile( + this.paths.getMemberOutboxPath(teamName, memberName), + isMemberOutboxFile, + { schemaVersion: 2, items: {} }, + { quarantineInvalid: true } + ); + } + + private async writeMemberOutboxFile( + teamName: string, + memberName: string, + file: MemberOutboxFile + ): Promise { + await this.paths.ensureMemberWorkSyncDir(teamName, memberName); + await writeJsonFile(this.paths.getMemberOutboxPath(teamName, memberName), file); + } + + private async readOutboxIndexFile(teamName: string): Promise { + return readJsonFile( + this.paths.getOutboxIndexPath(teamName), + isOutboxIndexFile, + { schemaVersion: 2, items: {} }, + { quarantineInvalid: true } + ); + } + + private async writeOutboxIndexFile(teamName: string, file: OutboxIndexFile): Promise { + await writeJsonFile(this.paths.getOutboxIndexPath(teamName), file); + } + + private async upsertOutboxIndexItem( + teamName: string, + item: MemberWorkSyncOutboxItem, + memberKey: string + ): Promise { + const index = await this.readOutboxIndexFile(teamName); + index.items[item.id] = toOutboxIndexItem(item, memberKey); + await this.writeOutboxIndexFile(teamName, index); } private async updateOutboxItem( @@ -590,24 +1051,305 @@ export class JsonMemberWorkSyncStore updater: (current: MemberWorkSyncOutboxItem | undefined) => MemberWorkSyncOutboxItem | undefined ): Promise { await this.enqueue(teamName, async () => { - await withFileLock(this.paths.getOutboxPath(teamName), async () => { - const existing = await this.readOutboxFile(teamName); - const next = updater(existing.items[id]); - if (!next) { + await withFileLock(this.paths.getOutboxIndexPath(teamName), async () => { + let index = await this.readOutboxIndexFile(teamName); + if (!index.items[id]) { + index = await this.repairOutboxIndex(teamName); + } + const route = index.items[id]; + if (!route) { return; } - existing.items[id] = next; - await this.writeOutboxFile(teamName, existing); + await withFileLock(this.paths.getMemberOutboxPath(teamName, route.memberName), async () => { + const outbox = await this.readMemberOutboxFile(teamName, route.memberName); + const next = updater(outbox.items[id]); + if (!next) { + return; + } + outbox.items[id] = next; + await this.writeMemberOutboxFile(teamName, route.memberName, outbox); + index.items[id] = toOutboxIndexItem(next, route.memberKey); + await this.writeOutboxIndexFile(teamName, index); + }); }); }); } - private async writePendingFile(teamName: string, file: PendingReportFile): Promise { - await mkdir(this.paths.getTeamDir(teamName), { recursive: true }); - await atomicWriteAsync( - this.paths.getPendingReportsPath(teamName), - JSON.stringify(file, null, 2) - ); + private async repairMetricsIndex(teamName: string): Promise { + let repaired: MetricsIndexFile | null = null; + await this.enqueue(teamName, async () => { + await withFileLock(this.paths.getMetricsIndexPath(teamName), async () => { + const current = await this.readMetricsIndexFile(teamName); + if (Object.keys(current.members).length > 0 || current.recentEvents.length > 0) { + repaired = current; + return; + } + + const next = emptyMetricsIndex(); + const memberStatuses = await this.scanMemberStatuses(teamName); + for (const status of memberStatuses) { + updateMetricsMember(next, status, this.paths.getMemberKey(status.memberName)); + } + const legacy = await this.readLegacyStatusFile(teamName); + for (const status of Object.values(legacy.members)) { + const memberKey = this.paths.getMemberKey(status.memberName); + if (!next.members[memberKey]) { + updateMetricsMember(next, status, memberKey); + } + } + for (const event of legacy.metrics?.recentEvents ?? []) { + if (!next.recentEvents.some((existing) => existing.id === event.id)) { + next.recentEvents.push(event); + } + } + next.recentEvents.sort((left, right) => left.recordedAt.localeCompare(right.recordedAt)); + next.recentEvents = next.recentEvents.slice(-200); + if (Object.keys(next.members).length === 0 && next.recentEvents.length === 0) { + repaired = null; + return; + } + await this.writeMetricsIndexFile(teamName, next); + repaired = next; + }); + }); + + const repairedIndex = repaired as MetricsIndexFile | null; + if (!repairedIndex) { + return null; + } + for (const member of Object.values(repairedIndex.members)) { + await this.appendAudit({ + teamName, + memberName: member.memberName, + event: 'index_repaired', + source: 'json_store', + reason: 'metrics', + agendaFingerprint: member.agendaFingerprint, + state: member.state, + actionableCount: member.actionableCount, + ...(member.providerId ? { providerId: member.providerId } : {}), + }); + } + return repairedIndex; + } + + private async repairPendingReportsIndex(teamName: string): Promise { + const index: PendingReportsIndexFile = { schemaVersion: 2, items: {} }; + const repairedMembers = new Set(); + const legacyMembers = new Set(); + for (const { memberName, reports } of await this.scanMemberReports(teamName)) { + const memberKey = this.paths.getMemberKey(memberName); + for (const intent of Object.values(reports.intents)) { + index.items[intent.id] = toPendingReportIndexItem(intent, memberKey); + repairedMembers.add(intent.memberName); + } + } + for (const intent of Object.values((await this.readLegacyPendingFile(teamName)).intents)) { + const memberKey = this.paths.getMemberKey(intent.memberName); + if (!index.items[intent.id]) { + await withFileLock( + this.paths.getMemberReportsPath(teamName, intent.memberName), + async () => { + const reports = await this.readMemberReportsFile(teamName, intent.memberName); + reports.intents[intent.id] = intent; + await this.writeMemberReportsFile(teamName, intent.memberName, reports); + } + ); + index.items[intent.id] = toPendingReportIndexItem(intent, memberKey); + repairedMembers.add(intent.memberName); + legacyMembers.add(intent.memberName); + } + } + await this.writePendingReportsIndexFile(teamName, index); + for (const memberName of repairedMembers) { + await this.appendAudit({ + teamName, + memberName, + event: 'index_repaired', + source: 'json_store', + reason: 'pending_reports', + }); + } + for (const memberName of legacyMembers) { + await this.appendAudit({ + teamName, + memberName, + event: 'legacy_fallback_used', + source: 'json_store', + reason: 'pending_reports_v1', + }); + } + return index; + } + + private async repairOutboxIndex(teamName: string): Promise { + const index: OutboxIndexFile = { schemaVersion: 2, items: {} }; + const repairedMembers = new Set(); + const legacyMembers = new Set(); + for (const { memberName, outbox } of await this.scanMemberOutboxes(teamName)) { + const memberKey = this.paths.getMemberKey(memberName); + for (const item of Object.values(outbox.items)) { + index.items[item.id] = toOutboxIndexItem(item, memberKey); + repairedMembers.add(item.memberName); + } + } + for (const item of Object.values((await this.readLegacyOutboxFile(teamName)).items)) { + const memberKey = this.paths.getMemberKey(item.memberName); + if (!index.items[item.id]) { + await withFileLock(this.paths.getMemberOutboxPath(teamName, item.memberName), async () => { + const outbox = await this.readMemberOutboxFile(teamName, item.memberName); + outbox.items[item.id] = item; + await this.writeMemberOutboxFile(teamName, item.memberName, outbox); + }); + index.items[item.id] = toOutboxIndexItem(item, memberKey); + repairedMembers.add(item.memberName); + legacyMembers.add(item.memberName); + } + } + await this.writeOutboxIndexFile(teamName, index); + for (const memberName of repairedMembers) { + await this.appendAudit({ + teamName, + memberName, + event: 'index_repaired', + source: 'json_store', + reason: 'outbox', + }); + } + for (const memberName of legacyMembers) { + await this.appendAudit({ + teamName, + memberName, + event: 'legacy_fallback_used', + source: 'json_store', + reason: 'outbox_v1', + }); + } + return index; + } + + private async scanMemberNames(teamName: string): Promise { + const membersDir = join(this.paths.getTeamRootDir(teamName), 'members'); + const entries = await readdir(membersDir, { withFileTypes: true }).catch(() => []); + const names: string[] = []; + for (const entry of entries) { + if (!entry.isDirectory()) { + continue; + } + const metaPath = join(membersDir, entry.name, 'member.meta.json'); + try { + const raw = await readFile(metaPath, 'utf8'); + const parsed = JSON.parse(raw) as { memberName?: unknown }; + if (typeof parsed.memberName === 'string' && parsed.memberName.trim()) { + names.push(parsed.memberName.trim()); + } + } catch { + // Ignore malformed member storage dirs during repair. + } + } + return names; + } + + private async scanMemberStatuses(teamName: string): Promise { + const statuses: MemberWorkSyncStatus[] = []; + for (const memberName of await this.scanMemberNames(teamName)) { + const file = await this.readMemberStatusFile(teamName, memberName); + if (file) { + statuses.push(file.status); + } + } + return statuses; + } + + private async scanMemberReports( + teamName: string + ): Promise<{ memberName: string; reports: MemberReportsFile }[]> { + const reports: { memberName: string; reports: MemberReportsFile }[] = []; + for (const memberName of await this.scanMemberNames(teamName)) { + reports.push({ memberName, reports: await this.readMemberReportsFile(teamName, memberName) }); + } + return reports; + } + + private async hasMissingIndexedPendingReport( + teamName: string, + index: PendingReportsIndexFile + ): Promise { + const indexedIds = new Set(Object.keys(index.items)); + for (const { reports } of await this.scanMemberReports(teamName)) { + for (const intent of Object.values(reports.intents)) { + if (intent.status === 'pending' && !indexedIds.has(intent.id)) { + return true; + } + } + } + for (const intent of Object.values((await this.readLegacyPendingFile(teamName)).intents)) { + if (intent.status === 'pending' && !indexedIds.has(intent.id)) { + return true; + } + } + return false; + } + + private async scanMemberOutboxes( + teamName: string + ): Promise<{ memberName: string; outbox: MemberOutboxFile }[]> { + const outboxes: { memberName: string; outbox: MemberOutboxFile }[] = []; + for (const memberName of await this.scanMemberNames(teamName)) { + outboxes.push({ memberName, outbox: await this.readMemberOutboxFile(teamName, memberName) }); + } + return outboxes; + } + + private async hasMissingIndexedDueOutboxItem( + teamName: string, + index: OutboxIndexFile, + nowIso: string + ): Promise { + const indexedIds = new Set(Object.keys(index.items)); + for (const { outbox } of await this.scanMemberOutboxes(teamName)) { + for (const item of Object.values(outbox.items)) { + if (canClaimOutboxItem(item, nowIso) && !indexedIds.has(item.id)) { + return true; + } + } + } + for (const item of Object.values((await this.readLegacyOutboxFile(teamName)).items)) { + if (canClaimOutboxItem(item, nowIso) && !indexedIds.has(item.id)) { + return true; + } + } + return false; + } + + private async appendAudit(input: { + teamName: string; + memberName: string; + event: 'status_written' | 'legacy_fallback_used' | 'index_repaired'; + source: string; + agendaFingerprint?: string; + state?: string; + actionableCount?: number; + reason?: string; + triggerReasons?: string[]; + providerId?: string; + }): Promise { + if (!this.deps.auditJournal) { + return; + } + try { + await this.deps.auditJournal.append({ + ...input, + timestamp: this.now().toISOString(), + }); + } catch (error) { + this.deps.logger?.warn('member work sync store audit append failed', { + teamName: input.teamName, + memberName: input.memberName, + event: input.event, + error: String(error), + }); + } } private async enqueue(teamName: string, operation: () => Promise): Promise { @@ -624,3 +1366,30 @@ export class JsonMemberWorkSyncStore await next; } } + +function toOutboxIndexItem( + item: MemberWorkSyncOutboxItem, + memberKey: string +): OutboxIndexFile['items'][string] { + return { + memberKey, + memberName: item.memberName, + status: item.status, + ...(item.nextAttemptAt ? { nextAttemptAt: item.nextAttemptAt } : {}), + updatedAt: item.updatedAt, + createdAt: item.createdAt, + }; +} + +function toPendingReportIndexItem( + intent: MemberWorkSyncReportIntent, + memberKey: string +): PendingReportsIndexFile['items'][string] { + return { + memberKey, + memberName: intent.memberName, + status: intent.status, + recordedAt: intent.recordedAt, + ...(intent.processedAt ? { processedAt: intent.processedAt } : {}), + }; +} diff --git a/src/features/member-work-sync/main/infrastructure/MemberWorkSyncEventQueue.ts b/src/features/member-work-sync/main/infrastructure/MemberWorkSyncEventQueue.ts index 87c963a5..82436b55 100644 --- a/src/features/member-work-sync/main/infrastructure/MemberWorkSyncEventQueue.ts +++ b/src/features/member-work-sync/main/infrastructure/MemberWorkSyncEventQueue.ts @@ -1,4 +1,8 @@ -import type { MemberWorkSyncLoggerPort } from '../../core/application'; +import type { + MemberWorkSyncAuditEvent, + MemberWorkSyncAuditJournalPort, + MemberWorkSyncLoggerPort, +} from '../../core/application'; import type { MemberWorkSyncReconcileContext } from '../../core/application/MemberWorkSyncReconciler'; export type MemberWorkSyncTriggerReason = @@ -43,6 +47,8 @@ export interface MemberWorkSyncEventQueueDeps { quietWindowMs?: number; concurrency?: number; now?: () => number; + nowIso?: () => string; + auditJournal?: MemberWorkSyncAuditJournalPort; logger?: MemberWorkSyncLoggerPort; } @@ -61,6 +67,7 @@ export class MemberWorkSyncEventQueue { private readonly quietWindowMs: number; private readonly concurrency: number; private readonly now: () => number; + private readonly nowIso: () => string; private timer: ReturnType | null = null; private stopped = false; private counters = { @@ -75,6 +82,7 @@ export class MemberWorkSyncEventQueue { this.quietWindowMs = deps.quietWindowMs ?? 90_000; this.concurrency = Math.max(1, deps.concurrency ?? 2); this.now = deps.now ?? Date.now; + this.nowIso = deps.nowIso ?? (() => new Date().toISOString()); } enqueue(input: { @@ -87,19 +95,27 @@ export class MemberWorkSyncEventQueue { return; } + const teamName = input.teamName.trim(); const memberName = input.memberName.trim(); - if (!input.teamName.trim() || !memberName) { + if (!teamName || !memberName) { this.counters.dropped += 1; return; } - const key = keyOf(input.teamName, memberName); + const key = keyOf(teamName, memberName); const runAt = this.now() + (input.runAfterMs ?? this.quietWindowMs); const running = this.running.get(key); if (running) { running.rerunRequested = true; running.triggerReasons.add(input.triggerReason); this.counters.coalesced += 1; + this.appendAudit({ + teamName, + memberName, + event: 'queue_coalesced', + source: 'event_queue', + reason: input.triggerReason, + }); return; } @@ -108,17 +124,31 @@ export class MemberWorkSyncEventQueue { existing.triggerReasons.add(input.triggerReason); existing.runAt = Math.max(existing.runAt, runAt); this.counters.coalesced += 1; + this.appendAudit({ + teamName, + memberName, + event: 'queue_coalesced', + source: 'event_queue', + reason: input.triggerReason, + }); this.schedule(); return; } this.items.set(key, { - teamName: input.teamName, + teamName, memberName, runAt, triggerReasons: new Set([input.triggerReason]), }); this.counters.enqueued += 1; + this.appendAudit({ + teamName, + memberName, + event: 'queue_enqueued', + source: 'event_queue', + reason: input.triggerReason, + }); this.schedule(); } @@ -231,6 +261,13 @@ export class MemberWorkSyncEventQueue { private async executeItem(_key: string, item: QueueItem, running: RunningItem): Promise { if (!(await this.deps.isTeamActive(item.teamName))) { this.counters.dropped += 1; + this.appendAudit({ + teamName: item.teamName, + memberName: item.memberName, + event: 'queue_dropped', + source: 'event_queue', + reason: 'team_inactive', + }); return; } @@ -242,5 +279,31 @@ export class MemberWorkSyncEventQueue { } ); this.counters.reconciled += 1; + this.appendAudit({ + teamName: item.teamName, + memberName: item.memberName, + event: 'queue_reconciled', + source: 'event_queue', + triggerReasons: [...running.triggerReasons].sort(), + }); + } + + private appendAudit(input: Omit): void { + if (!this.deps.auditJournal) { + return; + } + void this.deps.auditJournal + .append({ + ...input, + timestamp: this.nowIso(), + }) + .catch((error: unknown) => { + this.deps.logger?.warn('member work sync queue audit append failed', { + teamName: input.teamName, + memberName: input.memberName, + event: input.event, + error: String(error), + }); + }); } } diff --git a/src/features/member-work-sync/main/infrastructure/MemberWorkSyncStorePaths.ts b/src/features/member-work-sync/main/infrastructure/MemberWorkSyncStorePaths.ts index 55d1facc..27635ace 100644 --- a/src/features/member-work-sync/main/infrastructure/MemberWorkSyncStorePaths.ts +++ b/src/features/member-work-sync/main/infrastructure/MemberWorkSyncStorePaths.ts @@ -1,7 +1,17 @@ import { join } from 'path'; +import { TeamMemberStoragePaths } from '@main/services/team/TeamMemberStoragePaths'; + export class MemberWorkSyncStorePaths { - constructor(private readonly teamsBasePath: string) {} + private readonly memberStorage: TeamMemberStoragePaths; + + constructor(private readonly teamsBasePath: string) { + this.memberStorage = new TeamMemberStoragePaths(teamsBasePath); + } + + getTeamRootDir(teamName: string): string { + return join(this.teamsBasePath, teamName); + } getTeamDir(teamName: string): string { return join(this.teamsBasePath, teamName, '.member-work-sync'); @@ -22,4 +32,64 @@ export class MemberWorkSyncStorePaths { getReportTokenSecretPath(teamName: string): string { return join(this.getTeamDir(teamName), 'report-token-secret.json'); } + + getIndexesDir(teamName: string): string { + return join(this.getTeamDir(teamName), 'indexes'); + } + + getMetricsIndexPath(teamName: string): string { + return join(this.getIndexesDir(teamName), 'metrics.json'); + } + + getOutboxIndexPath(teamName: string): string { + return join(this.getIndexesDir(teamName), 'outbox-index.json'); + } + + getPendingReportsIndexPath(teamName: string): string { + return join(this.getIndexesDir(teamName), 'pending-reports-index.json'); + } + + getLegacyStatusPath(teamName: string): string { + return this.getStatusPath(teamName); + } + + getLegacyPendingReportsPath(teamName: string): string { + return this.getPendingReportsPath(teamName); + } + + getLegacyOutboxPath(teamName: string): string { + return this.getOutboxPath(teamName); + } + + getMemberKey(memberName: string): string { + return this.memberStorage.getMemberKey(memberName); + } + + getMemberDir(teamName: string, memberName: string): string { + return this.memberStorage.getMemberDir(teamName, memberName); + } + + getMemberWorkSyncDir(teamName: string, memberName: string): string { + return this.memberStorage.getMemberFeatureDir(teamName, memberName, '.member-work-sync'); + } + + getMemberStatusPath(teamName: string, memberName: string): string { + return join(this.getMemberWorkSyncDir(teamName, memberName), 'status.json'); + } + + getMemberReportsPath(teamName: string, memberName: string): string { + return join(this.getMemberWorkSyncDir(teamName, memberName), 'reports.json'); + } + + getMemberOutboxPath(teamName: string, memberName: string): string { + return join(this.getMemberWorkSyncDir(teamName, memberName), 'outbox.json'); + } + + getMemberJournalPath(teamName: string, memberName: string): string { + return join(this.getMemberWorkSyncDir(teamName, memberName), 'journal.jsonl'); + } + + async ensureMemberWorkSyncDir(teamName: string, memberName: string): Promise { + await this.memberStorage.ensureMemberMeta(teamName, memberName); + } } diff --git a/src/main/services/infrastructure/EventLoopLagMonitor.ts b/src/main/services/infrastructure/EventLoopLagMonitor.ts index c47fe9b8..58022cde 100644 --- a/src/main/services/infrastructure/EventLoopLagMonitor.ts +++ b/src/main/services/infrastructure/EventLoopLagMonitor.ts @@ -4,14 +4,24 @@ import { createLogger } from '@shared/utils/logger'; const logger = createLogger('Perf:EventLoop'); +const DEFAULT_MAX_STALL_THRESHOLD_MS = 750; +const DEFAULT_REPORT_INTERVAL_MS = 30_000; + let started = false; let currentOp: string | null = null; +let lastReportAt = 0; + +function isEnabled(): boolean { + const raw = process.env.CLAUDE_TEAM_EVENT_LOOP_LAG_MONITOR_ENABLED?.trim().toLowerCase(); + return raw === '1' || raw === 'true' || raw === 'yes' || raw === 'on'; +} export function setCurrentMainOp(op: string | null): void { currentOp = op; } export function startEventLoopLagMonitor(): void { + if (!isEnabled()) return; if (started) return; started = true; @@ -24,14 +34,19 @@ export function startEventLoopLagMonitor(): void { // Reset first so next window is clean even if logging throws h.reset(); - // Only report meaningful stalls - if (maxMs < 250) return; + // Only report severe stalls. Sub-second blips are common during expected + // Electron/main-process IO and are too noisy for default development logs. + if (maxMs < DEFAULT_MAX_STALL_THRESHOLD_MS) return; // For known IPC/main-thread operations we already emit operation-specific // timing diagnostics. Suppress the generic event-loop warning to avoid // duplicate noisy logs that do not add new debugging value. if (currentOp) return; + const now = Date.now(); + if (now - lastReportAt < DEFAULT_REPORT_INTERVAL_MS) return; + lastReportAt = now; + logger.warn( `Event loop stall detected: p95=${p95Ms.toFixed(1)}ms max=${maxMs.toFixed(1)}ms` + (currentOp ? ` op=${currentOp}` : '') diff --git a/src/main/services/team/TeamBackupService.ts b/src/main/services/team/TeamBackupService.ts index fca34e30..8035e045 100644 --- a/src/main/services/team/TeamBackupService.ts +++ b/src/main/services/team/TeamBackupService.ts @@ -70,8 +70,9 @@ const TEAM_ROOT_FILES = [ // Subdirs under ~/.claude/teams/{teamName}/ const TEAM_SUBDIRS = ['inboxes', 'review-decisions']; -const TEAM_RECURSIVE_SUBDIRS = ['.opencode-runtime']; +const TEAM_RECURSIVE_SUBDIRS = ['.opencode-runtime', 'members']; const ATOMIC_WRITE_TEMP_FILE_PREFIX = '.tmp.'; +const FILE_LOCK_SUFFIX = '.lock'; const QUARANTINED_OPENCODE_LANE_INDEX_RE = /^lanes\.invalid\.\d+\.json$/; // Subdirs under getAppDataPath() (our own storage, not in ~/.claude/) const APP_DATA_SUBDIRS = ['attachments']; @@ -112,6 +113,9 @@ function shouldCollectRecursiveBackupFile(relPath: string): boolean { if (fileName.startsWith(ATOMIC_WRITE_TEMP_FILE_PREFIX)) { return false; } + if (fileName.endsWith(FILE_LOCK_SUFFIX)) { + return false; + } // Runtime quarantine files are diagnostic snapshots of invalid JSON. if (QUARANTINED_OPENCODE_LANE_INDEX_RE.test(fileName)) { return false; diff --git a/src/main/services/team/TeamLaunchStateEvaluator.ts b/src/main/services/team/TeamLaunchStateEvaluator.ts index 9131b462..3c18ea0f 100644 --- a/src/main/services/team/TeamLaunchStateEvaluator.ts +++ b/src/main/services/team/TeamLaunchStateEvaluator.ts @@ -480,6 +480,7 @@ function normalizePersistedMemberState( parsed.pendingPermissionRequestIds ), runtimePid: normalizeRuntimePid(parsed.runtimePid), + runtimeRunId: normalizeOptionalString(parsed.runtimeRunId), runtimeSessionId: normalizeOptionalString(parsed.runtimeSessionId), livenessKind, pidSource: normalizePidSource(parsed.pidSource), diff --git a/src/main/services/team/TeamMemberStoragePaths.ts b/src/main/services/team/TeamMemberStoragePaths.ts new file mode 100644 index 00000000..92fd8cef --- /dev/null +++ b/src/main/services/team/TeamMemberStoragePaths.ts @@ -0,0 +1,109 @@ +import { mkdir, readFile } from 'fs/promises'; +import { join } from 'path'; + +import { atomicWriteAsync } from '@main/utils/atomicWrite'; + +export interface TeamMemberStorageMetaFile { + schemaVersion: 1; + memberName: string; + memberKey: string; + updatedAt: string; +} + +export function normalizeTeamMemberStorageName(memberName: string): string { + return memberName.trim().toLowerCase(); +} + +export function encodeTeamMemberStorageKey(memberName: string): string { + const normalized = normalizeTeamMemberStorageName(memberName); + if (!normalized) { + throw new Error('memberName is required for member-scoped storage'); + } + const encoded = encodeURIComponent(normalized); + if (encoded === '.') { + return '%2E'; + } + if (encoded === '..') { + return '%2E%2E'; + } + return encoded; +} + +function isMetaFile(value: unknown): value is TeamMemberStorageMetaFile { + return ( + value != null && + typeof value === 'object' && + (value as TeamMemberStorageMetaFile).schemaVersion === 1 && + typeof (value as TeamMemberStorageMetaFile).memberName === 'string' && + typeof (value as TeamMemberStorageMetaFile).memberKey === 'string' && + typeof (value as TeamMemberStorageMetaFile).updatedAt === 'string' + ); +} + +export class TeamMemberStoragePaths { + constructor(private readonly teamsBasePath: string) {} + + getTeamDir(teamName: string): string { + return join(this.teamsBasePath, teamName); + } + + getMembersDir(teamName: string): string { + return join(this.getTeamDir(teamName), 'members'); + } + + getMemberKey(memberName: string): string { + return encodeTeamMemberStorageKey(memberName); + } + + getMemberDir(teamName: string, memberName: string): string { + return join(this.getMembersDir(teamName), this.getMemberKey(memberName)); + } + + getMemberMetaPath(teamName: string, memberName: string): string { + return join(this.getMemberDir(teamName, memberName), 'member.meta.json'); + } + + getMemberFeatureDir(teamName: string, memberName: string, featureDirName: string): string { + const featureDirSegment = featureDirName.trim(); + if ( + !featureDirSegment || + featureDirSegment === '.' || + featureDirSegment === '..' || + featureDirSegment.includes('/') || + featureDirSegment.includes('\\') + ) { + throw new Error('featureDirName must be a single path segment'); + } + return join(this.getMemberDir(teamName, memberName), featureDirSegment); + } + + async ensureMemberMeta(teamName: string, memberName: string): Promise { + const canonicalMemberName = memberName.trim(); + const memberKey = this.getMemberKey(canonicalMemberName); + const metaPath = this.getMemberMetaPath(teamName, canonicalMemberName); + const existing = await this.readMeta(metaPath); + if (existing?.memberName === canonicalMemberName && existing.memberKey === memberKey) { + return existing; + } + + const next: TeamMemberStorageMetaFile = { + schemaVersion: 1, + memberName: canonicalMemberName, + memberKey, + updatedAt: new Date().toISOString(), + }; + await mkdir(this.getMemberDir(teamName, canonicalMemberName), { recursive: true }); + await atomicWriteAsync(metaPath, `${JSON.stringify(next, null, 2)}\n`); + return next; + } + + private async readMeta(filePath: string): Promise { + try { + const raw = await readFile(filePath, 'utf8'); + const parsed = JSON.parse(raw); + return isMetaFile(parsed) ? parsed : null; + } catch { + return null; + } + } +} diff --git a/src/main/services/team/TeamProvisioningService.ts b/src/main/services/team/TeamProvisioningService.ts index 9d55b060..87861de5 100644 --- a/src/main/services/team/TeamProvisioningService.ts +++ b/src/main/services/team/TeamProvisioningService.ts @@ -167,6 +167,7 @@ import { } from './opencode/store/OpenCodeRuntimeManifestEvidenceReader'; import { createRuntimeRunTombstoneStore, + RuntimeStaleEvidenceError, type RuntimeEvidenceKind, } from './opencode/store/RuntimeRunTombstoneStore'; import { OpenCodeTaskLogAttributionStore } from './taskLogs/stream/OpenCodeTaskLogAttributionStore'; @@ -1411,6 +1412,11 @@ interface ProvisioningRun { effectiveMembers: TeamCreateRequest['members']; launchIdentity: ProviderModelLaunchIdentity | null; mixedSecondaryLanes: MixedSecondaryRuntimeLaneState[]; + /** + * OpenCode secondary lanes share bridge state files. Launch them sequentially + * per team run to avoid file-lock contention while keeping launch non-blocking. + */ + mixedSecondaryLaneLaunchQueue?: Promise; lastLogProgressAt: number; /** Monotonic ms timestamp of last stdout/stderr data. For stall detection. */ lastDataReceivedAt: number; @@ -1619,7 +1625,7 @@ interface PromptSizeSummary { lines: number; } -const MEMBER_LAUNCH_GRACE_MS = 90_000; +const MEMBER_LAUNCH_GRACE_MS = 150_000; const MEMBER_BOOTSTRAP_STALL_MS = 5 * 60_000; export function shouldWarnOnUnreadableMemberAuditConfig(params: { @@ -1809,6 +1815,18 @@ function isRecoverablePersistedOpenCodeRuntimeCandidate( ); } +function isPersistedOpenCodeSecondaryLaneMember( + member: PersistedTeamLaunchMemberState | undefined | null +): boolean { + return ( + member?.providerId === 'opencode' && + member.laneKind === 'secondary' && + member.laneOwnerProviderId === 'opencode' && + typeof member.laneId === 'string' && + member.laneId.trim().length > 0 + ); +} + function isRecoverablePersistedOpenCodeTerminalRuntimeCandidate( member: PersistedTeamLaunchMemberState | undefined | null ): boolean { @@ -6081,6 +6099,26 @@ export class TeamProvisioningService { runtimeActive = true; } } + if ( + runtimeActive && + laneIdentity.laneKind === 'secondary' && + laneIdentity.laneOwnerProviderId === 'opencode' && + !liveSecondaryLaneRunId + ) { + const staleLane = await recoverStaleOpenCodeRuntimeLaneIndexEntry({ + teamsBasePath: getTeamsBasePath(), + teamName, + laneId: laneIdentity.laneId, + }); + if (staleLane.stale) { + this.deleteSecondaryRuntimeRun(teamName, laneIdentity.laneId); + return { + delivered: false, + reason: 'opencode_runtime_not_active', + diagnostics: staleLane.diagnostics, + }; + } + } if (!runtimeActive) { this.cleanupStoppedTeamOpenCodeRuntimeLanesInBackground(teamName); return { delivered: false, reason: 'opencode_runtime_not_active' }; @@ -8193,6 +8231,38 @@ export class TeamProvisioningService { laneId, evidenceKind: 'bootstrap_checkin', }); + const idempotent = await this.resolveOpenCodeRuntimeBootstrapCheckinIdempotency({ + teamName, + runId, + memberName, + runtimeSessionId, + }); + await this.assertOpenCodeRuntimeMemberCheckinAllowed({ + teamName, + memberName, + previousMember: idempotent.previousMember, + }); + if (idempotent.state === 'duplicate') { + return { + ok: true, + providerId: 'opencode', + teamName, + runId, + state: 'accepted', + memberName, + runtimeSessionId, + diagnostics: ['opencode_bootstrap_checkin_duplicate_accepted'], + observedAt, + }; + } + if (idempotent.state === 'conflict') { + throw new RuntimeStaleEvidenceError( + `opencode_bootstrap_checkin_session_conflict: existing runtime session ${idempotent.existingRuntimeSessionId}, received ${runtimeSessionId} for ${memberName}`, + 'run_mismatch', + 'bootstrap_checkin', + runId + ); + } await this.updateOpenCodeRuntimeMemberLiveness({ teamName, runId, @@ -8217,6 +8287,95 @@ export class TeamProvisioningService { }; } + private async resolveOpenCodeRuntimeBootstrapCheckinIdempotency(input: { + teamName: string; + runId: string; + memberName: string; + runtimeSessionId: string; + }): Promise< + | { + state: 'new'; + previousMember?: PersistedTeamLaunchMemberState; + } + | { + state: 'duplicate'; + previousMember: PersistedTeamLaunchMemberState; + } + | { + state: 'conflict'; + previousMember: PersistedTeamLaunchMemberState; + existingRuntimeSessionId: string; + } + > { + const snapshot = await this.launchStateStore.read(input.teamName); + const previousMember = snapshot?.members[input.memberName]; + if (!previousMember) { + return { state: 'new' }; + } + + const existingRuntimeSessionId = previousMember.runtimeSessionId?.trim(); + const existingRuntimeRunId = + typeof previousMember.runtimeRunId === 'string' ? previousMember.runtimeRunId.trim() : ''; + const hasAcceptedBootstrap = + previousMember.bootstrapConfirmed === true || + previousMember.livenessKind === 'confirmed_bootstrap' || + previousMember.launchState === 'confirmed_alive'; + + if (!hasAcceptedBootstrap || !existingRuntimeSessionId) { + return { state: 'new', previousMember }; + } + + if (existingRuntimeRunId && existingRuntimeRunId !== input.runId) { + return { state: 'new', previousMember }; + } + + if (existingRuntimeSessionId === input.runtimeSessionId) { + return { state: 'duplicate', previousMember }; + } + + if (!existingRuntimeRunId) { + return { state: 'new', previousMember }; + } + + return { + state: 'conflict', + previousMember, + existingRuntimeSessionId, + }; + } + + private async assertOpenCodeRuntimeMemberCheckinAllowed(input: { + teamName: string; + memberName: string; + previousMember?: PersistedTeamLaunchMemberState; + }): Promise { + const config = await this.configReader.getConfig(input.teamName).catch(() => null); + const metaMembers = await this.membersMetaStore.getMembers(input.teamName).catch(() => []); + const configuredMember = this.resolveEffectiveConfiguredMember( + config?.members ?? [], + metaMembers, + input.memberName + ); + + if (configuredMember?.removedAt != null) { + throw new RuntimeStaleEvidenceError( + `Rejected OpenCode bootstrap check-in for removed member "${input.memberName}"`, + 'run_mismatch', + 'bootstrap_checkin', + null + ); + } + + if (!configuredMember && !input.previousMember) { + throw new RuntimeStaleEvidenceError( + `Rejected OpenCode bootstrap check-in for unconfigured member "${input.memberName}"`, + 'run_mismatch', + 'bootstrap_checkin', + null + ); + } + } + async deliverOpenCodeRuntimeMessage(raw: unknown): Promise { const payload = asRuntimeRecord(raw); const teamName = requireRuntimeString(payload.teamName, 'teamName'); @@ -8386,6 +8545,16 @@ export class TeamProvisioningService { .map((member) => (typeof member.name === 'string' ? member.name.trim() : '')) .filter((name) => name.length > 0 && name !== 'user' && !isLeadMember({ name })); const previousMember = previous?.members[input.memberName]; + const previousRuntimeRunId = + typeof previousMember?.runtimeRunId === 'string' ? previousMember.runtimeRunId.trim() : ''; + const sameRuntimeRun = previousRuntimeRunId.length > 0 && previousRuntimeRunId === input.runId; + const runtimePid = + input.metadata?.runtimePid ?? (sameRuntimeRun ? previousMember?.runtimePid : undefined); + const pidSource = input.metadata?.runtimePid + ? ('runtime_bootstrap' as const) + : sameRuntimeRun + ? previousMember?.pidSource + : undefined; const persistedIdentity = this.resolvePersistedRuntimeMemberIdentity({ teamName: input.teamName, memberName: input.memberName, @@ -8400,10 +8569,11 @@ export class TeamProvisioningService { runtimeAlive: true, bootstrapConfirmed: true, hardFailure: false, - ...(input.metadata?.runtimePid ? { runtimePid: input.metadata.runtimePid } : {}), + runtimePid, + runtimeRunId: input.runId, runtimeSessionId: input.runtimeSessionId, livenessKind: 'confirmed_bootstrap', - ...(input.metadata?.runtimePid ? { pidSource: 'runtime_bootstrap' as const } : {}), + pidSource, runtimeDiagnostic: input.reason, runtimeDiagnosticSeverity: 'info', runtimeLastSeenAt: input.observedAt, @@ -10854,6 +11024,7 @@ export class TeamProvisioningService { return; } await this.reconcileBootstrapTranscriptFailures(run); + await this.reconcileBootstrapTranscriptSuccesses(run); if (this.shouldSkipMemberSpawnAudit(run)) { return; } @@ -10899,9 +11070,17 @@ export class TeamProvisioningService { private async reconcileBootstrapTranscriptSuccesses(run: ProvisioningRun): Promise { for (const memberName of run.expectedMembers ?? []) { const current = run.memberSpawnStatuses.get(memberName); + if (this.isOpenCodeSecondaryLaneMemberInRun(run, memberName)) { + continue; + } + const failureReason = current?.hardFailureReason ?? current?.error; + const canClearFailedBootstrap = + current?.launchState === 'failed_to_start' && + current.agentToolAccepted === true && + isAutoClearableLaunchFailureReason(failureReason); if ( !current || - current.launchState === 'failed_to_start' || + (current.launchState === 'failed_to_start' && !canClearFailedBootstrap) || current.launchState === 'confirmed_alive' || current.bootstrapConfirmed === true || current.agentToolAccepted !== true @@ -10922,6 +11101,11 @@ export class TeamProvisioningService { } } + private isOpenCodeSecondaryLaneMemberInRun(run: ProvisioningRun, memberName: string): boolean { + const lanes = Array.isArray(run.mixedSecondaryLanes) ? run.mixedSecondaryLanes : []; + return lanes.some((lane) => lane.providerId === 'opencode' && lane.member.name === memberName); + } + private static readonly CONTEXT_EMIT_THROTTLE_MS = 2000; private static readonly LEAD_TEXT_EMIT_THROTTLE_MS = 2000; @@ -17848,8 +18032,13 @@ export class TeamProvisioningService { lane.state = 'launching'; lane.runId = lane.runId ?? randomUUID(); - void (async () => { + const launch = async () => { try { + if (run.cancelRequested || run.processKilled) { + this.deleteSecondaryRuntimeRun(run.teamName, lane.laneId); + lane.state = 'finished'; + return; + } await this.launchSingleMixedSecondaryLane(run, lane); } catch (error) { if (run.cancelRequested || run.processKilled) { @@ -17879,7 +18068,18 @@ export class TeamProvisioningService { await this.publishMixedSecondaryLaneStatusChange(run, lane).catch(() => undefined); lane.state = 'finished'; } - })(); + }; + + const previousLaunch = run.mixedSecondaryLaneLaunchQueue ?? Promise.resolve(); + const nextLaunch = previousLaunch.catch(() => undefined).then(launch); + run.mixedSecondaryLaneLaunchQueue = nextLaunch.catch((error) => { + logger.warn( + `[${run.teamName}] OpenCode secondary lane launch queue failed: ${ + error instanceof Error ? error.message : String(error) + }` + ); + }); + void run.mixedSecondaryLaneLaunchQueue; } private async launchMixedSecondaryLaneIfNeeded( @@ -18085,7 +18285,7 @@ export class TeamProvisioningService { projectPath, previousLaunchState: persistedSnapshot ?? bootstrapSnapshot, }); - if (runtimeEvidence) { + if (isRecoverableOpenCodeRuntimeEvidence(runtimeEvidence)) { recoveredAny = true; secondaryMembers.push({ laneId: laneIdentity.laneId, @@ -18340,7 +18540,10 @@ export class TeamProvisioningService { expected, Number.isFinite(acceptedAtMs) ? acceptedAtMs : null ); - if (transcriptOutcome) { + if ( + transcriptOutcome && + (transcriptOutcome.kind !== 'success' || !isPersistedOpenCodeSecondaryLaneMember(current)) + ) { return true; } } @@ -18481,12 +18684,17 @@ export class TeamProvisioningService { hardFailure: false, lastEvaluatedAt: now, }; + const isOpenCodeSecondaryLaneMember = isPersistedOpenCodeSecondaryLaneMember(current); if (bootstrapMember?.agentToolAccepted && !current.agentToolAccepted) { current.agentToolAccepted = true; current.firstSpawnAcceptedAt = current.firstSpawnAcceptedAt ?? bootstrapMember.firstSpawnAcceptedAt; } - if (bootstrapMember?.bootstrapConfirmed && !current.bootstrapConfirmed) { + if ( + bootstrapMember?.bootstrapConfirmed && + !current.bootstrapConfirmed && + !isOpenCodeSecondaryLaneMember + ) { current.bootstrapConfirmed = true; current.lastHeartbeatAt = current.lastHeartbeatAt ?? bootstrapMember.lastHeartbeatAt; } @@ -18546,7 +18754,7 @@ export class TeamProvisioningService { current.hardFailure = true; current.hardFailureReason = heartbeatReason; current.sources.hardFailureSignal = true; - } else if (heartbeatMessage) { + } else if (heartbeatMessage && !isOpenCodeSecondaryLaneMember) { current.bootstrapConfirmed = true; current.lastHeartbeatAt = heartbeatMessage.timestamp; current.hardFailure = false; @@ -18558,7 +18766,7 @@ export class TeamProvisioningService { expected, Number.isFinite(acceptedAtMs) ? acceptedAtMs : null ); - if (transcriptOutcome?.kind === 'success') { + if (transcriptOutcome?.kind === 'success' && !isOpenCodeSecondaryLaneMember) { current.bootstrapConfirmed = true; current.lastHeartbeatAt = current.lastHeartbeatAt ?? transcriptOutcome.observedAt; current.hardFailure = false; diff --git a/src/main/services/team/opencode/store/OpenCodeRuntimeManifestEvidenceReader.ts b/src/main/services/team/opencode/store/OpenCodeRuntimeManifestEvidenceReader.ts index 8939a180..a0f5a6ea 100644 --- a/src/main/services/team/opencode/store/OpenCodeRuntimeManifestEvidenceReader.ts +++ b/src/main/services/team/opencode/store/OpenCodeRuntimeManifestEvidenceReader.ts @@ -9,6 +9,7 @@ import { withFileLock } from '../../fileLock'; import { createDefaultRuntimeStoreManifest, createRuntimeStoreManifestStore, + OPENCODE_RUNTIME_STORE_DESCRIPTORS, OPENCODE_RUNTIME_STORE_MANIFEST_SCHEMA_VERSION, validateRuntimeStoreManifest, } from './RuntimeStoreManifest'; @@ -28,11 +29,19 @@ const OPENCODE_TEAM_RUNTIME_LANES_DIR = 'lanes'; const OPENCODE_TEAM_RUNTIME_LANES_INDEX_FILE = 'lanes.json'; const OPENCODE_RUNTIME_MANIFEST_FILE = 'manifest.json'; const OPENCODE_RUNTIME_RUN_TOMBSTONES_FILE = 'opencode-run-tombstones.json'; +const OPENCODE_ACTIVE_EMPTY_LANE_STALE_MS = 150_000; const OPENCODE_LANE_INDEX_LOCK_OPTIONS = { acquireTimeoutMs: 30_000, staleTimeoutMs: 25_000, retryIntervalMs: 25, } as const; +const OPENCODE_RUNTIME_EVIDENCE_FILES = new Set( + OPENCODE_RUNTIME_STORE_DESCRIPTORS.filter( + (descriptor) => + descriptor.schemaName !== 'opencode.promptDeliveryLedger' && + descriptor.schemaName !== 'opencode.deliveryJournal' + ).map((descriptor) => descriptor.relativePath) +); export interface OpenCodeRuntimeLaneIndexEntry { laneId: string; @@ -318,6 +327,9 @@ export async function inspectOpenCodeRuntimeLaneStorage(params: { }): Promise<{ laneDirectoryExists: boolean; hasStateOnDisk: boolean; + hasRuntimeEvidenceOnDisk: boolean; + manifestEntryCount: number | null; + manifestUpdatedAt: string | null; fileNames: string[]; }> { const laneDir = getOpenCodeTeamRuntimeLaneDirectory( @@ -330,14 +342,38 @@ export async function inspectOpenCodeRuntimeLaneStorage(params: { return { laneDirectoryExists: false, hasStateOnDisk: false, + hasRuntimeEvidenceOnDisk: false, + manifestEntryCount: null, + manifestUpdatedAt: null, fileNames: [], }; } const fileNames = (await readdir(laneDir).catch(() => [] as string[])).sort(); + const manifestPath = getOpenCodeRuntimeManifestPath( + params.teamsBasePath, + params.teamName, + params.laneId + ); + const manifest = (await fileExists(manifestPath)) + ? await readRuntimeStoreManifestEvidenceData( + manifestPath, + params.teamName, + () => new Date() + ).catch(() => null) + : null; + const hasRuntimeEvidenceFile = fileNames.some((fileName) => + OPENCODE_RUNTIME_EVIDENCE_FILES.has(fileName) + ); + const hasRuntimeEvidenceManifestEntry = + manifest?.entries.some((entry) => OPENCODE_RUNTIME_EVIDENCE_FILES.has(entry.relativePath)) ?? + false; return { laneDirectoryExists: true, hasStateOnDisk: fileNames.length > 0, + hasRuntimeEvidenceOnDisk: hasRuntimeEvidenceFile || hasRuntimeEvidenceManifestEntry, + manifestEntryCount: manifest ? manifest.entries.length : null, + manifestUpdatedAt: manifest?.updatedAt ?? null, fileNames, }; } @@ -513,6 +549,8 @@ export async function recoverStaleOpenCodeRuntimeLaneIndexEntry(params: { teamsBasePath: string; teamName: string; laneId: string; + clock?: () => Date; + emptyLaneStaleAfterMs?: number; }): Promise<{ stale: boolean; degraded: boolean; @@ -529,7 +567,7 @@ export async function recoverStaleOpenCodeRuntimeLaneIndexEntry(params: { } const storage = await inspectOpenCodeRuntimeLaneStorage(params); - if (storage.hasStateOnDisk) { + if (storage.hasRuntimeEvidenceOnDisk) { return { stale: false, degraded: false, @@ -537,9 +575,26 @@ export async function recoverStaleOpenCodeRuntimeLaneIndexEntry(params: { }; } - const diagnostics = [ - `OpenCode lane ${params.laneId} is marked active in lanes.json, but no lane state exists on disk.`, - ]; + const now = params.clock?.() ?? new Date(); + const staleAfterMs = params.emptyLaneStaleAfterMs ?? OPENCODE_ACTIVE_EMPTY_LANE_STALE_MS; + const lastTouchedAt = + Date.parse(storage.manifestUpdatedAt ?? '') || Date.parse(entry.updatedAt) || NaN; + const laneAgeMs = Number.isFinite(lastTouchedAt) ? now.getTime() - lastTouchedAt : Infinity; + if (storage.hasStateOnDisk && laneAgeMs < staleAfterMs) { + return { + stale: false, + degraded: false, + diagnostics: [], + }; + } + + const diagnostics = storage.hasStateOnDisk + ? [ + `OpenCode lane ${params.laneId} is marked active in lanes.json, but its runtime manifest has no committed runtime evidence after launch grace.`, + ] + : [ + `OpenCode lane ${params.laneId} is marked active in lanes.json, but no lane state exists on disk.`, + ]; await upsertOpenCodeRuntimeLaneIndexEntry({ teamsBasePath: params.teamsBasePath, teamName: params.teamName, diff --git a/src/main/services/team/runtime/OpenCodeTeamRuntimeAdapter.ts b/src/main/services/team/runtime/OpenCodeTeamRuntimeAdapter.ts index d1d7f1ad..1df7928c 100644 --- a/src/main/services/team/runtime/OpenCodeTeamRuntimeAdapter.ts +++ b/src/main/services/team/runtime/OpenCodeTeamRuntimeAdapter.ts @@ -429,6 +429,7 @@ function mapOpenCodeLaunchDataToRuntimeResult( data: OpenCodeLaunchTeamCommandData, prepareWarnings: string[] ): TeamRuntimeLaunchResult { + const bridgeDiagnostics = data.diagnostics.map(formatOpenCodeBridgeDiagnostic); const checkpointNames = extractCheckpointNames(data); const readyCheckpointsPresent = [...REQUIRED_READY_CHECKPOINTS].every((name) => checkpointNames.has(name) @@ -491,6 +492,7 @@ function mapOpenCodeLaunchDataToRuntimeResult( ...(bridgeMember?.evidence ?? []).map( (evidence) => `${evidence.kind} at ${evidence.observedAt}` ), + ...bridgeDiagnostics, ...checkpointDiagnostic, ...(missingExpectedMembers.includes(member.name) ? incompleteReadyDiagnostic : []), ] @@ -518,11 +520,7 @@ function mapOpenCodeLaunchDataToRuntimeResult( : 'partial_failure', members, warnings: [...prepareWarnings, ...data.warnings.map((warning) => warning.message)], - diagnostics: [ - ...data.diagnostics.map(formatOpenCodeBridgeDiagnostic), - ...checkpointDiagnostic, - ...incompleteReadyDiagnostic, - ], + diagnostics: [...bridgeDiagnostics, ...checkpointDiagnostic, ...incompleteReadyDiagnostic], }; } @@ -539,29 +537,28 @@ function mapBridgeMemberToRuntimeEvidence( const failed = launchState === 'failed'; const hasRuntimePid = typeof runtimePid === 'number' && Number.isFinite(runtimePid) && runtimePid > 0; - const pendingRuntimeObserved = launchState === 'created' && hasRuntimePid; + const hasSessionId = typeof sessionId === 'string' && sessionId.trim().length > 0; + const hasRuntimeHandle = hasRuntimePid || hasSessionId; + const pendingRuntimeObserved = launchState === 'created' && hasRuntimeHandle; const livenessKind = confirmed ? 'confirmed_bootstrap' : pendingRuntimeObserved ? 'runtime_process_candidate' : launchState === 'permission_blocked' ? 'permission_blocked' - : runtimeMaterialized || sessionId - ? 'runtime_process_candidate' - : 'registered_only'; + : 'registered_only'; const runtimeDiagnostic = pendingRuntimeObserved - ? 'OpenCode runtime pid reported by bridge without local process verification' + ? hasRuntimePid + ? 'OpenCode runtime pid reported by bridge without local process verification' + : 'OpenCode session exists without verified runtime pid' : launchState === 'permission_blocked' ? 'OpenCode runtime is waiting for permission approval' - : runtimeMaterialized || sessionId - ? 'OpenCode session exists without verified runtime pid' + : runtimeMaterialized + ? 'OpenCode bridge did not report a runtime session or pid for this member' : undefined; const runtimeDiagnosticSeverity = failed ? 'error' - : pendingRuntimeObserved || - launchState === 'permission_blocked' || - runtimeMaterialized || - sessionId + : pendingRuntimeObserved || launchState === 'permission_blocked' || runtimeMaterialized ? 'warning' : undefined; return { @@ -578,8 +575,7 @@ function mapBridgeMemberToRuntimeEvidence( confirmed || pendingRuntimeObserved || launchState === 'permission_blocked' || - runtimeMaterialized || - Boolean(sessionId), + hasRuntimeHandle, runtimeAlive: confirmed, bootstrapConfirmed: confirmed, hardFailure: failed, diff --git a/src/renderer/components/team/CliLogsRichView.tsx b/src/renderer/components/team/CliLogsRichView.tsx index a72e5fb3..63480f94 100644 --- a/src/renderer/components/team/CliLogsRichView.tsx +++ b/src/renderer/components/team/CliLogsRichView.tsx @@ -376,6 +376,10 @@ export const CliLogsRichView = ({ const groups = useMemo(() => parseStreamJsonToGroups(cliLogsTail), [cliLogsTail]); const entries = useMemo(() => groupBySubagent(groups), [groups]); + const emptyMessage = + cliLogsTail.trim().length > 0 + ? 'No displayable assistant/runtime logs yet.' + : 'Waiting for response...'; // Derive expanded state: all groups expanded unless manually collapsed const expandedGroupIds = useMemo(() => { @@ -578,9 +582,7 @@ export const CliLogsRichView = ({ - - Waiting for response... - + {emptyMessage} {footer} diff --git a/src/renderer/components/team/members/MemberDetailDialog.tsx b/src/renderer/components/team/members/MemberDetailDialog.tsx index eed114d4..7e847467 100644 --- a/src/renderer/components/team/members/MemberDetailDialog.tsx +++ b/src/renderer/components/team/members/MemberDetailDialog.tsx @@ -1,6 +1,6 @@ import { useEffect, useMemo, useState } from 'react'; -import { MemberWorkSyncStatusPanel } from '@features/member-work-sync/renderer'; +// import { MemberWorkSyncStatusPanel } from '@features/member-work-sync/renderer'; import { Button } from '@renderer/components/ui/button'; import { Dialog, DialogContent, DialogFooter, DialogHeader } from '@renderer/components/ui/dialog'; import { Tabs, TabsContent, TabsList, TabsTrigger } from '@renderer/components/ui/tabs'; @@ -293,11 +293,13 @@ export const MemberDetailDialog = ({
+ {/* + */}
diff --git a/src/shared/types/team.ts b/src/shared/types/team.ts index 5ecafbf4..04d70a7a 100644 --- a/src/shared/types/team.ts +++ b/src/shared/types/team.ts @@ -1002,6 +1002,8 @@ export interface PersistedTeamLaunchMemberState { hardFailureReason?: string; pendingPermissionRequestIds?: string[]; runtimePid?: number; + /** OpenCode runtime run id that produced the current runtimeSessionId/liveness evidence. */ + runtimeRunId?: string; runtimeSessionId?: string; livenessKind?: TeamAgentRuntimeLivenessKind; pidSource?: TeamAgentRuntimePidSource; diff --git a/test/features/member-work-sync/core/MemberWorkSyncUseCases.test.ts b/test/features/member-work-sync/core/MemberWorkSyncUseCases.test.ts index 04bd3892..ec9f7b99 100644 --- a/test/features/member-work-sync/core/MemberWorkSyncUseCases.test.ts +++ b/test/features/member-work-sync/core/MemberWorkSyncUseCases.test.ts @@ -7,6 +7,7 @@ import { MemberWorkSyncReconciler, MemberWorkSyncReporter, type MemberWorkSyncAgendaSourceResult, + type MemberWorkSyncAuditEvent, type MemberWorkSyncInboxNudgePort, type MemberWorkSyncOutboxStorePort, type MemberWorkSyncStatusStorePort, @@ -247,6 +248,7 @@ function createDeps(options?: { }) { const clock = new MutableClock(); const store = new InMemoryStatusStore(); + const auditEvents: MemberWorkSyncAuditEvent[] = []; const source: MemberWorkSyncAgendaSourceResult = { agenda: { teamName: 'team-a', @@ -286,13 +288,18 @@ function createDeps(options?: { lifecycle: { isTeamActive: () => options?.teamActive ?? true, }, + auditJournal: { + append: async (event) => { + auditEvents.push(event); + }, + }, }; - return { clock, deps, source, store }; + return { auditEvents, clock, deps, source, store }; } describe('MemberWorkSync use cases', () => { it('reconciles actionable work into needs_sync without side effects', async () => { - const { deps, store } = createDeps(); + const { auditEvents, deps, store } = createDeps(); const status = await new MemberWorkSyncDiagnosticsReader(deps).execute({ teamName: 'team-a', memberName: 'bob', @@ -308,10 +315,15 @@ describe('MemberWorkSync use cases', () => { fingerprintChanged: false, }); expect(store.pendingReports).toEqual([]); + expect(auditEvents.map((event) => event.event)).toEqual([ + 'reconcile_started', + 'agenda_loaded', + 'decision_made', + ]); }); it('accepts still_working as a bounded lease for the current fingerprint', async () => { - const { clock, deps } = createDeps(); + const { auditEvents, clock, deps } = createDeps(); const reader = new MemberWorkSyncDiagnosticsReader(deps); const reporter = new MemberWorkSyncReporter(deps); const current = await reader.execute({ teamName: 'team-a', memberName: 'bob' }); @@ -340,6 +352,7 @@ describe('MemberWorkSync use cases', () => { const expired = await reader.execute({ teamName: 'team-a', memberName: 'bob' }); expect(expired.state).toBe('needs_sync'); expect(expired.diagnostics).toContain('report_lease_expired'); + expect(auditEvents.map((event) => event.event)).toContain('report_accepted'); }); it('uses app clock instead of model supplied reportedAt for lease timing', async () => { @@ -365,7 +378,7 @@ describe('MemberWorkSync use cases', () => { }); it('rejects stale reports without turning app-side validation failures into pending intents', async () => { - const { deps, store } = createDeps(); + const { auditEvents, deps, store } = createDeps(); const result = await new MemberWorkSyncReporter(deps).execute({ teamName: 'team-a', memberName: 'bob', @@ -384,6 +397,14 @@ describe('MemberWorkSync use cases', () => { }); expect(store.writes.at(-1)?.diagnostics).toContain('report_rejected:stale_fingerprint'); expect(store.pendingReports).toHaveLength(0); + expect(auditEvents).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + event: 'report_rejected', + reason: 'stale_fingerprint', + }), + ]) + ); }); it('accepts caught_up only when the app-side agenda is empty', async () => { @@ -619,7 +640,7 @@ describe('MemberWorkSync use cases', () => { it('defers nudge dispatch while the member has active or recent tool activity', async () => { const outbox = new InMemoryOutboxStore(); const inbox = new InMemoryInboxNudge(); - const { deps, store } = createDeps({ + const { auditEvents, deps, store } = createDeps({ outboxStore: outbox, inboxNudge: inbox, busySignal: { @@ -651,6 +672,14 @@ describe('MemberWorkSync use cases', () => { lastError: 'member_busy:active_tool_activity', nextAttemptAt: '2026-04-29T00:02:00.000Z', }); + expect(auditEvents).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + event: 'member_busy', + reason: 'member_busy:active_tool_activity', + }), + ]) + ); }); it('uses bounded retry backoff when inbox delivery fails', async () => { diff --git a/test/features/member-work-sync/main/FileMemberWorkSyncAuditJournal.test.ts b/test/features/member-work-sync/main/FileMemberWorkSyncAuditJournal.test.ts new file mode 100644 index 00000000..3f1c0611 --- /dev/null +++ b/test/features/member-work-sync/main/FileMemberWorkSyncAuditJournal.test.ts @@ -0,0 +1,113 @@ +import { mkdtemp, readFile, readdir, rm } from 'fs/promises'; +import { tmpdir } from 'os'; +import { join } from 'path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { FileMemberWorkSyncAuditJournal } from '@features/member-work-sync/main/infrastructure/FileMemberWorkSyncAuditJournal'; +import { MemberWorkSyncStorePaths } from '@features/member-work-sync/main/infrastructure/MemberWorkSyncStorePaths'; + +function journalPath(root: string): string { + return join(root, 'team-a', 'members', 'bob', '.member-work-sync', 'journal.jsonl'); +} + +describe('FileMemberWorkSyncAuditJournal', () => { + let root: string; + + beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'member-work-sync-audit-')); + }); + + afterEach(async () => { + await rm(root, { recursive: true, force: true }); + }); + + it('appends per-member JSONL audit events in order', async () => { + const journal = new FileMemberWorkSyncAuditJournal(new MemberWorkSyncStorePaths(root)); + + await journal.append({ + timestamp: '2026-04-30T00:00:00.000Z', + teamName: 'team-a', + memberName: 'bob', + event: 'reconcile_started', + source: 'test', + }); + await journal.append({ + timestamp: '2026-04-30T00:00:01.000Z', + teamName: 'team-a', + memberName: 'bob', + event: 'status_written', + source: 'test', + agendaFingerprint: 'agenda:v1:abc', + actionableCount: 1, + }); + + const lines = (await readFile(journalPath(root), 'utf8')).trim().split('\n'); + expect(lines.map((line) => JSON.parse(line).event)).toEqual([ + 'reconcile_started', + 'status_written', + ]); + expect(JSON.parse(lines[1])).toMatchObject({ + schemaVersion: 1, + teamName: 'team-a', + memberName: 'bob', + agendaFingerprint: 'agenda:v1:abc', + }); + }); + + it('truncates previews and rotates bounded journals', async () => { + const journal = new FileMemberWorkSyncAuditJournal( + new MemberWorkSyncStorePaths(root), + undefined, + { maxBytes: 200, rotatedFileCount: 2 } + ); + + for (let index = 0; index < 8; index += 1) { + await journal.append({ + timestamp: `2026-04-30T00:00:0${index}.000Z`, + teamName: 'team-a', + memberName: 'bob', + event: 'nudge_skipped', + source: 'test', + reason: 'r'.repeat(500), + diagnostics: ['d'.repeat(500)], + metadata: { long: 'm'.repeat(500), ['__proto__']: 'safe' }, + taskRefs: [{ taskId: 't'.repeat(500), displayId: 'x'.repeat(500) }], + messagePreview: 'x'.repeat(500), + }); + } + + const dirEntries = await readdir(join(root, 'team-a', 'members', 'bob', '.member-work-sync')); + expect(dirEntries).toEqual(expect.arrayContaining(['journal.jsonl', 'journal.jsonl.1'])); + expect(dirEntries).not.toContain('journal.jsonl.3'); + + const latestLine = (await readFile(journalPath(root), 'utf8')).trim().split('\n').at(-1); + const latest = JSON.parse(latestLine ?? '{}'); + expect(latest.messagePreview).toHaveLength(243); + expect(latest.reason).toHaveLength(243); + expect(latest.diagnostics[0]).toHaveLength(243); + expect(latest.metadata.long).toHaveLength(243); + expect(latest.metadata.__proto__).toBe('safe'); + expect(latest.taskRefs[0].taskId).toHaveLength(243); + }); + + it('logs and swallows append failures', async () => { + const logger = { debug: vi.fn(), warn: vi.fn(), error: vi.fn() }; + const paths = new MemberWorkSyncStorePaths(root); + vi.spyOn(paths, 'ensureMemberWorkSyncDir').mockRejectedValue(new Error('boom')); + const journal = new FileMemberWorkSyncAuditJournal(paths, logger); + + await expect( + journal.append({ + timestamp: '2026-04-30T00:00:00.000Z', + teamName: 'team-a', + memberName: 'bob', + event: 'reconcile_started', + source: 'test', + }) + ).resolves.toBeUndefined(); + expect(logger.warn).toHaveBeenCalledWith( + 'member work sync audit journal append failed', + expect.objectContaining({ error: 'Error: boom' }) + ); + }); +}); diff --git a/test/features/member-work-sync/main/JsonMemberWorkSyncStore.test.ts b/test/features/member-work-sync/main/JsonMemberWorkSyncStore.test.ts index 1f90fde5..7062c135 100644 --- a/test/features/member-work-sync/main/JsonMemberWorkSyncStore.test.ts +++ b/test/features/member-work-sync/main/JsonMemberWorkSyncStore.test.ts @@ -9,6 +9,7 @@ import type { MemberWorkSyncNudgePayload, MemberWorkSyncStatus, } from '@features/member-work-sync/contracts'; +import type { MemberWorkSyncAuditEvent } from '@features/member-work-sync/core/application'; function makeStatus(overrides: Partial): MemberWorkSyncStatus { return { @@ -58,6 +59,16 @@ function makeNudgePayload(overrides: Partial = {}): }; } +function memberWorkSyncDir(root: string, teamName: string, memberName: string): string { + return join( + root, + teamName, + 'members', + encodeURIComponent(memberName.trim().toLowerCase()), + '.member-work-sync' + ); +} + describe('JsonMemberWorkSyncStore', () => { let root: string; let store: JsonMemberWorkSyncStore; @@ -83,6 +94,56 @@ describe('JsonMemberWorkSyncStore', () => { expect(entries.some((entry) => entry.startsWith('status.json.invalid.'))).toBe(true); }); + it('writes status into member-scoped storage and keeps team metrics in an index', async () => { + await store.write(makeStatus({ providerId: 'opencode' })); + + const statusFile = JSON.parse( + await readFile(join(memberWorkSyncDir(root, 'team-a', 'bob'), 'status.json'), 'utf8') + ); + expect(statusFile).toMatchObject({ + schemaVersion: 2, + status: { + teamName: 'team-a', + memberName: 'bob', + providerId: 'opencode', + }, + }); + + const metaFile = JSON.parse( + await readFile(join(root, 'team-a', 'members', 'bob', 'member.meta.json'), 'utf8') + ); + expect(metaFile).toMatchObject({ + schemaVersion: 1, + memberName: 'bob', + memberKey: 'bob', + }); + + const metricsIndex = JSON.parse( + await readFile(join(root, 'team-a', '.member-work-sync', 'indexes', 'metrics.json'), 'utf8') + ); + expect(metricsIndex.members.bob).toMatchObject({ + memberName: 'bob', + state: 'needs_sync', + actionableCount: 1, + }); + }); + + it('prefers member-scoped v2 status over legacy v1 status', async () => { + await store.write(makeStatus({ state: 'caught_up', agenda: { ...makeStatus({}).agenda, items: [] } })); + + const legacyStatusPath = join(root, 'team-a', '.member-work-sync', 'status.json'); + await mkdir(join(root, 'team-a', '.member-work-sync'), { recursive: true }); + await writeFile( + legacyStatusPath, + JSON.stringify({ schemaVersion: 1, members: { bob: makeStatus({ state: 'needs_sync' }) } }), + 'utf8' + ); + + await expect(store.read({ teamName: 'team-a', memberName: 'bob' })).resolves.toMatchObject({ + state: 'caught_up', + }); + }); + it('deduplicates pending report intents and marks them processed', async () => { const request = { teamName: 'team-a', @@ -114,12 +175,129 @@ describe('JsonMemberWorkSyncStore', () => { expect(await store.listPendingReports('team-a')).toEqual([]); const file = JSON.parse( - await readFile(join(root, 'team-a', '.member-work-sync', 'pending-reports.json'), 'utf8') + await readFile(join(memberWorkSyncDir(root, 'team-a', 'bob'), 'reports.json'), 'utf8') ); expect(file.intents[pending[0].id]).toMatchObject({ status: 'accepted', resultCode: 'accepted', }); + const index = JSON.parse( + await readFile( + join(root, 'team-a', '.member-work-sync', 'indexes', 'pending-reports-index.json'), + 'utf8' + ) + ); + expect(index.items[pending[0].id]).toMatchObject({ + memberName: 'bob', + status: 'accepted', + }); + }); + + it('repairs a missing pending-report index from member-scoped report files', async () => { + const request = { + teamName: 'team-a', + memberName: 'bob', + state: 'still_working' as const, + agendaFingerprint: 'agenda:v1:abc', + reportToken: 'wrs:v1.test', + source: 'mcp' as const, + }; + + await store.appendPendingReport(request, 'control_api_unavailable'); + await rm(join(root, 'team-a', '.member-work-sync', 'indexes', 'pending-reports-index.json'), { + force: true, + }); + + await expect(store.listPendingReports('team-a')).resolves.toHaveLength(1); + const repaired = JSON.parse( + await readFile( + join(root, 'team-a', '.member-work-sync', 'indexes', 'pending-reports-index.json'), + 'utf8' + ) + ); + expect(Object.values(repaired.items)).toEqual([ + expect.objectContaining({ memberName: 'bob', status: 'pending' }), + ]); + }); + + it('repairs a stale pending-report index route from member-scoped report files', async () => { + const bobRequest = { + teamName: 'team-a', + memberName: 'bob', + state: 'still_working' as const, + agendaFingerprint: 'agenda:v1:bob', + reportToken: 'wrs:v1.bob', + source: 'mcp' as const, + }; + const tomRequest = { + ...bobRequest, + memberName: 'tom', + agendaFingerprint: 'agenda:v1:tom', + reportToken: 'wrs:v1.tom', + }; + + await store.appendPendingReport(bobRequest, 'control_api_unavailable'); + await store.appendPendingReport(tomRequest, 'control_api_unavailable'); + await writeFile( + join(root, 'team-a', 'members', 'bob', '.member-work-sync', 'reports.json'), + JSON.stringify({ schemaVersion: 2, intents: {} }), + 'utf8' + ); + + const pending = await store.listPendingReports('team-a'); + expect(pending.map((intent) => intent.memberName)).toEqual(['tom']); + const repaired = JSON.parse( + await readFile( + join(root, 'team-a', '.member-work-sync', 'indexes', 'pending-reports-index.json'), + 'utf8' + ) + ); + expect(Object.values(repaired.items).map((item) => (item as { memberName: string }).memberName)).toEqual([ + 'tom', + ]); + }); + + it('repairs a partially missing pending-report index route from member-scoped report files', async () => { + const bobRequest = { + teamName: 'team-a', + memberName: 'bob', + state: 'still_working' as const, + agendaFingerprint: 'agenda:v1:bob', + reportToken: 'wrs:v1.bob', + source: 'mcp' as const, + }; + const tomRequest = { + ...bobRequest, + memberName: 'tom', + agendaFingerprint: 'agenda:v1:tom', + reportToken: 'wrs:v1.tom', + }; + + await store.appendPendingReport(bobRequest, 'control_api_unavailable'); + await store.appendPendingReport(tomRequest, 'control_api_unavailable'); + const indexPath = join( + root, + 'team-a', + '.member-work-sync', + 'indexes', + 'pending-reports-index.json' + ); + const index = JSON.parse(await readFile(indexPath, 'utf8')); + for (const [id, route] of Object.entries(index.items)) { + if ((route as { memberName: string }).memberName === 'tom') { + delete index.items[id]; + } + } + await writeFile(indexPath, JSON.stringify(index), 'utf8'); + + const pending = await store.listPendingReports('team-a'); + expect(pending.map((intent) => intent.memberName).sort()).toEqual(['bob', 'tom']); + const repaired = JSON.parse(await readFile(indexPath, 'utf8')); + expect( + Object.values(repaired.items) + .map((item) => (item as { memberName: string }).memberName) + .sort() + ).toEqual(['bob', 'tom']); }); it('records bounded shadow metrics from status writes', async () => { @@ -263,7 +441,7 @@ describe('JsonMemberWorkSyncStore', () => { teamName: 'team-a', claimedBy: 'dispatcher-a', nowIso: '2026-04-29T00:01:00.000Z', - limit: 5, + limit: 1, }); expect(claimed).toHaveLength(1); expect(claimed[0]).toMatchObject({ @@ -305,12 +483,287 @@ describe('JsonMemberWorkSyncStore', () => { }); const file = JSON.parse( - await readFile(join(root, 'team-a', '.member-work-sync', 'outbox.json'), 'utf8') + await readFile(join(memberWorkSyncDir(root, 'team-a', 'bob'), 'outbox.json'), 'utf8') ); expect(file.items[input.id]).toMatchObject({ status: 'delivered', deliveredMessageId: 'message-1', attemptGeneration: 2, }); + const index = JSON.parse( + await readFile(join(root, 'team-a', '.member-work-sync', 'indexes', 'outbox-index.json'), 'utf8') + ); + expect(index.items[input.id]).toMatchObject({ + memberName: 'bob', + status: 'delivered', + }); + }); + + it('claims due outbox items from the index without scanning unrelated member outboxes', async () => { + const bobInput = { + id: 'member-work-sync:team-a:bob:agenda:v1:abc', + teamName: 'team-a', + memberName: 'bob', + agendaFingerprint: 'agenda:v1:abc', + payloadHash: 'hash-a', + payload: makeNudgePayload(), + nowIso: '2026-04-29T00:00:00.000Z', + }; + await store.ensurePending(bobInput); + + await mkdir(join(root, 'team-a', 'members', 'tom', '.member-work-sync'), { recursive: true }); + await writeFile( + join(root, 'team-a', 'members', 'tom', 'member.meta.json'), + JSON.stringify({ + schemaVersion: 1, + memberName: 'tom', + memberKey: 'tom', + updatedAt: '2026-04-29T00:00:00.000Z', + }), + 'utf8' + ); + await writeFile( + join(root, 'team-a', 'members', 'tom', '.member-work-sync', 'outbox.json'), + JSON.stringify({ + schemaVersion: 2, + items: { + 'member-work-sync:team-a:tom:agenda:v1:other': { + ...bobInput, + id: 'member-work-sync:team-a:tom:agenda:v1:other', + memberName: 'tom', + status: 'pending', + attemptGeneration: 0, + createdAt: '2026-04-29T00:00:00.000Z', + updatedAt: '2026-04-29T00:00:00.000Z', + }, + }, + }), + 'utf8' + ); + + const claimed = await store.claimDue({ + teamName: 'team-a', + claimedBy: 'dispatcher-a', + nowIso: '2026-04-29T00:01:00.000Z', + limit: 1, + }); + expect(claimed.map((item) => item.memberName)).toEqual(['bob']); + }); + + it('repairs a missing outbox index from member-scoped outbox files for delivered counts', async () => { + const input = { + id: 'member-work-sync:team-a:bob:agenda:v1:abc', + teamName: 'team-a', + memberName: 'bob', + agendaFingerprint: 'agenda:v1:abc', + payloadHash: 'hash-a', + payload: makeNudgePayload(), + nowIso: '2026-04-29T00:00:00.000Z', + }; + await store.ensurePending(input); + const [claimed] = await store.claimDue({ + teamName: 'team-a', + claimedBy: 'dispatcher-a', + nowIso: '2026-04-29T00:01:00.000Z', + limit: 1, + }); + await store.markDelivered({ + teamName: 'team-a', + id: input.id, + attemptGeneration: claimed.attemptGeneration, + deliveredMessageId: 'message-1', + nowIso: '2026-04-29T00:02:00.000Z', + }); + await rm(join(root, 'team-a', '.member-work-sync', 'indexes', 'outbox-index.json'), { + force: true, + }); + + await expect( + store.countRecentDelivered({ + teamName: 'team-a', + memberName: 'bob', + sinceIso: '2026-04-29T00:00:00.000Z', + }) + ).resolves.toBe(1); + }); + + it('counts delivered nudges from the member outbox when the outbox index is partially stale', async () => { + const bobInput = { + id: 'member-work-sync:team-a:bob:agenda:v1:abc', + teamName: 'team-a', + memberName: 'bob', + agendaFingerprint: 'agenda:v1:abc', + payloadHash: 'hash-a', + payload: makeNudgePayload(), + nowIso: '2026-04-29T00:00:00.000Z', + }; + const tomInput = { + ...bobInput, + id: 'member-work-sync:team-a:tom:agenda:v1:def', + memberName: 'tom', + payload: makeNudgePayload({ to: 'tom' }), + }; + await store.ensurePending(bobInput); + await store.ensurePending(tomInput); + const [claimedBob] = await store.claimDue({ + teamName: 'team-a', + claimedBy: 'dispatcher-a', + nowIso: '2026-04-29T00:01:00.000Z', + limit: 1, + }); + await store.markDelivered({ + teamName: 'team-a', + id: bobInput.id, + attemptGeneration: claimedBob.attemptGeneration, + deliveredMessageId: 'message-1', + nowIso: '2026-04-29T00:02:00.000Z', + }); + + const indexPath = join(root, 'team-a', '.member-work-sync', 'indexes', 'outbox-index.json'); + const index = JSON.parse(await readFile(indexPath, 'utf8')); + delete index.items[bobInput.id]; + await writeFile(indexPath, JSON.stringify(index), 'utf8'); + + await expect( + store.countRecentDelivered({ + teamName: 'team-a', + memberName: 'bob', + sinceIso: '2026-04-29T00:00:00.000Z', + }) + ).resolves.toBe(1); + const repaired = JSON.parse(await readFile(indexPath, 'utf8')); + expect(repaired.items[bobInput.id]).toMatchObject({ memberName: 'bob', status: 'delivered' }); + }); + + it('repairs stale due outbox index routes before persisting claim results', async () => { + const bobInput = { + id: 'member-work-sync:team-a:bob:agenda:v1:abc', + teamName: 'team-a', + memberName: 'bob', + agendaFingerprint: 'agenda:v1:abc', + payloadHash: 'hash-a', + payload: makeNudgePayload(), + nowIso: '2026-04-29T00:00:00.000Z', + }; + const tomInput = { + ...bobInput, + id: 'member-work-sync:team-a:tom:agenda:v1:def', + memberName: 'tom', + payload: makeNudgePayload({ to: 'tom' }), + }; + await store.ensurePending(bobInput); + await store.ensurePending(tomInput); + await writeFile( + join(root, 'team-a', 'members', 'bob', '.member-work-sync', 'outbox.json'), + JSON.stringify({ schemaVersion: 2, items: {} }), + 'utf8' + ); + + const claimed = await store.claimDue({ + teamName: 'team-a', + claimedBy: 'dispatcher-a', + nowIso: '2026-04-29T00:01:00.000Z', + limit: 5, + }); + expect(claimed.map((item) => item.memberName)).toEqual(['tom']); + const repaired = JSON.parse( + await readFile(join(root, 'team-a', '.member-work-sync', 'indexes', 'outbox-index.json'), 'utf8') + ); + expect(Object.values(repaired.items).map((item) => (item as { memberName: string }).memberName)).toEqual([ + 'tom', + ]); + }); + + it('repairs partially missing due outbox index routes before claiming', async () => { + const bobInput = { + id: 'member-work-sync:team-a:bob:agenda:v1:abc', + teamName: 'team-a', + memberName: 'bob', + agendaFingerprint: 'agenda:v1:abc', + payloadHash: 'hash-a', + payload: makeNudgePayload(), + nowIso: '2026-04-29T00:00:00.000Z', + }; + const tomInput = { + ...bobInput, + id: 'member-work-sync:team-a:tom:agenda:v1:def', + memberName: 'tom', + payload: makeNudgePayload({ to: 'tom' }), + }; + await store.ensurePending(bobInput); + await store.ensurePending(tomInput); + const indexPath = join(root, 'team-a', '.member-work-sync', 'indexes', 'outbox-index.json'); + const index = JSON.parse(await readFile(indexPath, 'utf8')); + delete index.items[tomInput.id]; + await writeFile(indexPath, JSON.stringify(index), 'utf8'); + + const claimed = await store.claimDue({ + teamName: 'team-a', + claimedBy: 'dispatcher-a', + nowIso: '2026-04-29T00:01:00.000Z', + limit: 5, + }); + expect(claimed.map((item) => item.memberName).sort()).toEqual(['bob', 'tom']); + }); + + it('falls back to legacy v1 status and materializes legacy outbox during claim', async () => { + const auditEvents: MemberWorkSyncAuditEvent[] = []; + store = new JsonMemberWorkSyncStore(new MemberWorkSyncStorePaths(root), { + auditJournal: { + append: async (event) => { + auditEvents.push(event); + }, + }, + now: () => new Date('2026-04-29T00:02:00.000Z'), + }); + const legacyStatusPath = join(root, 'team-a', '.member-work-sync', 'status.json'); + await mkdir(join(root, 'team-a', '.member-work-sync'), { recursive: true }); + await writeFile( + legacyStatusPath, + JSON.stringify({ schemaVersion: 1, members: { bob: makeStatus({}) } }), + 'utf8' + ); + + await expect(store.read({ teamName: 'team-a', memberName: 'bob' })).resolves.toMatchObject({ + memberName: 'bob', + state: 'needs_sync', + }); + + const input = { + id: 'member-work-sync:team-a:bob:agenda:v1:legacy', + teamName: 'team-a', + memberName: 'bob', + agendaFingerprint: 'agenda:v1:legacy', + payloadHash: 'hash-a', + payload: makeNudgePayload(), + status: 'pending' as const, + attemptGeneration: 0, + createdAt: '2026-04-29T00:00:00.000Z', + updatedAt: '2026-04-29T00:00:00.000Z', + }; + await writeFile( + join(root, 'team-a', '.member-work-sync', 'outbox.json'), + JSON.stringify({ schemaVersion: 1, items: { [input.id]: input } }), + 'utf8' + ); + + const claimed = await store.claimDue({ + teamName: 'team-a', + claimedBy: 'dispatcher-a', + nowIso: '2026-04-29T00:01:00.000Z', + limit: 1, + }); + expect(claimed).toHaveLength(1); + expect( + JSON.parse(await readFile(join(memberWorkSyncDir(root, 'team-a', 'bob'), 'outbox.json'), 'utf8')) + .items[input.id] + ).toMatchObject({ status: 'claimed' }); + expect(auditEvents.map((event) => `${event.event}:${event.reason}`)).toEqual( + expect.arrayContaining([ + 'legacy_fallback_used:status_v1', + 'index_repaired:outbox', + 'legacy_fallback_used:outbox_v1', + ]) + ); }); }); diff --git a/test/features/member-work-sync/main/MemberWorkSyncEventQueue.test.ts b/test/features/member-work-sync/main/MemberWorkSyncEventQueue.test.ts index dce25bb0..e0e8f66b 100644 --- a/test/features/member-work-sync/main/MemberWorkSyncEventQueue.test.ts +++ b/test/features/member-work-sync/main/MemberWorkSyncEventQueue.test.ts @@ -13,12 +13,18 @@ describe('MemberWorkSyncEventQueue', () => { it('coalesces duplicate member events into one queue reconcile', async () => { const reconciles: unknown[] = []; + const auditEvents: string[] = []; const queue = new MemberWorkSyncEventQueue({ quietWindowMs: 100, reconcile: async (request, context) => { reconciles.push({ request, context }); }, isTeamActive: () => true, + auditJournal: { + append: async (event) => { + auditEvents.push(event.event); + }, + }, }); queue.enqueue({ teamName: 'team-a', memberName: 'bob', triggerReason: 'task_changed' }); @@ -35,6 +41,7 @@ describe('MemberWorkSyncEventQueue', () => { }, }); expect(queue.getDiagnostics()).toMatchObject({ reconciled: 1, coalesced: 1 }); + expect(auditEvents).toEqual(['queue_enqueued', 'queue_coalesced', 'queue_reconciled']); await queue.stop(); }); diff --git a/test/features/member-work-sync/main/RuntimeTurnSettledIngestor.test.ts b/test/features/member-work-sync/main/RuntimeTurnSettledIngestor.test.ts index d536ba8d..53017f16 100644 --- a/test/features/member-work-sync/main/RuntimeTurnSettledIngestor.test.ts +++ b/test/features/member-work-sync/main/RuntimeTurnSettledIngestor.test.ts @@ -6,6 +6,7 @@ import { NodeHashAdapter } from '@features/member-work-sync/main/infrastructure/ import { OpenCodeTurnSettledPayloadNormalizer } from '@features/member-work-sync/main/infrastructure/OpenCodeTurnSettledPayloadNormalizer'; import type { + MemberWorkSyncAuditEvent, RuntimeTurnSettledClaimedPayload, RuntimeTurnSettledEventStorePort, RuntimeTurnSettledInvalidResult, @@ -56,6 +57,7 @@ describe('RuntimeTurnSettledIngestor', () => { resolve: vi.fn(async () => ({ ok: true as const, teamName: 'team-a', memberName: 'alice' })), }; const enqueueRuntimeTurnSettled = vi.fn(); + const auditEvents: MemberWorkSyncAuditEvent[] = []; const ingestor = new RuntimeTurnSettledIngestor({ eventStore: store, @@ -63,6 +65,11 @@ describe('RuntimeTurnSettledIngestor', () => { targetResolver: resolver, reconcileQueue: { enqueueRuntimeTurnSettled }, clock: { now: () => new Date('2026-04-29T12:01:00.000Z') }, + auditJournal: { + append: async (event) => { + auditEvents.push(event); + }, + }, }); await expect(ingestor.drainPending()).resolves.toEqual({ @@ -185,6 +192,7 @@ describe('RuntimeTurnSettledIngestor', () => { resolve: vi.fn(async () => ({ ok: true as const, teamName: 'team-a', memberName: 'jack' })), }; const enqueueRuntimeTurnSettled = vi.fn(); + const auditEvents: MemberWorkSyncAuditEvent[] = []; const ingestor = new RuntimeTurnSettledIngestor({ eventStore: store, @@ -192,6 +200,11 @@ describe('RuntimeTurnSettledIngestor', () => { targetResolver: resolver, reconcileQueue: { enqueueRuntimeTurnSettled }, clock: { now: () => new Date('2026-04-29T12:01:00.000Z') }, + auditJournal: { + append: async (event) => { + auditEvents.push(event); + }, + }, }); await expect(ingestor.drainPending()).resolves.toMatchObject({ @@ -220,6 +233,10 @@ describe('RuntimeTurnSettledIngestor', () => { teamName: 'team-a', memberName: 'jack', }); + expect(auditEvents.map((event) => event.event)).toEqual([ + 'turn_settled_claimed', + 'turn_settled_resolved', + ]); }); it.each([ diff --git a/test/main/services/team/OpenCodeRuntimeManifestEvidenceReader.test.ts b/test/main/services/team/OpenCodeRuntimeManifestEvidenceReader.test.ts index 945c9da0..c8473d60 100644 --- a/test/main/services/team/OpenCodeRuntimeManifestEvidenceReader.test.ts +++ b/test/main/services/team/OpenCodeRuntimeManifestEvidenceReader.test.ts @@ -264,6 +264,9 @@ describe('OpenCodeRuntimeManifestEvidenceReader migration', () => { ).resolves.toEqual({ laneDirectoryExists: false, hasStateOnDisk: false, + hasRuntimeEvidenceOnDisk: false, + manifestEntryCount: null, + manifestUpdatedAt: null, fileNames: [], }); @@ -293,6 +296,117 @@ describe('OpenCodeRuntimeManifestEvidenceReader migration', () => { }); }); + it('degrades an active lane that only has a stale empty runtime manifest', async () => { + const teamName = 'team-empty-manifest'; + const laneId = 'secondary:opencode:bob'; + + await upsertOpenCodeRuntimeLaneIndexEntry({ + teamsBasePath: tempDir, + teamName, + laneId, + state: 'active', + }); + await setOpenCodeRuntimeActiveRunManifest({ + teamsBasePath: tempDir, + teamName, + laneId, + runId: 'run-empty', + clock: () => new Date('2026-04-22T09:55:00.000Z'), + }); + await fs.writeFile( + getOpenCodeLaneScopedRuntimeFilePath({ + teamsBasePath: tempDir, + teamName, + laneId, + fileName: 'opencode-prompt-delivery-ledger.json', + }), + JSON.stringify({ records: [] }), + 'utf8' + ); + + await expect( + inspectOpenCodeRuntimeLaneStorage({ + teamsBasePath: tempDir, + teamName, + laneId, + }) + ).resolves.toMatchObject({ + laneDirectoryExists: true, + hasStateOnDisk: true, + hasRuntimeEvidenceOnDisk: false, + manifestEntryCount: 0, + fileNames: ['manifest.json', 'opencode-prompt-delivery-ledger.json'], + }); + + const result = await recoverStaleOpenCodeRuntimeLaneIndexEntry({ + teamsBasePath: tempDir, + teamName, + laneId, + clock: () => now, + emptyLaneStaleAfterMs: 150_000, + }); + + expect(result).toEqual({ + stale: true, + degraded: true, + diagnostics: [ + `OpenCode lane ${laneId} is marked active in lanes.json, but its runtime manifest has no committed runtime evidence after launch grace.`, + ], + }); + await expect(readOpenCodeRuntimeLaneIndex(tempDir, teamName)).resolves.toMatchObject({ + lanes: { + [laneId]: { + laneId, + state: 'degraded', + diagnostics: [ + `OpenCode lane ${laneId} is marked active in lanes.json, but its runtime manifest has no committed runtime evidence after launch grace.`, + ], + }, + }, + }); + }); + + it('does not degrade a fresh active lane while the empty runtime manifest is still inside launch grace', async () => { + const teamName = 'team-fresh-empty-manifest'; + const laneId = 'secondary:opencode:bob'; + + await upsertOpenCodeRuntimeLaneIndexEntry({ + teamsBasePath: tempDir, + teamName, + laneId, + state: 'active', + }); + await setOpenCodeRuntimeActiveRunManifest({ + teamsBasePath: tempDir, + teamName, + laneId, + runId: 'run-fresh', + clock: () => new Date('2026-04-22T09:59:00.000Z'), + }); + + const result = await recoverStaleOpenCodeRuntimeLaneIndexEntry({ + teamsBasePath: tempDir, + teamName, + laneId, + clock: () => now, + emptyLaneStaleAfterMs: 150_000, + }); + + expect(result).toEqual({ + stale: false, + degraded: false, + diagnostics: [], + }); + await expect(readOpenCodeRuntimeLaneIndex(tempDir, teamName)).resolves.toMatchObject({ + lanes: { + [laneId]: { + laneId, + state: 'active', + }, + }, + }); + }); + it('quarantines malformed lanes.json and falls back to an empty index', async () => { const teamName = 'team-zeta'; const runtimeDir = getOpenCodeTeamRuntimeDirectory(tempDir, teamName); diff --git a/test/main/services/team/OpenCodeTeamRuntimeAdapter.test.ts b/test/main/services/team/OpenCodeTeamRuntimeAdapter.test.ts index 924502ac..47bbc2d9 100644 --- a/test/main/services/team/OpenCodeTeamRuntimeAdapter.test.ts +++ b/test/main/services/team/OpenCodeTeamRuntimeAdapter.test.ts @@ -620,7 +620,7 @@ describe('OpenCodeTeamRuntimeAdapter', () => { }); }); - it('treats materialized bridge members without session or pid as accepted but not alive', async () => { + it('does not treat bridge members without session or pid as runtime candidates', async () => { const launchOpenCodeTeam = vi.fn( async () => ({ @@ -648,10 +648,10 @@ describe('OpenCodeTeamRuntimeAdapter', () => { expect(result.members.alice).toMatchObject({ launchState: 'runtime_pending_bootstrap', - agentToolAccepted: true, + agentToolAccepted: false, runtimeAlive: false, - livenessKind: 'runtime_process_candidate', - runtimeDiagnostic: 'OpenCode session exists without verified runtime pid', + livenessKind: 'registered_only', + runtimeDiagnostic: 'OpenCode bridge did not report a runtime session or pid for this member', }); }); diff --git a/test/main/services/team/TeamAgentLaunchMatrix.safe-e2e.test.ts b/test/main/services/team/TeamAgentLaunchMatrix.safe-e2e.test.ts index 2ddce750..91eeac67 100644 --- a/test/main/services/team/TeamAgentLaunchMatrix.safe-e2e.test.ts +++ b/test/main/services/team/TeamAgentLaunchMatrix.safe-e2e.test.ts @@ -780,7 +780,7 @@ describe('Team agent launch matrix safe e2e', () => { const second = await secondPromise; expect(second.runId).toBeTruthy(); expect(second.runId).not.toBe(firstRunId); - await waitForCondition(() => adapter.launchInputs.length === 2); + await waitForCondition(() => adapter.launchInputs.length === 1); expect(svc.isTeamAlive(teamName)).toBe(true); expect(svc.getAliveTeams()).toEqual([teamName]); @@ -1372,7 +1372,7 @@ describe('Team agent launch matrix safe e2e', () => { () => undefined ); - await waitForCondition(() => adapter.pendingLaunchInputs.length === 2); + await waitForCondition(() => adapter.pendingLaunchInputs.length === 1); const cancelledRunId = adapter.pendingLaunchInputs.find( (input) => input.teamName === cancelledTeamName )?.runId; @@ -1461,7 +1461,7 @@ describe('Team agent launch matrix safe e2e', () => { () => undefined ); - await waitForCondition(() => adapter.pendingLaunchInputs.length === 2); + await waitForCondition(() => adapter.pendingLaunchInputs.length === 1); const cancelledRunId = adapter.pendingLaunchInputs.find( (input) => input.teamName === cancelledTeamName )?.runId; @@ -1805,7 +1805,7 @@ describe('Team agent launch matrix safe e2e', () => { }, () => undefined ); - await waitForCondition(() => adapter.pendingLaunchInputs.length === 2); + await waitForCondition(() => adapter.pendingLaunchInputs.length === 1); const firstRunId = adapter.pendingLaunchInputs.find( (input) => input.teamName === firstTeamName )?.runId; @@ -1817,7 +1817,7 @@ describe('Team agent launch matrix safe e2e', () => { svc.stopAllTeams(); - await waitForCondition(() => adapter.stopInputs.length === 2); + await waitForCondition(() => adapter.stopInputs.length === 1); expect(adapter.stopInputs.map((input) => input.teamName).sort()).toEqual([ firstTeamName, secondTeamName, @@ -1867,7 +1867,7 @@ describe('Team agent launch matrix safe e2e', () => { expect(svc.getAliveTeams()).toEqual([]); adapter.releaseStops(); - await waitForCondition(() => adapter.stopInputs.length === 2); + await waitForCondition(() => adapter.stopInputs.length === 1); await expect( readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), firstTeamName) ).resolves.toMatchObject({ @@ -1909,7 +1909,7 @@ describe('Team agent launch matrix safe e2e', () => { }, () => undefined ); - await waitForCondition(() => adapter.pendingLaunchInputs.length === 2); + await waitForCondition(() => adapter.pendingLaunchInputs.length === 1); const firstRunId = adapter.pendingLaunchInputs.find( (input) => input.teamName === firstTeamName )?.runId; @@ -1921,7 +1921,7 @@ describe('Team agent launch matrix safe e2e', () => { svc.stopAllTeams(); - await waitForCondition(() => adapter.stopInputs.length === 2); + await waitForCondition(() => adapter.stopInputs.length === 1); expect(adapter.stopInputs.map((input) => input.teamName).sort()).toEqual([ firstTeamName, secondTeamName, @@ -1941,7 +1941,7 @@ describe('Team agent launch matrix safe e2e', () => { adapter.releaseLaunches(); await expect(firstPromise).resolves.toEqual({ runId: firstRunId }); await expect(secondPromise).resolves.toEqual({ runId: secondRunId }); - await waitForCondition(() => adapter.launchInputs.length === 2); + await waitForCondition(() => adapter.launchInputs.length === 1); expect(svc.getAliveTeams()).toEqual([]); await expect( @@ -2070,19 +2070,18 @@ describe('Team agent launch matrix safe e2e', () => { trackLiveRun(svc, run); await (svc as any).launchMixedSecondaryLaneIfNeeded(run); - await waitForCondition(() => adapter.pendingLaunchInputs.length === 2); + await waitForCondition(() => adapter.pendingLaunchInputs.length === 1); svc.stopAllTeams(); - await waitForCondition(() => adapter.stopInputs.length === 2); + await waitForCondition(() => adapter.stopInputs.length === 1); expect(adapter.stopInputs.map((input) => input.laneId).sort()).toEqual([ 'secondary:opencode:bob', - 'secondary:opencode:tom', ]); expect(svc.isTeamAlive(teamName)).toBe(false); adapter.releaseLaunches(); - await waitForCondition(() => adapter.rejectedLaunchCount === 2); + await waitForCondition(() => adapter.rejectedLaunchCount === 1); await expect(readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName)).resolves.toMatchObject( { @@ -2113,19 +2112,18 @@ describe('Team agent launch matrix safe e2e', () => { trackLiveRun(svc, cancelledRun); await (svc as any).launchMixedSecondaryLaneIfNeeded(cancelledRun); - await waitForCondition(() => adapter.pendingLaunchInputs.length === 2); + await waitForCondition(() => adapter.pendingLaunchInputs.length === 1); svc.stopAllTeams(); - await waitForCondition(() => adapter.stopInputs.length === 2); + await waitForCondition(() => adapter.stopInputs.length === 1); expect(adapter.stopInputs.map((input) => input.laneId).sort()).toEqual([ 'secondary:opencode:bob', - 'secondary:opencode:tom', ]); expect(svc.isTeamAlive(teamName)).toBe(false); adapter.releaseLaunches(); - await waitForCondition(() => adapter.launchInputs.length === 2); + await waitForCondition(() => adapter.launchInputs.length === 1); await expect(readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName)).resolves.toMatchObject( { lanes: {}, @@ -2139,7 +2137,7 @@ describe('Team agent launch matrix safe e2e', () => { trackLiveRun(svc, freshRun); await (svc as any).launchMixedSecondaryLaneIfNeeded(freshRun); - await waitForCondition(() => adapter.launchInputs.length === 4); + await waitForCondition(() => adapter.launchInputs.length === 3); await waitForCondition(() => freshRun.mixedSecondaryLanes.every((lane: { state: string }) => lane.state === 'finished') ); @@ -2200,27 +2198,23 @@ describe('Team agent launch matrix safe e2e', () => { await (svc as any).launchMixedSecondaryLaneIfNeeded(firstRun); await (svc as any).launchMixedSecondaryLaneIfNeeded(secondRun); - await waitForCondition(() => adapter.pendingLaunchInputs.length === 4); + await waitForCondition(() => adapter.pendingLaunchInputs.length === 1); svc.stopAllTeams(); - await waitForCondition(() => adapter.stopInputs.length === 4); + await waitForCondition(() => adapter.stopInputs.length === 1); expect(adapter.stopInputs.map((input) => input.teamName).sort()).toEqual([ firstTeamName, - firstTeamName, - secondTeamName, secondTeamName, ]); expect(adapter.stopInputs.map((input) => input.laneId).sort()).toEqual([ 'secondary:opencode:bob', 'secondary:opencode:bob', - 'secondary:opencode:tom', - 'secondary:opencode:tom', ]); expect(svc.getAliveTeams()).toEqual([]); adapter.releaseLaunches(); - await waitForCondition(() => adapter.rejectedLaunchCount === 4); + await waitForCondition(() => adapter.rejectedLaunchCount === 1); await expect( readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), firstTeamName) @@ -5619,14 +5613,14 @@ describe('Team agent launch matrix safe e2e', () => { const statuses = await svc.getMemberSpawnStatuses(teamName); expect(statuses.expectedMembers).toEqual(['alice', 'bob']); - expect(statuses.teamLaunchState).toBe('clean_success'); + expect(statuses.teamLaunchState).toBe('partial_pending'); expect(statuses.statuses.bob).toMatchObject({ - status: 'online', - launchState: 'confirmed_alive', - bootstrapConfirmed: true, + status: 'spawning', + launchState: 'starting', + bootstrapConfirmed: false, hardFailure: false, - livenessSource: 'heartbeat', }); + expect(statuses.statuses.bob?.livenessSource).not.toBe('heartbeat'); expect(statuses.statuses['bob-2']).toBeUndefined(); }); @@ -6387,14 +6381,14 @@ describe('Team agent launch matrix safe e2e', () => { const statuses = await new TeamProvisioningService().getMemberSpawnStatuses(teamName); expect(statuses.expectedMembers).toEqual(['alice', 'bob']); - expect(statuses.teamLaunchState).toBe('clean_success'); + expect(statuses.teamLaunchState).toBe('partial_pending'); expect(statuses.statuses.bob).toMatchObject({ - status: 'online', - launchState: 'confirmed_alive', - bootstrapConfirmed: true, + status: 'waiting', + launchState: 'runtime_pending_bootstrap', + bootstrapConfirmed: false, hardFailure: false, - lastHeartbeatAt: newerSignalAt, }); + expect(statuses.statuses.bob?.lastHeartbeatAt).not.toBe(newerSignalAt); expect(statuses.statuses['bob-2']).toBeUndefined(); }); @@ -6672,14 +6666,14 @@ describe('Team agent launch matrix safe e2e', () => { const statuses = await new TeamProvisioningService().getMemberSpawnStatuses(teamName); expect(statuses.expectedMembers).toEqual(['alice', 'bob']); - expect(statuses.teamLaunchState).toBe('clean_success'); + expect(statuses.teamLaunchState).toBe('partial_pending'); expect(statuses.statuses.bob).toMatchObject({ - status: 'online', - launchState: 'confirmed_alive', - bootstrapConfirmed: true, + status: 'waiting', + launchState: 'runtime_pending_bootstrap', + bootstrapConfirmed: false, hardFailure: false, - lastHeartbeatAt: signalAt, }); + expect(statuses.statuses.bob?.lastHeartbeatAt).not.toBe(signalAt); expect(statuses.statuses['bob-2']).toBeUndefined(); }); @@ -10076,7 +10070,7 @@ describe('Team agent launch matrix safe e2e', () => { await (svc as any).launchMixedSecondaryLaneIfNeeded(cancelledRun); await (svc as any).launchMixedSecondaryLaneIfNeeded(survivingRun); - await waitForCondition(() => adapter.pendingLaunchInputs.length === 4); + await waitForCondition(() => adapter.pendingLaunchInputs.length === 1); await svc.cancelProvisioning(cancelledRun.runId); @@ -10086,7 +10080,7 @@ describe('Team agent launch matrix safe e2e', () => { expect(adapter.stopInputs.some((input) => input.teamName === survivingTeamName)).toBe(false); adapter.releaseLaunches(); - await waitForCondition(() => adapter.launchInputs.length === 4); + await waitForCondition(() => adapter.launchInputs.length === 3); await waitForCondition(() => survivingRun.mixedSecondaryLanes.every((lane: { state: string }) => lane.state === 'finished') ); @@ -10103,7 +10097,7 @@ describe('Team agent launch matrix safe e2e', () => { trackLiveRun(svc, freshRun); await (svc as any).launchMixedSecondaryLaneIfNeeded(freshRun); - await waitForCondition(() => adapter.launchInputs.length === 6); + await waitForCondition(() => adapter.launchInputs.length === 5); await waitForCondition(() => freshRun.mixedSecondaryLanes.every((lane: { state: string }) => lane.state === 'finished') ); @@ -10201,19 +10195,19 @@ describe('Team agent launch matrix safe e2e', () => { await (svc as any).launchMixedSecondaryLaneIfNeeded(cancelledRun); await (svc as any).launchMixedSecondaryLaneIfNeeded(survivingRun); - await waitForCondition(() => adapter.pendingLaunchInputs.length === 4); + await waitForCondition(() => adapter.pendingLaunchInputs.length === 2); await svc.cancelProvisioning(cancelledRun.runId); await waitForCondition( - () => adapter.stopInputs.filter((input) => input.teamName === cancelledTeamName).length === 2 + () => adapter.stopInputs.filter((input) => input.teamName === cancelledTeamName).length === 1 ); expect(adapter.stopInputs.some((input) => input.teamName === survivingTeamName)).toBe(false); expect(svc.isTeamAlive(cancelledTeamName)).toBe(false); expect(svc.isTeamAlive(survivingTeamName)).toBe(true); adapter.releaseLaunches(); - await waitForCondition(() => adapter.launchInputs.length === 4); + await waitForCondition(() => adapter.launchInputs.length === 3); await waitForCondition(() => survivingRun.mixedSecondaryLanes.every((lane: { state: string }) => lane.state === 'finished') ); @@ -11004,7 +10998,7 @@ describe('Team agent launch matrix safe e2e', () => { await waitForCondition(() => adapter.launchInputs.length === 2); svc.stopTeam(teamName); - await waitForCondition(() => adapter.stopInputs.length === 2); + await waitForCondition(() => adapter.stopInputs.length === 1); await waitForCondition(() => !svc.isTeamAlive(teamName)); expect((await readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName)).lanes).toEqual({}); @@ -12777,18 +12771,17 @@ describe('Team agent launch matrix safe e2e', () => { trackLiveRun(svc, currentRun); await (svc as any).launchMixedSecondaryLaneIfNeeded(currentRun); - await waitForCondition(() => adapter.pendingLaunchInputs.length === 2); + await waitForCondition(() => adapter.pendingLaunchInputs.length === 1); svc.stopTeam(teamName); - await waitForCondition(() => adapter.stopInputs.length === 2); + await waitForCondition(() => adapter.stopInputs.length === 1); expectDirectChildKillCount(staleKillCount, 0); expectDirectChildKillCount(currentKillCount, 1); expect(staleRun.cancelRequested).toBe(false); expect(currentRun.cancelRequested).toBe(true); expect(adapter.stopInputs.map((input) => input.laneId).sort()).toEqual([ 'secondary:opencode:bob', - 'secondary:opencode:tom', ]); expect(await svc.getRuntimeState(teamName)).toMatchObject({ teamName, @@ -12835,7 +12828,7 @@ describe('Team agent launch matrix safe e2e', () => { trackLiveRun(svc, currentRun); await (svc as any).launchMixedSecondaryLaneIfNeeded(currentRun); - await waitForCondition(() => adapter.pendingLaunchInputs.length === 2); + await waitForCondition(() => adapter.pendingLaunchInputs.length === 1); await svc.cancelProvisioning(staleRun.runId); @@ -14691,7 +14684,7 @@ describe('Team agent launch matrix safe e2e', () => { const initialSnapshot = await (svc as any).launchMixedSecondaryLaneIfNeeded(run); expect(initialSnapshot.teamLaunchState).toBe('partial_pending'); - await waitForCondition(() => adapter.pendingLaunchInputs.length === 2); + await waitForCondition(() => adapter.pendingLaunchInputs.length === 1); const inFlightStatuses = await svc.getMemberSpawnStatuses(teamName); expect(inFlightStatuses.teamLaunchState).toBe('partial_pending'); @@ -14747,7 +14740,7 @@ describe('Team agent launch matrix safe e2e', () => { const initialSnapshot = await (svc as any).launchMixedSecondaryLaneIfNeeded(run); expect(initialSnapshot.teamLaunchState).toBe('partial_pending'); - await waitForCondition(() => adapter.pendingLaunchInputs.length === 2); + await waitForCondition(() => adapter.pendingLaunchInputs.length === 1); const inFlightStatuses = await svc.getMemberSpawnStatuses(teamName); expect(inFlightStatuses.teamLaunchState).toBe('partial_pending'); @@ -14806,14 +14799,14 @@ describe('Team agent launch matrix safe e2e', () => { trackLiveRun(svc, run); await (svc as any).launchMixedSecondaryLaneIfNeeded(run); - await waitForCondition(() => adapter.pendingLaunchInputs.length === 2); + await waitForCondition(() => adapter.pendingLaunchInputs.length === 1); const firstLaneRunIds = run.mixedSecondaryLanes.map( (lane: { runId: string | null }) => lane.runId ); await (svc as any).launchMixedSecondaryLaneIfNeeded(run); - expect(adapter.pendingLaunchInputs).toHaveLength(2); + expect(adapter.pendingLaunchInputs).toHaveLength(1); expect(adapter.launchInputs).toHaveLength(0); expect(run.mixedSecondaryLanes.map((lane: { state: string }) => lane.state)).toEqual([ 'launching', @@ -14857,14 +14850,14 @@ describe('Team agent launch matrix safe e2e', () => { trackLiveRun(svc, run); await (svc as any).launchMixedSecondaryLaneIfNeeded(run); - await waitForCondition(() => adapter.pendingLaunchInputs.length === 2); + await waitForCondition(() => adapter.pendingLaunchInputs.length === 1); const firstLaneRunIds = run.mixedSecondaryLanes.map( (lane: { runId: string | null }) => lane.runId ); await (svc as any).launchMixedSecondaryLaneIfNeeded(run); - expect(adapter.pendingLaunchInputs).toHaveLength(2); + expect(adapter.pendingLaunchInputs).toHaveLength(1); expect(adapter.launchInputs).toHaveLength(0); expect(run.mixedSecondaryLanes.map((lane: { state: string }) => lane.state)).toEqual([ 'launching', @@ -15011,19 +15004,18 @@ describe('Team agent launch matrix safe e2e', () => { trackLiveRun(svc, run); await (svc as any).launchMixedSecondaryLaneIfNeeded(run); - await waitForCondition(() => adapter.pendingLaunchInputs.length === 2); + await waitForCondition(() => adapter.pendingLaunchInputs.length === 1); svc.stopTeam(teamName); await waitForCondition(() => !svc.isTeamAlive(teamName)); - await waitForCondition(() => adapter.stopInputs.length === 2); + await waitForCondition(() => adapter.stopInputs.length === 1); expect(adapter.stopInputs.map((input) => input.laneId).sort()).toEqual([ 'secondary:opencode:bob', - 'secondary:opencode:tom', ]); adapter.releaseLaunches(); - await waitForCondition(() => adapter.launchInputs.length === 2); + await waitForCondition(() => adapter.launchInputs.length === 1); const statuses = await svc.getMemberSpawnStatuses(teamName); expect(svc.isTeamAlive(teamName)).toBe(false); @@ -15045,19 +15037,18 @@ describe('Team agent launch matrix safe e2e', () => { trackLiveRun(svc, run); await (svc as any).launchMixedSecondaryLaneIfNeeded(run); - await waitForCondition(() => adapter.pendingLaunchInputs.length === 2); + await waitForCondition(() => adapter.pendingLaunchInputs.length === 1); svc.stopTeam(teamName); await waitForCondition(() => !svc.isTeamAlive(teamName)); - await waitForCondition(() => adapter.stopInputs.length === 2); + await waitForCondition(() => adapter.stopInputs.length === 1); expect(adapter.stopInputs.map((input) => input.laneId).sort()).toEqual([ 'secondary:opencode:bob', - 'secondary:opencode:tom', ]); adapter.releaseLaunches(); - await waitForCondition(() => adapter.launchInputs.length === 2); + await waitForCondition(() => adapter.launchInputs.length === 1); const statuses = await svc.getMemberSpawnStatuses(teamName); expect(svc.isTeamAlive(teamName)).toBe(false); @@ -15124,19 +15115,18 @@ describe('Team agent launch matrix safe e2e', () => { trackLiveRun(svc, run); await (svc as any).launchMixedSecondaryLaneIfNeeded(run); - await waitForCondition(() => adapter.pendingLaunchInputs.length === 2); + await waitForCondition(() => adapter.pendingLaunchInputs.length === 1); svc.stopTeam(teamName); await waitForCondition(() => !svc.isTeamAlive(teamName)); - await waitForCondition(() => adapter.stopInputs.length === 2); + await waitForCondition(() => adapter.stopInputs.length === 1); expect(adapter.stopInputs.map((input) => input.laneId).sort()).toEqual([ 'secondary:opencode:bob', - 'secondary:opencode:tom', ]); adapter.releaseLaunches(); - await waitForCondition(() => adapter.launchInputs.length === 2); + await waitForCondition(() => adapter.launchInputs.length === 1); const statuses = await svc.getMemberSpawnStatuses(teamName); expect(svc.isTeamAlive(teamName)).toBe(false); @@ -15175,19 +15165,19 @@ describe('Team agent launch matrix safe e2e', () => { await (svc as any).launchMixedSecondaryLaneIfNeeded(stoppedRun); await (svc as any).launchMixedSecondaryLaneIfNeeded(survivingRun); - await waitForCondition(() => adapter.pendingLaunchInputs.length === 4); + await waitForCondition(() => adapter.pendingLaunchInputs.length === 2); svc.stopTeam(stoppedTeamName); await waitForCondition( - () => adapter.stopInputs.filter((input) => input.teamName === stoppedTeamName).length === 2 + () => adapter.stopInputs.filter((input) => input.teamName === stoppedTeamName).length === 1 ); expect(adapter.stopInputs.some((input) => input.teamName === survivingTeamName)).toBe(false); expect(svc.isTeamAlive(stoppedTeamName)).toBe(false); expect(svc.isTeamAlive(survivingTeamName)).toBe(true); adapter.releaseLaunches(); - await waitForCondition(() => adapter.launchInputs.length === 4); + await waitForCondition(() => adapter.launchInputs.length === 3); await waitForCondition(() => survivingRun.mixedSecondaryLanes.every((lane: { state: string }) => lane.state === 'finished') ); @@ -15223,11 +15213,11 @@ describe('Team agent launch matrix safe e2e', () => { trackLiveRun(svc, oldRun); await (svc as any).launchMixedSecondaryLaneIfNeeded(oldRun); - await waitForCondition(() => adapter.pendingLaunchInputs.length === 2); + await waitForCondition(() => adapter.pendingLaunchInputs.length === 1); svc.stopTeam(teamName); await waitForCondition(() => !svc.isTeamAlive(teamName)); - await waitForCondition(() => adapter.stopInputs.length === 2); + await waitForCondition(() => adapter.stopInputs.length === 1); await writeMixedTeamLaunchState({ teamName, @@ -15275,7 +15265,7 @@ describe('Team agent launch matrix safe e2e', () => { }); adapter.releaseLaunches(); - await waitForCondition(() => adapter.launchInputs.length === 2); + await waitForCondition(() => adapter.launchInputs.length === 1); const statuses = await svc.getMemberSpawnStatuses(teamName); expect(statuses.teamLaunchState).toBe('partial_failure'); @@ -15307,11 +15297,11 @@ describe('Team agent launch matrix safe e2e', () => { trackLiveRun(svc, oldRun); await (svc as any).launchMixedSecondaryLaneIfNeeded(oldRun); - await waitForCondition(() => adapter.pendingLaunchInputs.length === 2); + await waitForCondition(() => adapter.pendingLaunchInputs.length === 1); svc.stopTeam(teamName); await waitForCondition(() => !svc.isTeamAlive(teamName)); - await waitForCondition(() => adapter.stopInputs.length === 2); + await waitForCondition(() => adapter.stopInputs.length === 1); await writeMixedTeamLaunchState({ teamName, @@ -15358,7 +15348,7 @@ describe('Team agent launch matrix safe e2e', () => { }); adapter.releaseLaunches(); - await waitForCondition(() => adapter.launchInputs.length === 2); + await waitForCondition(() => adapter.launchInputs.length === 1); const statuses = await svc.getMemberSpawnStatuses(teamName); expect(statuses.teamLaunchState).toBe('partial_failure'); @@ -15428,11 +15418,11 @@ describe('Team agent launch matrix safe e2e', () => { trackLiveRun(svc, oldRun); await (svc as any).launchMixedSecondaryLaneIfNeeded(oldRun); - await waitForCondition(() => adapter.pendingLaunchInputs.length === 2); + await waitForCondition(() => adapter.pendingLaunchInputs.length === 1); svc.stopTeam(teamName); await waitForCondition(() => !svc.isTeamAlive(teamName)); - await waitForCondition(() => adapter.stopInputs.length === 2); + await waitForCondition(() => adapter.stopInputs.length === 1); await writeMixedTeamLaunchState({ teamName, @@ -15526,14 +15516,14 @@ describe('Team agent launch matrix safe e2e', () => { trackLiveRun(svc, run); await (svc as any).launchMixedSecondaryLaneIfNeeded(run); - await waitForCondition(() => adapter.pendingLaunchInputs.length === 2); + await waitForCondition(() => adapter.pendingLaunchInputs.length === 1); svc.stopTeam(teamName); await waitForCondition(() => !svc.isTeamAlive(teamName)); - await waitForCondition(() => adapter.stopInputs.length === 2); + await waitForCondition(() => adapter.stopInputs.length === 1); adapter.releaseLaunches(); - await waitForCondition(() => adapter.rejectedLaunchCount === 2); + await waitForCondition(() => adapter.rejectedLaunchCount === 1); await expect(readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName)).resolves.toMatchObject( { @@ -15564,14 +15554,14 @@ describe('Team agent launch matrix safe e2e', () => { trackLiveRun(svc, run); await (svc as any).launchMixedSecondaryLaneIfNeeded(run); - await waitForCondition(() => adapter.pendingLaunchInputs.length === 2); + await waitForCondition(() => adapter.pendingLaunchInputs.length === 1); svc.stopTeam(teamName); await waitForCondition(() => !svc.isTeamAlive(teamName)); - await waitForCondition(() => adapter.stopInputs.length === 2); + await waitForCondition(() => adapter.stopInputs.length === 1); adapter.releaseLaunches(); - await waitForCondition(() => adapter.rejectedLaunchCount === 2); + await waitForCondition(() => adapter.rejectedLaunchCount === 1); await expect(readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName)).resolves.toMatchObject( { @@ -15642,14 +15632,14 @@ describe('Team agent launch matrix safe e2e', () => { trackLiveRun(svc, run); await (svc as any).launchMixedSecondaryLaneIfNeeded(run); - await waitForCondition(() => adapter.pendingLaunchInputs.length === 2); + await waitForCondition(() => adapter.pendingLaunchInputs.length === 1); svc.stopTeam(teamName); await waitForCondition(() => !svc.isTeamAlive(teamName)); - await waitForCondition(() => adapter.stopInputs.length === 2); + await waitForCondition(() => adapter.stopInputs.length === 1); adapter.releaseLaunches(); - await waitForCondition(() => adapter.rejectedLaunchCount === 2); + await waitForCondition(() => adapter.rejectedLaunchCount === 1); await expect(readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName)).resolves.toMatchObject( { @@ -15694,7 +15684,6 @@ describe('Team agent launch matrix safe e2e', () => { await waitForCondition(() => adapter.stopInputs.length === 2); expect(adapter.stopInputs.map((input) => input.laneId).sort()).toEqual([ 'secondary:opencode:bob', - 'secondary:opencode:tom', ]); expect(svc.isTeamAlive(teamName)).toBe(false); @@ -15727,12 +15716,11 @@ describe('Team agent launch matrix safe e2e', () => { await waitForCondition(() => adapter.stopInputs.length === 2); expect(adapter.stopInputs.map((input) => input.laneId).sort()).toEqual([ 'secondary:opencode:bob', - 'secondary:opencode:tom', ]); expect(svc.isTeamAlive(teamName)).toBe(false); adapter.releaseLaunches(); - await waitForCondition(() => adapter.launchInputs.length === 2); + await waitForCondition(() => adapter.launchInputs.length === 1); const statuses = await svc.getMemberSpawnStatuses(teamName); expect(statuses.teamLaunchState).not.toBe('clean_success'); @@ -15767,19 +15755,18 @@ describe('Team agent launch matrix safe e2e', () => { trackLiveRun(svc, run); await (svc as any).launchMixedSecondaryLaneIfNeeded(run); - await waitForCondition(() => adapter.pendingLaunchInputs.length === 2); + await waitForCondition(() => adapter.pendingLaunchInputs.length === 1); await svc.cancelProvisioning(run.runId); - await waitForCondition(() => adapter.stopInputs.length === 2); + await waitForCondition(() => adapter.stopInputs.length === 1); expect(adapter.stopInputs.map((input) => input.laneId).sort()).toEqual([ 'secondary:opencode:bob', - 'secondary:opencode:tom', ]); expect(svc.isTeamAlive(teamName)).toBe(false); adapter.releaseLaunches(); - await waitForCondition(() => adapter.launchInputs.length === 2); + await waitForCondition(() => adapter.launchInputs.length === 1); const statuses = await svc.getMemberSpawnStatuses(teamName); expect(statuses.teamLaunchState).not.toBe('clean_success'); @@ -15825,15 +15812,14 @@ describe('Team agent launch matrix safe e2e', () => { await svc.cancelProvisioning(cancelledRun.runId); - await waitForCondition(() => adapter.stopInputs.length === 2); + await waitForCondition(() => adapter.stopInputs.length === 1); expect(adapter.stopInputs.map((input) => input.laneId).sort()).toEqual([ 'secondary:opencode:bob', - 'secondary:opencode:tom', ]); expect(svc.isTeamAlive(teamName)).toBe(false); adapter.releaseLaunches(); - await waitForCondition(() => adapter.launchInputs.length === 2); + await waitForCondition(() => adapter.launchInputs.length === 1); const cancelledStatuses = await svc.getMemberSpawnStatuses(teamName); expect(cancelledStatuses.teamLaunchState).not.toBe('clean_success'); @@ -15849,7 +15835,7 @@ describe('Team agent launch matrix safe e2e', () => { trackLiveRun(svc, freshRun); await (svc as any).launchMixedSecondaryLaneIfNeeded(freshRun); - await waitForCondition(() => adapter.launchInputs.length === 4); + await waitForCondition(() => adapter.launchInputs.length === 3); await waitForCondition(() => freshRun.mixedSecondaryLanes.every((lane: { state: string }) => lane.state === 'finished') ); @@ -15902,19 +15888,19 @@ describe('Team agent launch matrix safe e2e', () => { await (svc as any).launchMixedSecondaryLaneIfNeeded(cancelledRun); await (svc as any).launchMixedSecondaryLaneIfNeeded(survivingRun); - await waitForCondition(() => adapter.pendingLaunchInputs.length === 4); + await waitForCondition(() => adapter.pendingLaunchInputs.length === 2); await svc.cancelProvisioning(cancelledRun.runId); await waitForCondition( - () => adapter.stopInputs.filter((input) => input.teamName === cancelledTeamName).length === 2 + () => adapter.stopInputs.filter((input) => input.teamName === cancelledTeamName).length === 1 ); expect(adapter.stopInputs.some((input) => input.teamName === survivingTeamName)).toBe(false); expect(svc.isTeamAlive(cancelledTeamName)).toBe(false); expect(svc.isTeamAlive(survivingTeamName)).toBe(true); adapter.releaseLaunches(); - await waitForCondition(() => adapter.launchInputs.length === 4); + await waitForCondition(() => adapter.launchInputs.length === 3); await waitForCondition(() => survivingRun.mixedSecondaryLanes.every((lane: { state: string }) => lane.state === 'finished') ); @@ -15987,19 +15973,19 @@ describe('Team agent launch matrix safe e2e', () => { await (svc as any).launchMixedSecondaryLaneIfNeeded(cancelledRun); await (svc as any).launchMixedSecondaryLaneIfNeeded(survivingRun); - await waitForCondition(() => adapter.pendingLaunchInputs.length === 4); + await waitForCondition(() => adapter.pendingLaunchInputs.length === 2); await svc.cancelProvisioning(cancelledRun.runId); await waitForCondition( - () => adapter.stopInputs.filter((input) => input.teamName === cancelledTeamName).length === 2 + () => adapter.stopInputs.filter((input) => input.teamName === cancelledTeamName).length === 1 ); expect(adapter.stopInputs.some((input) => input.teamName === survivingTeamName)).toBe(false); expect(svc.isTeamAlive(cancelledTeamName)).toBe(false); expect(svc.isTeamAlive(survivingTeamName)).toBe(true); adapter.releaseLaunches(); - await waitForCondition(() => adapter.launchInputs.length === 4); + await waitForCondition(() => adapter.launchInputs.length === 3); await waitForCondition(() => survivingRun.mixedSecondaryLanes.every((lane: { state: string }) => lane.state === 'finished') ); @@ -16051,17 +16037,17 @@ describe('Team agent launch matrix safe e2e', () => { await (svc as any).launchMixedSecondaryLaneIfNeeded(cancelledRun); await (svc as any).launchMixedSecondaryLaneIfNeeded(survivingRun); - await waitForCondition(() => adapter.pendingLaunchInputs.length === 4); + await waitForCondition(() => adapter.pendingLaunchInputs.length === 2); await svc.cancelProvisioning(cancelledRun.runId); await waitForCondition( - () => adapter.stopInputs.filter((input) => input.teamName === cancelledTeamName).length === 2 + () => adapter.stopInputs.filter((input) => input.teamName === cancelledTeamName).length === 1 ); expect(adapter.stopInputs.some((input) => input.teamName === survivingTeamName)).toBe(false); adapter.releaseLaunches(); - await waitForCondition(() => adapter.launchInputs.length === 4); + await waitForCondition(() => adapter.launchInputs.length === 3); await waitForCondition(() => survivingRun.mixedSecondaryLanes.every((lane: { state: string }) => lane.state === 'finished') ); @@ -16073,7 +16059,7 @@ describe('Team agent launch matrix safe e2e', () => { trackLiveRun(svc, freshRun); await (svc as any).launchMixedSecondaryLaneIfNeeded(freshRun); - await waitForCondition(() => adapter.launchInputs.length === 6); + await waitForCondition(() => adapter.launchInputs.length === 5); await waitForCondition(() => freshRun.mixedSecondaryLanes.every((lane: { state: string }) => lane.state === 'finished') ); @@ -16154,17 +16140,17 @@ describe('Team agent launch matrix safe e2e', () => { await (svc as any).launchMixedSecondaryLaneIfNeeded(cancelledRun); await (svc as any).launchMixedSecondaryLaneIfNeeded(survivingRun); - await waitForCondition(() => adapter.pendingLaunchInputs.length === 4); + await waitForCondition(() => adapter.pendingLaunchInputs.length === 2); await svc.cancelProvisioning(cancelledRun.runId); await waitForCondition( - () => adapter.stopInputs.filter((input) => input.teamName === cancelledTeamName).length === 2 + () => adapter.stopInputs.filter((input) => input.teamName === cancelledTeamName).length === 1 ); expect(adapter.stopInputs.some((input) => input.teamName === survivingTeamName)).toBe(false); adapter.releaseLaunches(); - await waitForCondition(() => adapter.launchInputs.length === 4); + await waitForCondition(() => adapter.launchInputs.length === 3); await waitForCondition(() => survivingRun.mixedSecondaryLanes.every((lane: { state: string }) => lane.state === 'finished') ); @@ -16181,7 +16167,7 @@ describe('Team agent launch matrix safe e2e', () => { trackLiveRun(svc, freshRun); await (svc as any).launchMixedSecondaryLaneIfNeeded(freshRun); - await waitForCondition(() => adapter.launchInputs.length === 6); + await waitForCondition(() => adapter.launchInputs.length === 5); await waitForCondition(() => freshRun.mixedSecondaryLanes.every((lane: { state: string }) => lane.state === 'finished') ); diff --git a/test/main/services/team/TeamBackupService.test.ts b/test/main/services/team/TeamBackupService.test.ts index b67d7a3d..e22e7d62 100644 --- a/test/main/services/team/TeamBackupService.test.ts +++ b/test/main/services/team/TeamBackupService.test.ts @@ -315,4 +315,84 @@ describe('TeamBackupService', () => { warnSpy.mockRestore(); } }); + + it('backs up member-scoped work sync files', async () => { + const service = new TeamBackupService(); + const teamName = 'member-work-sync-team'; + const teamDir = path.join(hoisted.teamsBase, teamName); + const memberDir = path.join(teamDir, 'members', 'jack'); + const workSyncDir = path.join(memberDir, '.member-work-sync'); + const status = { + teamName, + memberName: 'jack', + state: 'caught_up', + evaluatedAt: '2026-04-30T12:00:00.000Z', + agenda: { fingerprint: 'abc123', items: [] }, + }; + + try { + await fs.mkdir(workSyncDir, { recursive: true }); + await fs.writeFile( + path.join(teamDir, 'config.json'), + JSON.stringify({ name: 'Member Work Sync Team' }), + 'utf8' + ); + await fs.writeFile( + path.join(memberDir, 'member.meta.json'), + JSON.stringify({ + schemaVersion: 1, + memberName: 'jack', + memberKey: 'jack', + updatedAt: '2026-04-30T12:00:00.000Z', + }), + 'utf8' + ); + await fs.writeFile( + path.join(workSyncDir, 'status.json'), + JSON.stringify({ schemaVersion: 2, status }), + 'utf8' + ); + await fs.writeFile( + path.join(workSyncDir, 'journal.jsonl'), + `${JSON.stringify({ + schemaVersion: 1, + timestamp: '2026-04-30T12:00:00.000Z', + teamName, + memberName: 'jack', + event: 'status_written', + source: 'test', + })}\n`, + 'utf8' + ); + await fs.writeFile(path.join(workSyncDir, '.tmp.deadbeef'), '{"partial":', 'utf8'); + await fs.writeFile(path.join(workSyncDir, 'journal.jsonl.lock'), '123\n', 'utf8'); + + await service.initialize(); + await service.backupTeam(teamName); + + const backupMemberDir = path.join(hoisted.backupsBase, 'teams', teamName, 'members', 'jack'); + await expect(fs.readFile(path.join(backupMemberDir, 'member.meta.json'), 'utf8')).resolves.toBe( + JSON.stringify({ + schemaVersion: 1, + memberName: 'jack', + memberKey: 'jack', + updatedAt: '2026-04-30T12:00:00.000Z', + }) + ); + await expect( + fs.readFile(path.join(backupMemberDir, '.member-work-sync', 'status.json'), 'utf8') + ).resolves.toBe(JSON.stringify({ schemaVersion: 2, status })); + await expect( + fs.readFile(path.join(backupMemberDir, '.member-work-sync', 'journal.jsonl'), 'utf8') + ).resolves.toContain('"event":"status_written"'); + await expect( + fs.stat(path.join(backupMemberDir, '.member-work-sync', '.tmp.deadbeef')) + ).rejects.toMatchObject({ code: 'ENOENT' }); + await expect( + fs.stat(path.join(backupMemberDir, '.member-work-sync', 'journal.jsonl.lock')) + ).rejects.toMatchObject({ code: 'ENOENT' }); + } finally { + service.dispose(); + } + }); }); diff --git a/test/main/services/team/TeamMemberStoragePaths.test.ts b/test/main/services/team/TeamMemberStoragePaths.test.ts new file mode 100644 index 00000000..501ee28f --- /dev/null +++ b/test/main/services/team/TeamMemberStoragePaths.test.ts @@ -0,0 +1,76 @@ +import { mkdtemp, readFile, rm } from 'fs/promises'; +import { tmpdir } from 'os'; +import { join } from 'path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { + encodeTeamMemberStorageKey, + TeamMemberStoragePaths, +} from '@main/services/team/TeamMemberStoragePaths'; + +describe('TeamMemberStoragePaths', () => { + let root: string; + let paths: TeamMemberStoragePaths; + + beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'team-member-storage-paths-')); + paths = new TeamMemberStoragePaths(root); + }); + + afterEach(async () => { + await rm(root, { recursive: true, force: true }); + }); + + it('builds stable path-safe keys from canonical member names', () => { + expect(encodeTeamMemberStorageKey(' Bob ')).toBe('bob'); + expect(encodeTeamMemberStorageKey('Jack Smith')).toBe('jack%20smith'); + expect(encodeTeamMemberStorageKey('../Alice')).toBe('..%2Falice'); + expect(encodeTeamMemberStorageKey('.')).toBe('%2E'); + expect(encodeTeamMemberStorageKey('..')).toBe('%2E%2E'); + expect(encodeTeamMemberStorageKey('Том')).toBe('%D1%82%D0%BE%D0%BC'); + }); + + it('keeps member storage inside the team members directory', () => { + expect(paths.getMemberDir('team-a', '../Alice')).toBe( + join(root, 'team-a', 'members', '..%2Falice') + ); + expect(paths.getMemberDir('team-a', '..')).toBe( + join(root, 'team-a', 'members', '%2E%2E') + ); + expect(paths.getMemberDir('team-a', '.')).toBe( + join(root, 'team-a', 'members', '%2E') + ); + expect(paths.getMemberFeatureDir('team-a', 'Bob', '.member-work-sync')).toBe( + join(root, 'team-a', 'members', 'bob', '.member-work-sync') + ); + }); + + it('rejects empty member names and nested feature directory names', () => { + expect(() => encodeTeamMemberStorageKey(' ')).toThrow('memberName is required'); + expect(() => paths.getMemberFeatureDir('team-a', 'Bob', '../unsafe')).toThrow( + 'featureDirName must be a single path segment' + ); + expect(() => paths.getMemberFeatureDir('team-a', 'Bob', 'nested/unsafe')).toThrow( + 'featureDirName must be a single path segment' + ); + expect(() => paths.getMemberFeatureDir('team-a', 'Bob', '..')).toThrow( + 'featureDirName must be a single path segment' + ); + expect(() => paths.getMemberFeatureDir('team-a', 'Bob', '.')).toThrow( + 'featureDirName must be a single path segment' + ); + }); + + it('materializes canonical member meta without changing the path key', async () => { + await paths.ensureMemberMeta('team-a', 'Bob'); + + const meta = JSON.parse( + await readFile(join(root, 'team-a', 'members', 'bob', 'member.meta.json'), 'utf8') + ); + expect(meta).toMatchObject({ + schemaVersion: 1, + memberName: 'Bob', + memberKey: 'bob', + }); + }); +}); diff --git a/test/main/services/team/TeamProvisioningService.test.ts b/test/main/services/team/TeamProvisioningService.test.ts index 014dc2ab..7f7d3117 100644 --- a/test/main/services/team/TeamProvisioningService.test.ts +++ b/test/main/services/team/TeamProvisioningService.test.ts @@ -136,6 +136,7 @@ import { getOpenCodeRuntimeManifestPath, OpenCodeRuntimeManifestEvidenceReader, readOpenCodeRuntimeLaneIndex, + setOpenCodeRuntimeActiveRunManifest, upsertOpenCodeRuntimeLaneIndexEntry, } from '@main/services/team/opencode/store/OpenCodeRuntimeManifestEvidenceReader'; import { createDefaultRuntimeStoreManifest } from '@main/services/team/opencode/store/RuntimeStoreManifest'; @@ -1049,7 +1050,7 @@ describe('TeamProvisioningService', () => { it('does not carry stale persisted runtimeAlive through launch-state reconcile', async () => { const teamName = 'persisted-stale-runtime-status-team'; const projectPath = '/Users/test/project'; - const acceptedAt = new Date(Date.now() - 120_000).toISOString(); + const acceptedAt = new Date(Date.now() - 220_000).toISOString(); writeLaunchConfig(teamName, projectPath, 'lead-session', ['alice']); writeLaunchState(teamName, 'lead-session', { alice: { @@ -5322,6 +5323,11 @@ describe('TeamProvisioningService', () => { )}\n`, 'utf8' ); + await fsPromises.writeFile( + path.join(path.dirname(manifestPath), 'opencode-sessions.json'), + `${JSON.stringify({ sessions: [{ id: 'oc-session-bob' }] })}\n`, + 'utf8' + ); await expect( svc.deliverOpenCodeMemberMessage(teamName, { @@ -5346,6 +5352,97 @@ describe('TeamProvisioningService', () => { ); }); + it('rejects stale active lane manifest without runtime evidence before delivery', async () => { + const svc = new TeamProvisioningService(); + const teamName = 'team-a'; + const laneId = 'secondary:opencode:bob'; + const sendMessageToMember = vi.fn(async (input: Record) => ({ + ok: true, + providerId: 'opencode', + memberName: String(input.memberName), + sessionId: 'oc-session-bob', + diagnostics: [], + })); + const registry = new TeamRuntimeAdapterRegistry([ + { + providerId: 'opencode', + prepare: vi.fn(), + launch: vi.fn(), + reconcile: vi.fn(), + stop: vi.fn(), + sendMessageToMember, + } as any, + ]); + svc.setRuntimeAdapterRegistry(registry); + + (svc as any).configReader = { + getConfig: vi.fn(async () => ({ + projectPath: '/repo', + members: [ + { name: 'team-lead', providerId: 'codex', model: 'gpt-5.4' }, + { name: 'bob', providerId: 'opencode', model: 'minimax-m2.5-free' }, + ], + })), + }; + (svc as any).teamMetaStore = { + getMeta: vi.fn(async () => ({ + launchIdentity: { providerId: 'codex' }, + providerId: 'codex', + })), + }; + (svc as any).membersMetaStore = { + getMembers: vi.fn(async () => [ + { + name: 'bob', + providerId: 'opencode', + model: 'opencode/minimax-m2.5-free', + }, + ]), + }; + await upsertOpenCodeRuntimeLaneIndexEntry({ + teamsBasePath: tempTeamsBase, + teamName, + laneId, + state: 'active', + }); + const manifestPath = getOpenCodeRuntimeManifestPath(tempTeamsBase, teamName, laneId); + await fsPromises.mkdir(path.dirname(manifestPath), { recursive: true }); + await fsPromises.writeFile( + manifestPath, + `${JSON.stringify( + { + ...createDefaultRuntimeStoreManifest(teamName, '2026-04-22T12:00:00.000Z'), + activeRunId: 'opencode-run-stale-empty', + }, + null, + 2 + )}\n`, + 'utf8' + ); + + await expect( + svc.deliverOpenCodeMemberMessage(teamName, { + memberName: 'bob', + text: 'must not deliver to empty durable lane', + messageId: 'msg-stale-empty-manifest', + }) + ).resolves.toMatchObject({ + delivered: false, + reason: 'opencode_runtime_not_active', + diagnostics: [ + expect.stringContaining('runtime manifest has no committed runtime evidence'), + ], + }); + expect(sendMessageToMember).not.toHaveBeenCalled(); + await expect(readOpenCodeRuntimeLaneIndex(tempTeamsBase, teamName)).resolves.toMatchObject({ + lanes: { + [laneId]: { + state: 'degraded', + }, + }, + }); + }); + it('falls back to lane manifest when a tracked primary run lacks the secondary lane snapshot', async () => { const svc = new TeamProvisioningService(); const teamName = 'team-a'; @@ -5417,6 +5514,11 @@ describe('TeamProvisioningService', () => { )}\n`, 'utf8' ); + await fsPromises.writeFile( + path.join(path.dirname(manifestPath), 'opencode-sessions.json'), + `${JSON.stringify({ sessions: [{ id: 'oc-session-bob' }] })}\n`, + 'utf8' + ); await expect( svc.deliverOpenCodeMemberMessage(teamName, { @@ -5578,7 +5680,7 @@ describe('TeamProvisioningService', () => { ); }); - it('starts all queued OpenCode secondary lanes without letting the first in-flight lane block its siblings', async () => { + it('starts queued OpenCode secondary lanes sequentially without blocking launch progress', async () => { const svc = new TeamProvisioningService(); const registry = new TeamRuntimeAdapterRegistry([ { @@ -5684,7 +5786,7 @@ describe('TeamProvisioningService', () => { await Promise.resolve(); await Promise.resolve(); - expect(launchSingleMixedSecondaryLane).toHaveBeenCalledTimes(3); + expect(launchSingleMixedSecondaryLane).toHaveBeenCalledTimes(1); expect(run.mixedSecondaryLanes.map((lane: { state: string }) => lane.state)).toEqual([ 'launching', 'launching', @@ -5696,6 +5798,7 @@ describe('TeamProvisioningService', () => { resolveFirstLaunch(); await Promise.resolve(); + await vi.waitFor(() => expect(launchSingleMixedSecondaryLane).toHaveBeenCalledTimes(3)); }); it('preserves mixed lane metadata when OpenCode runtime liveness updates a secondary lane member', async () => { @@ -5797,6 +5900,8 @@ describe('TeamProvisioningService', () => { launchState: 'confirmed_alive', runtimeAlive: true, bootstrapConfirmed: true, + runtimeRunId: 'run-member-spawn-1', + runtimeSessionId: 'session-bob', }); }); @@ -5876,6 +5981,71 @@ describe('TeamProvisioningService', () => { expect(diagnostics.join('\n')).not.toContain('super-secret'); }); + it('does not carry a stale OpenCode runtime pid into a fresh runtime run check-in', async () => { + const svc = new TeamProvisioningService(); + const previousSnapshot = { + version: 2 as const, + teamName: 'mixed-team', + updatedAt: '2026-04-22T12:00:00.000Z', + launchPhase: 'active' as const, + expectedMembers: ['bob'], + members: { + bob: { + name: 'bob', + providerId: 'opencode' as const, + laneId: 'secondary:opencode:bob', + laneKind: 'secondary' as const, + laneOwnerProviderId: 'opencode' as const, + launchState: 'confirmed_alive' as const, + agentToolAccepted: true, + runtimeAlive: true, + bootstrapConfirmed: true, + hardFailure: false, + runtimePid: 1111, + runtimeRunId: 'opencode-run-old', + runtimeSessionId: 'session-bob-old', + livenessKind: 'confirmed_bootstrap' as const, + pidSource: 'runtime_bootstrap' as const, + lastEvaluatedAt: '2026-04-22T12:00:00.000Z', + }, + }, + summary: { + confirmedCount: 1, + pendingCount: 0, + failedCount: 0, + runtimeAlivePendingCount: 0, + }, + teamLaunchState: 'ready' as const, + }; + const write = vi.fn(async () => {}); + + (svc as any).launchStateStore = { + read: vi.fn(async () => previousSnapshot), + write, + }; + + await (svc as any).updateOpenCodeRuntimeMemberLiveness({ + teamName: 'mixed-team', + runId: 'opencode-run-new', + memberName: 'bob', + runtimeSessionId: 'session-bob-new', + observedAt: '2026-04-22T12:05:00.000Z', + diagnostics: [], + reason: 'OpenCode runtime bootstrap check-in accepted', + }); + + const writtenSnapshot = ( + write.mock.calls[0] as unknown as [string, Record] | undefined + )?.[1] as { members?: Record> } | undefined; + expect(writtenSnapshot?.members?.bob).toMatchObject({ + runtimeRunId: 'opencode-run-new', + runtimeSessionId: 'session-bob-new', + launchState: 'confirmed_alive', + }); + expect(writtenSnapshot?.members?.bob?.runtimePid).toBeUndefined(); + expect(writtenSnapshot?.members?.bob?.pidSource).toBeUndefined(); + }); + it('preserves richer persisted expectedMembers when OpenCode runtime liveness updates a stale snapshot', async () => { const svc = new TeamProvisioningService(); const previousSnapshot = { @@ -5940,6 +6110,298 @@ describe('TeamProvisioningService', () => { expect(writtenSnapshot?.expectedMembers).toEqual(['bob', 'alice']); }); + it('accepts duplicate OpenCode bootstrap check-ins for the same runtime session without rewriting liveness', async () => { + const svc = new TeamProvisioningService(); + const previousSnapshot = { + version: 2 as const, + teamName: 'mixed-team', + updatedAt: '2026-04-22T12:00:00.000Z', + launchPhase: 'active' as const, + expectedMembers: ['bob'], + members: { + bob: { + name: 'bob', + providerId: 'opencode' as const, + laneId: 'secondary:opencode:bob', + laneKind: 'secondary' as const, + laneOwnerProviderId: 'opencode' as const, + launchState: 'confirmed_alive' as const, + agentToolAccepted: true, + runtimeAlive: true, + bootstrapConfirmed: true, + hardFailure: false, + runtimeRunId: 'opencode-run-1', + runtimeSessionId: 'session-bob', + livenessKind: 'confirmed_bootstrap' as const, + lastEvaluatedAt: '2026-04-22T12:00:00.000Z', + }, + }, + summary: { + confirmedCount: 1, + pendingCount: 0, + failedCount: 0, + runtimeAlivePendingCount: 0, + }, + teamLaunchState: 'ready' as const, + }; + const updateLiveness = vi.spyOn(svc as any, 'updateOpenCodeRuntimeMemberLiveness'); + + (svc as any).launchStateStore = { + read: vi.fn(async () => previousSnapshot), + write: vi.fn(async () => {}), + }; + (svc as any).resolveOpenCodeRuntimeLaneId = vi.fn(async () => 'secondary:opencode:bob'); + (svc as any).assertOpenCodeRuntimeEvidenceAccepted = vi.fn(async () => {}); + + const ack = await svc.recordOpenCodeRuntimeBootstrapCheckin({ + teamName: 'mixed-team', + runId: 'opencode-run-1', + memberName: 'bob', + runtimeSessionId: 'session-bob', + observedAt: '2026-04-22T12:05:00.000Z', + }); + + expect(ack).toMatchObject({ + ok: true, + state: 'accepted', + diagnostics: ['opencode_bootstrap_checkin_duplicate_accepted'], + runtimeSessionId: 'session-bob', + }); + expect(updateLiveness).not.toHaveBeenCalled(); + }); + + it('rejects duplicate OpenCode bootstrap check-ins for members removed after the first check-in', async () => { + const svc = new TeamProvisioningService(); + const previousSnapshot = { + version: 2 as const, + teamName: 'mixed-team', + updatedAt: '2026-04-22T12:00:00.000Z', + launchPhase: 'active' as const, + expectedMembers: ['bob'], + members: { + bob: { + name: 'bob', + providerId: 'opencode' as const, + laneId: 'secondary:opencode:bob', + laneKind: 'secondary' as const, + laneOwnerProviderId: 'opencode' as const, + launchState: 'confirmed_alive' as const, + agentToolAccepted: true, + runtimeAlive: true, + bootstrapConfirmed: true, + hardFailure: false, + runtimeRunId: 'opencode-run-1', + runtimeSessionId: 'session-bob', + livenessKind: 'confirmed_bootstrap' as const, + lastEvaluatedAt: '2026-04-22T12:00:00.000Z', + }, + }, + summary: { + confirmedCount: 1, + pendingCount: 0, + failedCount: 0, + runtimeAlivePendingCount: 0, + }, + teamLaunchState: 'ready' as const, + }; + const updateLiveness = vi.spyOn(svc as any, 'updateOpenCodeRuntimeMemberLiveness'); + + (svc as any).launchStateStore = { + read: vi.fn(async () => previousSnapshot), + write: vi.fn(async () => {}), + }; + (svc as any).resolveOpenCodeRuntimeLaneId = vi.fn(async () => 'secondary:opencode:bob'); + (svc as any).assertOpenCodeRuntimeEvidenceAccepted = vi.fn(async () => {}); + (svc as any).configReader = { + getConfig: vi.fn(async () => ({ + teamName: 'mixed-team', + members: [{ name: 'bob', providerId: 'opencode', removedAt: 123 }], + })), + }; + (svc as any).membersMetaStore = { + getMembers: vi.fn(async () => []), + }; + + await expect( + svc.recordOpenCodeRuntimeBootstrapCheckin({ + teamName: 'mixed-team', + runId: 'opencode-run-1', + memberName: 'bob', + runtimeSessionId: 'session-bob', + observedAt: '2026-04-22T12:05:00.000Z', + }) + ).rejects.toMatchObject({ + name: 'RuntimeStaleEvidenceError', + }); + expect(updateLiveness).not.toHaveBeenCalled(); + }); + + it('rejects conflicting OpenCode bootstrap check-ins for an already confirmed runtime session', async () => { + const svc = new TeamProvisioningService(); + const previousSnapshot = { + version: 2 as const, + teamName: 'mixed-team', + updatedAt: '2026-04-22T12:00:00.000Z', + launchPhase: 'active' as const, + expectedMembers: ['bob'], + members: { + bob: { + name: 'bob', + providerId: 'opencode' as const, + laneId: 'secondary:opencode:bob', + laneKind: 'secondary' as const, + laneOwnerProviderId: 'opencode' as const, + launchState: 'confirmed_alive' as const, + agentToolAccepted: true, + runtimeAlive: true, + bootstrapConfirmed: true, + hardFailure: false, + runtimeRunId: 'opencode-run-1', + runtimeSessionId: 'session-bob-1', + livenessKind: 'confirmed_bootstrap' as const, + lastEvaluatedAt: '2026-04-22T12:00:00.000Z', + }, + }, + summary: { + confirmedCount: 1, + pendingCount: 0, + failedCount: 0, + runtimeAlivePendingCount: 0, + }, + teamLaunchState: 'ready' as const, + }; + const updateLiveness = vi.spyOn(svc as any, 'updateOpenCodeRuntimeMemberLiveness'); + + (svc as any).launchStateStore = { + read: vi.fn(async () => previousSnapshot), + write: vi.fn(async () => {}), + }; + (svc as any).resolveOpenCodeRuntimeLaneId = vi.fn(async () => 'secondary:opencode:bob'); + (svc as any).assertOpenCodeRuntimeEvidenceAccepted = vi.fn(async () => {}); + + await expect( + svc.recordOpenCodeRuntimeBootstrapCheckin({ + teamName: 'mixed-team', + runId: 'opencode-run-1', + memberName: 'bob', + runtimeSessionId: 'session-bob-2', + observedAt: '2026-04-22T12:05:00.000Z', + }) + ).rejects.toMatchObject({ + name: 'RuntimeStaleEvidenceError', + message: expect.stringContaining('opencode_bootstrap_checkin_session_conflict'), + }); + expect(updateLiveness).not.toHaveBeenCalled(); + }); + + it('does not let stale confirmed OpenCode evidence from an older run block a fresh check-in', async () => { + const svc = new TeamProvisioningService(); + const previousSnapshot = { + version: 2 as const, + teamName: 'mixed-team', + updatedAt: '2026-04-22T12:00:00.000Z', + launchPhase: 'active' as const, + expectedMembers: ['bob'], + members: { + bob: { + name: 'bob', + providerId: 'opencode' as const, + laneId: 'secondary:opencode:bob', + laneKind: 'secondary' as const, + laneOwnerProviderId: 'opencode' as const, + launchState: 'confirmed_alive' as const, + agentToolAccepted: true, + runtimeAlive: true, + bootstrapConfirmed: true, + hardFailure: false, + runtimeRunId: 'opencode-run-old', + runtimeSessionId: 'session-bob-old', + livenessKind: 'confirmed_bootstrap' as const, + lastEvaluatedAt: '2026-04-22T12:00:00.000Z', + }, + }, + summary: { + confirmedCount: 1, + pendingCount: 0, + failedCount: 0, + runtimeAlivePendingCount: 0, + }, + teamLaunchState: 'ready' as const, + }; + const updateLiveness = vi + .spyOn(svc as any, 'updateOpenCodeRuntimeMemberLiveness') + .mockResolvedValue(undefined); + + (svc as any).launchStateStore = { + read: vi.fn(async () => previousSnapshot), + write: vi.fn(async () => {}), + }; + (svc as any).resolveOpenCodeRuntimeLaneId = vi.fn(async () => 'secondary:opencode:bob'); + (svc as any).assertOpenCodeRuntimeEvidenceAccepted = vi.fn(async () => {}); + (svc as any).configReader = { + getConfig: vi.fn(async () => ({ + teamName: 'mixed-team', + members: [{ name: 'bob', providerId: 'opencode' }], + })), + }; + (svc as any).membersMetaStore = { + getMembers: vi.fn(async () => []), + }; + + await expect( + svc.recordOpenCodeRuntimeBootstrapCheckin({ + teamName: 'mixed-team', + runId: 'opencode-run-new', + memberName: 'bob', + runtimeSessionId: 'session-bob-new', + observedAt: '2026-04-22T12:05:00.000Z', + }) + ).resolves.toMatchObject({ + ok: true, + state: 'accepted', + runtimeSessionId: 'session-bob-new', + }); + expect(updateLiveness).toHaveBeenCalledWith( + expect.objectContaining({ + runId: 'opencode-run-new', + runtimeSessionId: 'session-bob-new', + }) + ); + }); + + it('rejects OpenCode bootstrap check-ins for removed members before writing runtime evidence', async () => { + const svc = new TeamProvisioningService(); + const updateLiveness = vi.spyOn(svc as any, 'updateOpenCodeRuntimeMemberLiveness'); + + (svc as any).launchStateStore = { + read: vi.fn(async () => null), + write: vi.fn(async () => {}), + }; + (svc as any).resolveOpenCodeRuntimeLaneId = vi.fn(async () => 'secondary:opencode:bob'); + (svc as any).assertOpenCodeRuntimeEvidenceAccepted = vi.fn(async () => {}); + (svc as any).configReader = { + getConfig: vi.fn(async () => ({ + teamName: 'mixed-team', + members: [{ name: 'bob', providerId: 'opencode', removedAt: 123 }], + })), + }; + (svc as any).membersMetaStore = { + getMembers: vi.fn(async () => []), + }; + + await expect( + svc.recordOpenCodeRuntimeBootstrapCheckin({ + teamName: 'mixed-team', + runId: 'opencode-run-1', + memberName: 'bob', + runtimeSessionId: 'session-bob', + }) + ).rejects.toMatchObject({ + name: 'RuntimeStaleEvidenceError', + }); + expect(updateLiveness).not.toHaveBeenCalled(); + }); + it('accepts secondary OpenCode lane evidence using the lane run id instead of the lead run id', async () => { const svc = new TeamProvisioningService(); @@ -9434,6 +9896,264 @@ describe('TeamProvisioningService', () => { expect(run.provisioningOutputParts.join('\n')).toContain('bootstrap confirmed via transcript'); }); + it('clears a live grace-window failure when member transcript later shows successful member_briefing', async () => { + allowConsoleLogs(); + const teamName = 'zz-live-bootstrap-late-transcript-success'; + const leadSessionId = 'lead-session'; + const memberSessionId = 'jack-session'; + const projectPath = '/Users/test/proj'; + const projectId = '-Users-test-proj'; + const acceptedAt = new Date(Date.now() - 170_000).toISOString(); + const successAt = new Date(Date.now() - 5_000).toISOString(); + + writeLaunchConfig(teamName, projectPath, leadSessionId, ['jack']); + + const projectRoot = path.join(tempProjectsBase, projectId); + fs.mkdirSync(projectRoot, { recursive: true }); + fs.writeFileSync( + path.join(projectRoot, `${memberSessionId}.jsonl`), + [ + JSON.stringify({ + timestamp: acceptedAt, + teamName, + agentName: 'jack', + type: 'user', + message: { + role: 'user', + content: `You are bootstrapping into team "${teamName}" as member "jack".`, + }, + }), + JSON.stringify({ + timestamp: successAt, + teamName, + agentName: 'jack', + type: 'user', + message: { + role: 'user', + content: [ + { + type: 'tool_result', + tool_use_id: 'item_1', + content: `Member briefing for jack on team "${teamName}" (${teamName}).\nTask briefing for jack:\nNo actionable tasks.`, + is_error: false, + }, + ], + }, + }), + ].join('\n') + '\n', + 'utf8' + ); + + const svc = new TeamProvisioningService(); + const run = { + runId: 'run-live-late-success-1', + teamName, + startedAt: new Date(Date.now() - 220_000).toISOString(), + request: { + members: [], + }, + expectedMembers: ['jack'], + memberSpawnStatuses: new Map([ + [ + 'jack', + { + status: 'error', + launchState: 'failed_to_start', + error: 'Teammate did not join within the launch grace window.', + updatedAt: new Date(Date.now() - 10_000).toISOString(), + runtimeAlive: false, + livenessSource: undefined, + bootstrapConfirmed: false, + hardFailure: true, + hardFailureReason: 'Teammate did not join within the launch grace window.', + agentToolAccepted: true, + firstSpawnAcceptedAt: acceptedAt, + lastHeartbeatAt: undefined, + }, + ], + ]), + lastMemberSpawnAuditAt: Date.now(), + provisioningOutputParts: [], + activeToolCalls: new Map(), + isLaunch: false, + } as any; + + await (svc as any).maybeAuditMemberSpawnStatuses(run); + + expect(run.memberSpawnStatuses.get('jack')).toMatchObject({ + status: 'online', + launchState: 'confirmed_alive', + bootstrapConfirmed: true, + hardFailure: false, + }); + expect(run.memberSpawnStatuses.get('jack')?.hardFailureReason).toBeUndefined(); + expect(run.provisioningOutputParts.join('\n')).toContain('bootstrap confirmed via transcript'); + }); + + it('does not treat OpenCode member_briefing transcript success as runtime bootstrap evidence', async () => { + allowConsoleLogs(); + const teamName = 'zz-opencode-bootstrap-transcript-not-evidence'; + const leadSessionId = 'lead-session'; + const memberSessionId = 'jack-opencode-session'; + const projectPath = '/Users/test/proj'; + const projectId = '-Users-test-proj'; + const acceptedAt = new Date(Date.now() - 30_000).toISOString(); + const successAt = new Date(Date.now() - 5_000).toISOString(); + + writeLaunchConfig(teamName, projectPath, leadSessionId, ['jack']); + + const projectRoot = path.join(tempProjectsBase, projectId); + fs.mkdirSync(projectRoot, { recursive: true }); + fs.writeFileSync( + path.join(projectRoot, `${memberSessionId}.jsonl`), + [ + JSON.stringify({ + timestamp: acceptedAt, + teamName, + agentName: 'jack', + type: 'user', + message: { + role: 'user', + content: `You are bootstrapping into team "${teamName}" as member "jack".`, + }, + }), + JSON.stringify({ + timestamp: successAt, + teamName, + agentName: 'jack', + type: 'user', + message: { + role: 'user', + content: [ + { + type: 'tool_result', + tool_use_id: 'item_1', + content: `Member briefing for jack on team "${teamName}" (${teamName}).\nTask briefing for jack:\nNo actionable tasks.`, + is_error: false, + }, + ], + }, + }), + ].join('\n') + '\n', + 'utf8' + ); + + const svc = new TeamProvisioningService(); + const run = { + runId: 'run-opencode-transcript-not-evidence', + teamName, + startedAt: new Date(Date.now() - 60_000).toISOString(), + request: { + members: [], + }, + expectedMembers: ['jack'], + mixedSecondaryLanes: [ + { + laneId: 'secondary:opencode:jack', + providerId: 'opencode', + member: { name: 'jack', providerId: 'opencode', model: 'openrouter/qwen/qwen3-coder' }, + runId: 'opencode-run-jack', + state: 'launching', + result: null, + warnings: [], + diagnostics: [], + }, + ], + memberSpawnStatuses: new Map([ + [ + 'jack', + { + status: 'waiting', + launchState: 'runtime_pending_bootstrap', + error: undefined, + updatedAt: acceptedAt, + runtimeAlive: true, + livenessSource: 'process', + bootstrapConfirmed: false, + hardFailure: false, + agentToolAccepted: true, + firstSpawnAcceptedAt: acceptedAt, + lastHeartbeatAt: undefined, + }, + ], + ]), + provisioningOutputParts: [], + activeToolCalls: new Map(), + isLaunch: false, + } as any; + + await (svc as any).reconcileBootstrapTranscriptSuccesses(run); + + expect(run.memberSpawnStatuses.get('jack')).toMatchObject({ + status: 'waiting', + launchState: 'runtime_pending_bootstrap', + runtimeAlive: true, + bootstrapConfirmed: false, + }); + expect(run.provisioningOutputParts.join('\n')).not.toContain( + 'bootstrap confirmed via transcript' + ); + }); + + it('does not copy bootstrap-state success into OpenCode secondary runtime evidence', async () => { + const teamName = 'zz-opencode-bootstrap-state-not-evidence'; + const leadSessionId = 'lead-session'; + const acceptedAt = Date.now() - 30_000; + const observedAt = Date.now() - 5_000; + + writeTeamMeta(teamName, { + providerId: 'codex', + providerBackendId: 'codex-native', + model: 'gpt-5.4', + }); + writeMembersMeta(teamName, [ + { + name: 'jack', + providerId: 'opencode', + model: 'openrouter/qwen/qwen3-coder', + }, + ]); + writeLaunchConfig(teamName, '/Users/test/proj', leadSessionId, ['jack']); + writeBootstrapState( + teamName, + [ + { + name: 'jack', + status: 'bootstrap_confirmed', + lastAttemptAt: acceptedAt, + lastObservedAt: observedAt, + }, + ], + new Date(observedAt - 10_000).toISOString() + ); + writeLaunchState(teamName, leadSessionId, { + jack: { + providerId: 'opencode', + laneId: 'secondary:opencode:jack', + laneKind: 'secondary', + laneOwnerProviderId: 'opencode', + launchState: 'runtime_pending_bootstrap', + agentToolAccepted: true, + runtimeAlive: true, + bootstrapConfirmed: false, + hardFailure: false, + hardFailureReason: undefined, + firstSpawnAcceptedAt: new Date(acceptedAt).toISOString(), + lastRuntimeAliveAt: new Date(observedAt).toISOString(), + lastEvaluatedAt: new Date().toISOString(), + }, + }); + + const svc = new TeamProvisioningService(); + const result = await svc.getMemberSpawnStatuses(teamName); + + expect(result.statuses.jack).toMatchObject({ + launchState: 'runtime_pending_bootstrap', + runtimeAlive: false, + bootstrapConfirmed: false, + }); + }); + it('marks a live teammate bootstrap as confirmed from transcript even when runtime discovery is stale', async () => { allowConsoleLogs(); const teamName = 'zz-live-bootstrap-transcript-success-without-runtime'; @@ -11144,6 +11864,59 @@ describe('TeamProvisioningService', () => { ); }); + it('degrades mixed secondary lanes when lanes.json is active but the lane manifest has no runtime evidence', async () => { + const teamName = 'atlas-hq-empty-lane'; + writeTeamMeta(teamName, { + providerId: 'codex', + providerBackendId: 'codex-native', + model: 'gpt-5.4', + }); + writeMembersMeta(teamName, [ + { + name: 'bob', + providerId: 'opencode', + model: 'openrouter/moonshotai/kimi-k2.6', + }, + { + name: 'jack', + providerId: 'codex', + model: 'gpt-5.4', + }, + ]); + writeLaunchConfig(teamName, '/Users/test/proj', 'lead-session', ['jack']); + writeBootstrapState(teamName, [{ name: 'jack', status: 'registered' }]); + await upsertOpenCodeRuntimeLaneIndexEntry({ + teamsBasePath: tempTeamsBase, + teamName, + laneId: 'secondary:opencode:bob', + state: 'active', + }); + await setOpenCodeRuntimeActiveRunManifest({ + teamsBasePath: tempTeamsBase, + teamName, + laneId: 'secondary:opencode:bob', + runId: 'run-empty-bob', + clock: () => new Date('2026-04-20T10:00:00.000Z'), + }); + + const svc = new TeamProvisioningService(); + const result = await svc.getMemberSpawnStatuses(teamName); + + expect(result.teamLaunchState).toBe('partial_failure'); + expect(result.statuses.bob).toMatchObject({ + status: 'error', + launchState: 'failed_to_start', + error: expect.stringContaining('no committed runtime evidence after launch grace'), + }); + await expect(readOpenCodeRuntimeLaneIndex(tempTeamsBase, teamName)).resolves.toMatchObject({ + lanes: { + 'secondary:opencode:bob': { + state: 'degraded', + }, + }, + }); + }); + it('recovers stale mixed secondary lanes from live OpenCode runtime reconcile before degrading them', async () => { const teamName = 'relay-works-7'; writeTeamMeta(teamName, { @@ -11282,6 +12055,80 @@ describe('TeamProvisioningService', () => { }); }); + it('does not keep an empty active OpenCode lane pending when runtime reconcile has no runtime handle', async () => { + const teamName = 'atlas-hq-empty-lane-nonrecoverable'; + writeTeamMeta(teamName, { + providerId: 'codex', + providerBackendId: 'codex-native', + model: 'gpt-5.4', + }); + writeMembersMeta(teamName, [ + { + name: 'bob', + providerId: 'opencode', + model: 'openrouter/moonshotai/kimi-k2.6', + }, + ]); + writeLaunchConfig(teamName, '/Users/test/proj', 'lead-session', []); + await upsertOpenCodeRuntimeLaneIndexEntry({ + teamsBasePath: tempTeamsBase, + teamName, + laneId: 'secondary:opencode:bob', + state: 'active', + }); + await setOpenCodeRuntimeActiveRunManifest({ + teamsBasePath: tempTeamsBase, + teamName, + laneId: 'secondary:opencode:bob', + runId: 'run-empty-bob', + clock: () => new Date('2026-04-20T10:00:00.000Z'), + }); + + const adapterReconcile = vi.fn(async () => ({ + runId: 'reconcile-run', + teamName, + launchPhase: 'reconciled' as const, + teamLaunchState: 'partial_pending' as const, + members: { + bob: { + memberName: 'bob', + providerId: 'opencode' as const, + launchState: 'runtime_pending_bootstrap' as const, + agentToolAccepted: false, + runtimeAlive: false, + bootstrapConfirmed: false, + hardFailure: false, + livenessKind: 'registered_only' as const, + diagnostics: ['bridge has no runtime session'], + }, + }, + snapshot: null, + warnings: [], + diagnostics: [], + })); + const svc = new TeamProvisioningService(); + svc.setRuntimeAdapterRegistry( + new TeamRuntimeAdapterRegistry([ + { + providerId: 'opencode', + prepare: vi.fn(), + launch: vi.fn(), + reconcile: adapterReconcile, + stop: vi.fn(), + } as any, + ]) + ); + + const result = await svc.getMemberSpawnStatuses(teamName); + + expect(adapterReconcile).toHaveBeenCalledTimes(1); + expect(result.statuses.bob).toMatchObject({ + status: 'error', + launchState: 'failed_to_start', + error: expect.stringContaining('no committed runtime evidence after launch grace'), + }); + }); + it('recovers missing mixed secondary lane index from materialized OpenCode runtime evidence', async () => { const teamName = 'relay-works-missing-lane-recovery'; writeTeamMeta(teamName, {