chore: snapshot dev work sync state
This commit is contained in:
777genius
parent
62691e203d
commit
9fb9e5f66a
38 changed files with 4195 additions and 454 deletions
|
|
@ -8,6 +8,9 @@ const MAX_WAIT_TIMEOUT_MS = 10 * 60 * 1000;
|
|||
const POLL_INTERVAL_MS = 1000;
|
||||
const TEAM_CONTROL_API_STATE_FILE = 'team-control-api.json';
|
||||
const RETRYABLE_CONTROL_ERROR = 'retryableControlError';
|
||||
const BOOTSTRAP_CHECKIN_MAX_ATTEMPTS = 3;
|
||||
const BOOTSTRAP_CHECKIN_ATTEMPT_TIMEOUT_MS = 4000;
|
||||
const BOOTSTRAP_CHECKIN_RETRY_DELAYS_MS = [300, 900];
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
|
@ -67,9 +70,15 @@ function resolveControlBaseUrls(context, flags = {}) {
|
|||
return candidates;
|
||||
}
|
||||
|
||||
function makeRetryableControlError(message, cause) {
|
||||
function makeRetryableControlError(message, cause, metadata = {}) {
|
||||
const error = new Error(message);
|
||||
error[RETRYABLE_CONTROL_ERROR] = true;
|
||||
if (metadata.kind) {
|
||||
error.retryableKind = metadata.kind;
|
||||
}
|
||||
if (metadata.statusCode) {
|
||||
error.statusCode = metadata.statusCode;
|
||||
}
|
||||
if (cause) {
|
||||
error.cause = cause;
|
||||
}
|
||||
|
|
@ -114,7 +123,9 @@ async function requestJson(baseUrl, pathname, options = {}) {
|
|||
: `${response.status} ${response.statusText}`.trim();
|
||||
if (isRetryableStatusCode(response.status)) {
|
||||
throw makeRetryableControlError(
|
||||
`Team control API ${response.status} at ${baseUrl}${pathname}: ${detail || 'request failed'}`
|
||||
`Team control API ${response.status} at ${baseUrl}${pathname}: ${detail || 'request failed'}`,
|
||||
undefined,
|
||||
{ kind: 'status', statusCode: response.status }
|
||||
);
|
||||
}
|
||||
throw new Error(detail || 'Team control API request failed');
|
||||
|
|
@ -122,19 +133,24 @@ async function requestJson(baseUrl, pathname, options = {}) {
|
|||
|
||||
if (payload == null) {
|
||||
throw makeRetryableControlError(
|
||||
`Team control API returned empty or non-JSON response at ${baseUrl}${pathname}`
|
||||
`Team control API returned empty or non-JSON response at ${baseUrl}${pathname}`,
|
||||
undefined,
|
||||
{ kind: 'empty' }
|
||||
);
|
||||
}
|
||||
|
||||
return payload;
|
||||
} catch (error) {
|
||||
if (error && error.name === 'AbortError') {
|
||||
throw makeRetryableControlError(`Timed out calling team control API: ${pathname}`, error);
|
||||
throw makeRetryableControlError(`Timed out calling team control API: ${pathname}`, error, {
|
||||
kind: 'timeout',
|
||||
});
|
||||
}
|
||||
if (error && error.name === 'TypeError') {
|
||||
throw makeRetryableControlError(
|
||||
`Failed to reach team control API at ${baseUrl}: ${error.message || 'fetch failed'}`,
|
||||
error
|
||||
error,
|
||||
{ kind: 'network' }
|
||||
);
|
||||
}
|
||||
throw error;
|
||||
|
|
@ -161,6 +177,54 @@ async function requestJsonWithFallback(baseUrls, pathname, options = {}) {
|
|||
throw lastError || new Error('Team control API request failed');
|
||||
}
|
||||
|
||||
function isBootstrapCheckinRetryableControlError(error) {
|
||||
if (!isRetryableControlError(error)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (error.retryableKind === 'timeout' || error.retryableKind === 'network') {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (error.retryableKind === 'status') {
|
||||
return typeof error.statusCode === 'number' && error.statusCode >= 500;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
async function requestJsonWithBoundedRetry(baseUrls, pathname, options = {}, retryOptions = {}) {
|
||||
const maxAttempts = Math.max(1, retryOptions.maxAttempts || 1);
|
||||
let lastError = null;
|
||||
|
||||
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
||||
try {
|
||||
const result = await requestJsonWithFallback(baseUrls, pathname, options);
|
||||
if (attempt > 1 && result && typeof result === 'object' && !Array.isArray(result)) {
|
||||
return {
|
||||
...result,
|
||||
diagnostics: uniqueNonEmpty([
|
||||
...(Array.isArray(result.diagnostics) ? result.diagnostics : []),
|
||||
'opencode_bootstrap_checkin_retry',
|
||||
]),
|
||||
};
|
||||
}
|
||||
return result;
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
if (attempt >= maxAttempts || !isBootstrapCheckinRetryableControlError(error)) {
|
||||
throw error;
|
||||
}
|
||||
const delayMs = retryOptions.delaysMs?.[attempt - 1] || 0;
|
||||
if (delayMs > 0) {
|
||||
await sleep(delayMs);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError || new Error('Team control API request failed');
|
||||
}
|
||||
|
||||
function buildLaunchRequest(flags = {}) {
|
||||
const cwd = typeof flags.cwd === 'string' ? flags.cwd.trim() : '';
|
||||
if (!cwd) {
|
||||
|
|
@ -412,18 +476,31 @@ async function getRuntimeState(context, flags = {}) {
|
|||
}
|
||||
|
||||
async function runtimeBootstrapCheckin(context, flags = {}) {
|
||||
return postRuntimeTool(
|
||||
context,
|
||||
flags,
|
||||
'bootstrap-checkin',
|
||||
compactRuntimeToolBody(context, flags, [
|
||||
'runId',
|
||||
'memberName',
|
||||
'runtimeSessionId',
|
||||
'observedAt',
|
||||
'diagnostics',
|
||||
'metadata',
|
||||
])
|
||||
const baseUrls = resolveControlBaseUrls(context, flags);
|
||||
const explicitTimeoutMs = flags.waitTimeoutMs || flags['wait-timeout-ms'];
|
||||
const timeoutMs = Math.min(
|
||||
normalizeTimeoutMs(explicitTimeoutMs || BOOTSTRAP_CHECKIN_ATTEMPT_TIMEOUT_MS),
|
||||
BOOTSTRAP_CHECKIN_ATTEMPT_TIMEOUT_MS
|
||||
);
|
||||
return requestJsonWithBoundedRetry(
|
||||
baseUrls,
|
||||
`/api/teams/${encodeURIComponent(context.teamName)}/opencode/runtime/bootstrap-checkin`,
|
||||
{
|
||||
method: 'POST',
|
||||
body: compactRuntimeToolBody(context, flags, [
|
||||
'runId',
|
||||
'memberName',
|
||||
'runtimeSessionId',
|
||||
'observedAt',
|
||||
'diagnostics',
|
||||
'metadata',
|
||||
]),
|
||||
timeoutMs,
|
||||
},
|
||||
{
|
||||
maxAttempts: BOOTSTRAP_CHECKIN_MAX_ATTEMPTS,
|
||||
delaysMs: BOOTSTRAP_CHECKIN_RETRY_DELAYS_MS,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -2251,6 +2251,155 @@ describe('agent-teams-controller API', () => {
|
|||
}
|
||||
});
|
||||
|
||||
it('retries OpenCode bootstrap check-in on retryable control API failures', async () => {
|
||||
const claudeDir = makeClaudeDir();
|
||||
const controller = createController({ teamName: 'my-team', claudeDir });
|
||||
const calls = [];
|
||||
|
||||
const server = await startControlServer(async ({ method, url, body }) => {
|
||||
calls.push({ method, url, body });
|
||||
if (calls.length < 3) {
|
||||
return { statusCode: 500, body: { error: 'temporary bootstrap failure' } };
|
||||
}
|
||||
return { body: { ok: true, state: 'accepted', diagnostics: [] } };
|
||||
});
|
||||
|
||||
try {
|
||||
const result = await controller.runtime.runtimeBootstrapCheckin({
|
||||
controlUrl: server.baseUrl,
|
||||
runId: 'run-oc',
|
||||
memberName: 'bob',
|
||||
runtimeSessionId: 'ses-1',
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
ok: true,
|
||||
state: 'accepted',
|
||||
diagnostics: expect.arrayContaining(['opencode_bootstrap_checkin_retry']),
|
||||
});
|
||||
expect(calls).toHaveLength(3);
|
||||
expect(calls.map((call) => call.body)).toEqual([
|
||||
{
|
||||
teamName: 'my-team',
|
||||
runId: 'run-oc',
|
||||
memberName: 'bob',
|
||||
runtimeSessionId: 'ses-1',
|
||||
},
|
||||
{
|
||||
teamName: 'my-team',
|
||||
runId: 'run-oc',
|
||||
memberName: 'bob',
|
||||
runtimeSessionId: 'ses-1',
|
||||
},
|
||||
{
|
||||
teamName: 'my-team',
|
||||
runId: 'run-oc',
|
||||
memberName: 'bob',
|
||||
runtimeSessionId: 'ses-1',
|
||||
},
|
||||
]);
|
||||
} finally {
|
||||
await server.close();
|
||||
}
|
||||
});
|
||||
|
||||
it('accepts idempotent OpenCode bootstrap check-in after a timed-out committed request', async () => {
|
||||
const claudeDir = makeClaudeDir();
|
||||
const controller = createController({ teamName: 'my-team', claudeDir });
|
||||
const calls = [];
|
||||
let committed = false;
|
||||
|
||||
const server = await startControlServer(async ({ method, url, body }) => {
|
||||
calls.push({ method, url, body });
|
||||
if (!committed) {
|
||||
committed = true;
|
||||
await new Promise((resolve) => setTimeout(resolve, 1200));
|
||||
return { body: { ok: true, state: 'accepted', diagnostics: [] } };
|
||||
}
|
||||
return {
|
||||
body: {
|
||||
ok: true,
|
||||
state: 'accepted',
|
||||
diagnostics: ['opencode_bootstrap_checkin_duplicate_accepted'],
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
try {
|
||||
const result = await controller.runtime.runtimeBootstrapCheckin({
|
||||
controlUrl: server.baseUrl,
|
||||
waitTimeoutMs: 1000,
|
||||
runId: 'run-oc',
|
||||
memberName: 'bob',
|
||||
runtimeSessionId: 'ses-1',
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
ok: true,
|
||||
state: 'accepted',
|
||||
diagnostics: expect.arrayContaining([
|
||||
'opencode_bootstrap_checkin_duplicate_accepted',
|
||||
'opencode_bootstrap_checkin_retry',
|
||||
]),
|
||||
});
|
||||
expect(calls).toHaveLength(2);
|
||||
} finally {
|
||||
await server.close();
|
||||
}
|
||||
});
|
||||
|
||||
it('does not retry OpenCode bootstrap check-in on validation failures', async () => {
|
||||
const claudeDir = makeClaudeDir();
|
||||
const controller = createController({ teamName: 'my-team', claudeDir });
|
||||
const calls = [];
|
||||
|
||||
const server = await startControlServer(async ({ method, url, body }) => {
|
||||
calls.push({ method, url, body });
|
||||
return { statusCode: 400, body: { error: 'invalid bootstrap payload' } };
|
||||
});
|
||||
|
||||
try {
|
||||
await expect(
|
||||
controller.runtime.runtimeBootstrapCheckin({
|
||||
controlUrl: server.baseUrl,
|
||||
runId: 'run-oc',
|
||||
memberName: 'bob',
|
||||
runtimeSessionId: 'ses-1',
|
||||
})
|
||||
).rejects.toThrow('invalid bootstrap payload');
|
||||
expect(calls).toHaveLength(1);
|
||||
} finally {
|
||||
await server.close();
|
||||
}
|
||||
});
|
||||
|
||||
it('fails OpenCode bootstrap check-in clearly after bounded timeout retries', async () => {
|
||||
const claudeDir = makeClaudeDir();
|
||||
const controller = createController({ teamName: 'my-team', claudeDir });
|
||||
const calls = [];
|
||||
|
||||
const server = await startControlServer(async ({ method, url, body }) => {
|
||||
calls.push({ method, url, body });
|
||||
await new Promise((resolve) => setTimeout(resolve, 1200));
|
||||
return { body: { ok: true, state: 'accepted' } };
|
||||
});
|
||||
|
||||
try {
|
||||
await expect(
|
||||
controller.runtime.runtimeBootstrapCheckin({
|
||||
controlUrl: server.baseUrl,
|
||||
waitTimeoutMs: 1000,
|
||||
runId: 'run-oc',
|
||||
memberName: 'bob',
|
||||
runtimeSessionId: 'ses-1',
|
||||
})
|
||||
).rejects.toThrow('Timed out calling team control API');
|
||||
expect(calls).toHaveLength(3);
|
||||
} finally {
|
||||
await server.close();
|
||||
}
|
||||
});
|
||||
|
||||
it('forwards member work sync status and reports to the app validator', async () => {
|
||||
const claudeDir = makeClaudeDir();
|
||||
const controller = createController({ teamName: 'my-team', claudeDir });
|
||||
|
|
|
|||
40
docs/team-management/member-work-sync-debugging.md
Normal file
40
docs/team-management/member-work-sync-debugging.md
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
# Member Work Sync Debugging
|
||||
|
||||
`member-work-sync` stores member-scoped control-plane state under each team member:
|
||||
|
||||
```text
|
||||
~/.claude/teams/<team>/members/<member-key>/.member-work-sync/
|
||||
status.json
|
||||
reports.json
|
||||
outbox.json
|
||||
journal.jsonl
|
||||
```
|
||||
|
||||
`member-key` is the normalized, percent-encoded member name. The canonical name is stored in:
|
||||
|
||||
```text
|
||||
~/.claude/teams/<team>/members/<member-key>/member.meta.json
|
||||
```
|
||||
|
||||
Use the journal for local debugging:
|
||||
|
||||
```bash
|
||||
tail -f ~/.claude/teams/<team>/members/<member-key>/.member-work-sync/journal.jsonl
|
||||
```
|
||||
|
||||
The journal is append-only JSONL and records sync decisions, not raw agent transcripts. Useful events:
|
||||
|
||||
- `reconcile_started`, `agenda_loaded`, `decision_made`, `status_written`
|
||||
- `report_received`, `report_accepted`, `report_rejected`
|
||||
- `nudge_planned`, `nudge_delivered`, `nudge_skipped`, `nudge_retryable`, `nudge_superseded`
|
||||
- `member_busy`, `watchdog_cooldown_active`, `team_inactive`, `legacy_fallback_used`
|
||||
|
||||
Team-level shared/index state remains under:
|
||||
|
||||
```text
|
||||
~/.claude/teams/<team>/.member-work-sync/
|
||||
indexes/
|
||||
report-token-secret.json
|
||||
```
|
||||
|
||||
The indexes are implementation details used to avoid scanning every member directory on the hot path.
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
import type {
|
||||
MemberWorkSyncAuditEvent,
|
||||
MemberWorkSyncAuditEventName,
|
||||
MemberWorkSyncUseCaseDeps,
|
||||
} from './ports';
|
||||
|
||||
export type MemberWorkSyncAuditEventInput = Omit<MemberWorkSyncAuditEvent, 'timestamp'> & {
|
||||
timestamp?: string;
|
||||
};
|
||||
|
||||
export async function appendMemberWorkSyncAudit(
|
||||
deps: Pick<MemberWorkSyncUseCaseDeps, 'auditJournal' | 'clock' | 'logger'>,
|
||||
input: MemberWorkSyncAuditEventInput
|
||||
): Promise<void> {
|
||||
if (!deps.auditJournal) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await deps.auditJournal.append({
|
||||
...input,
|
||||
timestamp: input.timestamp ?? deps.clock.now().toISOString(),
|
||||
});
|
||||
} catch (error) {
|
||||
deps.logger?.warn('member work sync audit event failed', {
|
||||
teamName: input.teamName,
|
||||
memberName: input.memberName,
|
||||
event: input.event,
|
||||
error: String(error),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export function reasonToAuditEvent(reason: string): MemberWorkSyncAuditEventName {
|
||||
if (reason.startsWith('member_busy:')) {
|
||||
return 'member_busy';
|
||||
}
|
||||
if (reason === 'watchdog_cooldown_active') {
|
||||
return 'watchdog_cooldown_active';
|
||||
}
|
||||
if (reason === 'team_inactive') {
|
||||
return 'team_inactive';
|
||||
}
|
||||
return 'nudge_skipped';
|
||||
}
|
||||
|
|
@ -1,5 +1,6 @@
|
|||
import type { MemberWorkSyncOutboxItem } from '../../contracts';
|
||||
import type { MemberWorkSyncUseCaseDeps } from './ports';
|
||||
import { appendMemberWorkSyncAudit, reasonToAuditEvent } from './MemberWorkSyncAudit';
|
||||
import type { MemberWorkSyncAuditEventName, MemberWorkSyncUseCaseDeps } from './ports';
|
||||
|
||||
const MEMBER_WORK_SYNC_MAX_NUDGES_PER_MEMBER_PER_HOUR = 2;
|
||||
const MEMBER_WORK_SYNC_RETRY_BASE_MINUTES = 10;
|
||||
|
|
@ -50,7 +51,9 @@ function nextRetryAt(item: MemberWorkSyncOutboxItem, nowIso: string): string {
|
|||
export class MemberWorkSyncNudgeDispatcher {
|
||||
constructor(private readonly deps: MemberWorkSyncUseCaseDeps) {}
|
||||
|
||||
async dispatchDue(options: MemberWorkSyncNudgeDispatchOptions): Promise<MemberWorkSyncNudgeDispatchSummary> {
|
||||
async dispatchDue(
|
||||
options: MemberWorkSyncNudgeDispatchOptions
|
||||
): Promise<MemberWorkSyncNudgeDispatchSummary> {
|
||||
const outbox = this.deps.outboxStore;
|
||||
const inbox = this.deps.inboxNudge;
|
||||
if (!outbox || !inbox) {
|
||||
|
|
@ -59,7 +62,9 @@ export class MemberWorkSyncNudgeDispatcher {
|
|||
|
||||
const nowIso = this.deps.clock.now().toISOString();
|
||||
const summary = emptySummary();
|
||||
for (const teamName of [...new Set(options.teamNames.map((name) => name.trim()).filter(Boolean))]) {
|
||||
for (const teamName of [
|
||||
...new Set(options.teamNames.map((name) => name.trim()).filter(Boolean)),
|
||||
]) {
|
||||
const claimed = await outbox.claimDue({
|
||||
teamName,
|
||||
claimedBy: options.claimedBy,
|
||||
|
|
@ -97,6 +102,11 @@ export class MemberWorkSyncNudgeDispatcher {
|
|||
nowIso,
|
||||
nextAttemptAt: revalidation.nextAttemptAt ?? nextRetryAt(item, nowIso),
|
||||
});
|
||||
await this.appendDispatchAudit(
|
||||
item,
|
||||
reasonToAuditEvent(revalidation.reason),
|
||||
revalidation.reason
|
||||
);
|
||||
return 'retryable';
|
||||
}
|
||||
await outbox.markSuperseded({
|
||||
|
|
@ -105,6 +115,7 @@ export class MemberWorkSyncNudgeDispatcher {
|
|||
reason: revalidation.reason,
|
||||
nowIso,
|
||||
});
|
||||
await this.appendDispatchAudit(item, 'nudge_superseded', revalidation.reason);
|
||||
return 'superseded';
|
||||
}
|
||||
|
||||
|
|
@ -126,6 +137,7 @@ export class MemberWorkSyncNudgeDispatcher {
|
|||
retryable: false,
|
||||
nowIso,
|
||||
});
|
||||
await this.appendDispatchAudit(item, 'nudge_skipped', 'inbox_payload_conflict');
|
||||
return 'terminal';
|
||||
}
|
||||
await outbox.markDelivered({
|
||||
|
|
@ -135,6 +147,7 @@ export class MemberWorkSyncNudgeDispatcher {
|
|||
deliveredMessageId: inserted.messageId,
|
||||
nowIso,
|
||||
});
|
||||
await this.appendDispatchAudit(item, 'nudge_delivered', 'inbox_inserted');
|
||||
return 'delivered';
|
||||
} catch (error) {
|
||||
await outbox.markFailed({
|
||||
|
|
@ -146,16 +159,33 @@ export class MemberWorkSyncNudgeDispatcher {
|
|||
nowIso,
|
||||
nextAttemptAt: nextRetryAt(item, nowIso),
|
||||
});
|
||||
await this.appendDispatchAudit(item, 'nudge_retryable', String(error));
|
||||
return 'retryable';
|
||||
}
|
||||
}
|
||||
|
||||
private async appendDispatchAudit(
|
||||
item: MemberWorkSyncOutboxItem,
|
||||
event: MemberWorkSyncAuditEventName,
|
||||
reason: string
|
||||
): Promise<void> {
|
||||
await appendMemberWorkSyncAudit(this.deps, {
|
||||
teamName: item.teamName,
|
||||
memberName: item.memberName,
|
||||
event,
|
||||
source: 'nudge_dispatcher',
|
||||
agendaFingerprint: item.agendaFingerprint,
|
||||
reason,
|
||||
taskRefs: item.payload.taskRefs,
|
||||
messagePreview: item.payload.text,
|
||||
});
|
||||
}
|
||||
|
||||
private async revalidate(
|
||||
item: MemberWorkSyncOutboxItem,
|
||||
nowIso: string
|
||||
): Promise<
|
||||
| { ok: true }
|
||||
| { ok: false; reason: string; retryable: boolean; nextAttemptAt?: string }
|
||||
{ ok: true } | { ok: false; reason: string; retryable: boolean; nextAttemptAt?: string }
|
||||
> {
|
||||
if (this.deps.lifecycle && !(await this.deps.lifecycle.isTeamActive(item.teamName))) {
|
||||
return { ok: false, reason: 'team_inactive', retryable: false };
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
import { buildMemberWorkSyncOutboxEnsureInput } from '../domain';
|
||||
|
||||
import { appendMemberWorkSyncAudit } from './MemberWorkSyncAudit';
|
||||
|
||||
import type { MemberWorkSyncStatus } from '../../contracts';
|
||||
import type { MemberWorkSyncUseCaseDeps } from './ports';
|
||||
|
||||
|
|
@ -37,6 +39,7 @@ export class MemberWorkSyncNudgeOutboxPlanner {
|
|||
|
||||
const metrics = await this.deps.statusStore.readTeamMetrics(status.teamName);
|
||||
if (metrics.phase2Readiness.state !== 'shadow_ready') {
|
||||
await this.appendPlanAudit(status, { planned: false, code: 'phase2_not_ready' });
|
||||
return { planned: false, code: 'phase2_not_ready' };
|
||||
}
|
||||
|
||||
|
|
@ -49,9 +52,34 @@ export class MemberWorkSyncNudgeOutboxPlanner {
|
|||
existingPayloadHash: result.existingPayloadHash,
|
||||
requestedPayloadHash: result.requestedPayloadHash,
|
||||
});
|
||||
await this.appendPlanAudit(status, { planned: false, code: 'payload_conflict' });
|
||||
return { planned: false, code: 'payload_conflict' };
|
||||
}
|
||||
|
||||
return { planned: true, code: result.outcome };
|
||||
const planResult = { planned: true, code: result.outcome } as const;
|
||||
await this.appendPlanAudit(status, planResult);
|
||||
return planResult;
|
||||
}
|
||||
|
||||
private async appendPlanAudit(
|
||||
status: MemberWorkSyncStatus,
|
||||
result: MemberWorkSyncNudgeOutboxPlanResult
|
||||
): Promise<void> {
|
||||
await appendMemberWorkSyncAudit(this.deps, {
|
||||
teamName: status.teamName,
|
||||
memberName: status.memberName,
|
||||
event: result.planned ? 'nudge_planned' : 'nudge_skipped',
|
||||
source: 'nudge_planner',
|
||||
agendaFingerprint: status.agenda.fingerprint,
|
||||
state: status.state,
|
||||
actionableCount: status.agenda.items.length,
|
||||
reason: result.code,
|
||||
...(status.providerId ? { providerId: status.providerId } : {}),
|
||||
taskRefs: status.agenda.items.map((item) => ({
|
||||
taskId: item.taskId,
|
||||
displayId: item.displayId,
|
||||
teamName: status.teamName,
|
||||
})),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import {
|
|||
formatAgendaFingerprint,
|
||||
} from '../domain';
|
||||
|
||||
import { appendMemberWorkSyncAudit } from './MemberWorkSyncAudit';
|
||||
import { MemberWorkSyncNudgeOutboxPlanner } from './MemberWorkSyncNudgeOutboxPlanner';
|
||||
|
||||
import type { MemberWorkSyncStatus, MemberWorkSyncStatusRequest } from '../../contracts';
|
||||
|
|
@ -46,8 +47,25 @@ export class MemberWorkSyncReconciler {
|
|||
request: MemberWorkSyncStatusRequest,
|
||||
context: MemberWorkSyncReconcileContext = {}
|
||||
): Promise<MemberWorkSyncStatus> {
|
||||
await appendMemberWorkSyncAudit(this.deps, {
|
||||
teamName: request.teamName,
|
||||
memberName: request.memberName,
|
||||
event: 'reconcile_started',
|
||||
source: 'reconciler',
|
||||
...(context.triggerReasons?.length ? { triggerReasons: context.triggerReasons } : {}),
|
||||
});
|
||||
const source = await this.deps.agendaSource.loadAgenda(request);
|
||||
const agenda = finalizeMemberWorkSyncAgenda(this.deps, source);
|
||||
await appendMemberWorkSyncAudit(this.deps, {
|
||||
teamName: agenda.teamName,
|
||||
memberName: agenda.memberName,
|
||||
event: 'agenda_loaded',
|
||||
source: 'reconciler',
|
||||
agendaFingerprint: agenda.fingerprint,
|
||||
actionableCount: agenda.items.length,
|
||||
...(source.providerId ? { providerId: source.providerId } : {}),
|
||||
diagnostics: agenda.diagnostics,
|
||||
});
|
||||
const previous = await this.deps.statusStore.read(request);
|
||||
const nowIso = this.deps.clock.now().toISOString();
|
||||
const teamActive = this.deps.lifecycle
|
||||
|
|
@ -59,6 +77,17 @@ export class MemberWorkSyncReconciler {
|
|||
nowIso,
|
||||
inactive: source.inactive || !teamActive,
|
||||
});
|
||||
await appendMemberWorkSyncAudit(this.deps, {
|
||||
teamName: agenda.teamName,
|
||||
memberName: agenda.memberName,
|
||||
event: source.inactive || !teamActive ? 'team_inactive' : 'decision_made',
|
||||
source: 'reconciler',
|
||||
agendaFingerprint: agenda.fingerprint,
|
||||
state: decision.state,
|
||||
actionableCount: agenda.items.length,
|
||||
...(source.providerId ? { providerId: source.providerId } : {}),
|
||||
diagnostics: decision.diagnostics,
|
||||
});
|
||||
|
||||
const status = await attachMemberWorkSyncReportToken(this.deps, {
|
||||
teamName: agenda.teamName,
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import { validateMemberWorkSyncReport } from '../domain';
|
||||
|
||||
import { appendMemberWorkSyncAudit } from './MemberWorkSyncAudit';
|
||||
import {
|
||||
attachMemberWorkSyncReportToken,
|
||||
finalizeMemberWorkSyncAgenda,
|
||||
|
|
@ -22,6 +23,22 @@ export class MemberWorkSyncReporter {
|
|||
}
|
||||
|
||||
async execute(request: MemberWorkSyncReportRequest): Promise<MemberWorkSyncReportResult> {
|
||||
await appendMemberWorkSyncAudit(this.deps, {
|
||||
teamName: request.teamName,
|
||||
memberName: request.memberName,
|
||||
event: 'report_received',
|
||||
source: 'reporter',
|
||||
agendaFingerprint: request.agendaFingerprint,
|
||||
state: request.state,
|
||||
...(request.taskIds?.length
|
||||
? {
|
||||
taskRefs: request.taskIds.map((taskId) => ({
|
||||
taskId,
|
||||
teamName: request.teamName,
|
||||
})),
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
const source = await this.deps.agendaSource.loadAgenda(request);
|
||||
const agenda = finalizeMemberWorkSyncAgenda(this.deps, source);
|
||||
const nowIso = this.deps.clock.now().toISOString();
|
||||
|
|
@ -105,6 +122,16 @@ export class MemberWorkSyncReporter {
|
|||
});
|
||||
|
||||
await this.deps.statusStore.write(status);
|
||||
await appendMemberWorkSyncAudit(this.deps, {
|
||||
teamName: status.teamName,
|
||||
memberName: status.memberName,
|
||||
event: 'report_accepted',
|
||||
source: 'reporter',
|
||||
agendaFingerprint: agenda.fingerprint,
|
||||
state: status.state,
|
||||
actionableCount: agenda.items.length,
|
||||
...(source.providerId ? { providerId: source.providerId } : {}),
|
||||
});
|
||||
return {
|
||||
accepted: true,
|
||||
code: 'accepted',
|
||||
|
|
@ -135,6 +162,17 @@ export class MemberWorkSyncReporter {
|
|||
diagnostics: [...status.diagnostics, `report_rejected:${rejectionCode}`],
|
||||
};
|
||||
await this.deps.statusStore.write(rejectedStatus);
|
||||
await appendMemberWorkSyncAudit(this.deps, {
|
||||
teamName: status.teamName,
|
||||
memberName: status.memberName,
|
||||
event: 'report_rejected',
|
||||
source: 'reporter',
|
||||
agendaFingerprint: request.agendaFingerprint,
|
||||
state: request.state,
|
||||
actionableCount: status.agenda.items.length,
|
||||
reason: rejectionCode,
|
||||
...(status.providerId ? { providerId: status.providerId } : {}),
|
||||
});
|
||||
return rejectedStatus;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,9 @@
|
|||
import type { MemberWorkSyncClockPort, MemberWorkSyncLoggerPort } from './ports';
|
||||
import { appendMemberWorkSyncAudit } from './MemberWorkSyncAudit';
|
||||
import type {
|
||||
MemberWorkSyncAuditJournalPort,
|
||||
MemberWorkSyncClockPort,
|
||||
MemberWorkSyncLoggerPort,
|
||||
} from './ports';
|
||||
import type { RuntimeTurnSettledEvent } from '../domain';
|
||||
import type {
|
||||
RuntimeTurnSettledEventStorePort,
|
||||
|
|
@ -13,6 +18,7 @@ export interface RuntimeTurnSettledIngestorDeps {
|
|||
targetResolver: RuntimeTurnSettledTargetResolverPort;
|
||||
reconcileQueue: RuntimeTurnSettledReconcileQueuePort;
|
||||
clock: MemberWorkSyncClockPort;
|
||||
auditJournal?: MemberWorkSyncAuditJournalPort;
|
||||
logger?: MemberWorkSyncLoggerPort;
|
||||
}
|
||||
|
||||
|
|
@ -78,6 +84,20 @@ export class RuntimeTurnSettledIngestor {
|
|||
continue;
|
||||
}
|
||||
|
||||
if (normalized.event.teamName && normalized.event.memberName) {
|
||||
await appendMemberWorkSyncAudit(this.deps, {
|
||||
teamName: normalized.event.teamName,
|
||||
memberName: normalized.event.memberName,
|
||||
event: 'turn_settled_claimed',
|
||||
source: 'runtime_turn_settled_ingestor',
|
||||
reason: normalized.event.provider,
|
||||
metadata: {
|
||||
sourceId: normalized.event.sourceId,
|
||||
provider: normalized.event.provider,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
const ignoredReason = getIgnoredReason(normalized.event);
|
||||
if (ignoredReason) {
|
||||
summary.ignored += 1;
|
||||
|
|
@ -87,6 +107,19 @@ export class RuntimeTurnSettledIngestor {
|
|||
reason: ignoredReason,
|
||||
processedAt,
|
||||
});
|
||||
if (normalized.event.teamName && normalized.event.memberName) {
|
||||
await appendMemberWorkSyncAudit(this.deps, {
|
||||
teamName: normalized.event.teamName,
|
||||
memberName: normalized.event.memberName,
|
||||
event: 'turn_settled_ignored',
|
||||
source: 'runtime_turn_settled_ingestor',
|
||||
reason: ignoredReason,
|
||||
metadata: {
|
||||
sourceId: normalized.event.sourceId,
|
||||
provider: normalized.event.provider,
|
||||
},
|
||||
});
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
|
|
@ -99,6 +132,19 @@ export class RuntimeTurnSettledIngestor {
|
|||
reason: resolution.reason,
|
||||
processedAt,
|
||||
});
|
||||
if (normalized.event.teamName && normalized.event.memberName) {
|
||||
await appendMemberWorkSyncAudit(this.deps, {
|
||||
teamName: normalized.event.teamName,
|
||||
memberName: normalized.event.memberName,
|
||||
event: 'turn_settled_unresolved',
|
||||
source: 'runtime_turn_settled_ingestor',
|
||||
reason: resolution.reason,
|
||||
metadata: {
|
||||
sourceId: normalized.event.sourceId,
|
||||
provider: normalized.event.provider,
|
||||
},
|
||||
});
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
|
|
@ -115,6 +161,17 @@ export class RuntimeTurnSettledIngestor {
|
|||
outcome: 'enqueued',
|
||||
processedAt,
|
||||
});
|
||||
await appendMemberWorkSyncAudit(this.deps, {
|
||||
teamName: resolution.teamName,
|
||||
memberName: resolution.memberName,
|
||||
event: 'turn_settled_resolved',
|
||||
source: 'runtime_turn_settled_ingestor',
|
||||
reason: normalized.event.provider,
|
||||
metadata: {
|
||||
sourceId: normalized.event.sourceId,
|
||||
provider: normalized.event.provider,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
summary.failed += 1;
|
||||
this.deps.logger?.warn('runtime turn settled ingest failed', {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
export * from './MemberWorkSyncDiagnosticsReader';
|
||||
export * from './MemberWorkSyncMetricsReader';
|
||||
export * from './MemberWorkSyncAudit';
|
||||
export * from './MemberWorkSyncNudgeDispatcher';
|
||||
export * from './MemberWorkSyncNudgeOutboxPlanner';
|
||||
export * from './MemberWorkSyncPendingReportIntentReplayer';
|
||||
|
|
|
|||
|
|
@ -64,6 +64,55 @@ export interface MemberWorkSyncLoggerPort {
|
|||
error(message: string, metadata?: Record<string, unknown>): void;
|
||||
}
|
||||
|
||||
export type MemberWorkSyncAuditEventName =
|
||||
| 'turn_settled_claimed'
|
||||
| 'turn_settled_resolved'
|
||||
| 'turn_settled_unresolved'
|
||||
| 'turn_settled_ignored'
|
||||
| 'queue_enqueued'
|
||||
| 'queue_coalesced'
|
||||
| 'queue_reconciled'
|
||||
| 'queue_dropped'
|
||||
| 'reconcile_started'
|
||||
| 'agenda_loaded'
|
||||
| 'decision_made'
|
||||
| 'status_written'
|
||||
| 'report_received'
|
||||
| 'report_accepted'
|
||||
| 'report_rejected'
|
||||
| 'nudge_planned'
|
||||
| 'nudge_delivered'
|
||||
| 'nudge_skipped'
|
||||
| 'nudge_retryable'
|
||||
| 'nudge_superseded'
|
||||
| 'watchdog_cooldown_active'
|
||||
| 'member_busy'
|
||||
| 'team_inactive'
|
||||
| 'index_repaired'
|
||||
| 'legacy_fallback_used';
|
||||
|
||||
export interface MemberWorkSyncAuditEvent {
|
||||
timestamp: string;
|
||||
teamName: string;
|
||||
memberName: string;
|
||||
event: MemberWorkSyncAuditEventName;
|
||||
source: string;
|
||||
agendaFingerprint?: string;
|
||||
state?: string;
|
||||
actionableCount?: number;
|
||||
reason?: string;
|
||||
triggerReasons?: string[];
|
||||
providerId?: string;
|
||||
taskRefs?: { taskId: string; displayId?: string; teamName?: string }[];
|
||||
diagnostics?: string[];
|
||||
messagePreview?: string;
|
||||
metadata?: Record<string, string | number | boolean | null>;
|
||||
}
|
||||
|
||||
export interface MemberWorkSyncAuditJournalPort {
|
||||
append(event: MemberWorkSyncAuditEvent): Promise<void>;
|
||||
}
|
||||
|
||||
export interface MemberWorkSyncAgendaSourceResult {
|
||||
agenda: Omit<MemberWorkSyncAgenda, 'fingerprint'>;
|
||||
activeMemberNames: string[];
|
||||
|
|
@ -143,6 +192,7 @@ export interface MemberWorkSyncUseCaseDeps {
|
|||
watchdogCooldown?: MemberWorkSyncWatchdogCooldownPort;
|
||||
busySignal?: MemberWorkSyncBusySignalPort;
|
||||
reportToken?: MemberWorkSyncReportTokenPort;
|
||||
auditJournal?: MemberWorkSyncAuditJournalPort;
|
||||
lifecycle?: MemberWorkSyncLifecyclePort;
|
||||
logger?: MemberWorkSyncLoggerPort;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,6 +14,10 @@ interface MemberWorkSyncRosterSource {
|
|||
loadActiveMemberNames(teamName: string): Promise<string[]>;
|
||||
}
|
||||
|
||||
interface MemberWorkSyncMemberStorageMaterializer {
|
||||
materializeMember(teamName: string, memberName: string): Promise<void>;
|
||||
}
|
||||
|
||||
const TEAM_WIDE_REASONS: Partial<Record<TeamChangeEvent['type'], MemberWorkSyncTriggerReason>> = {
|
||||
config: 'config_changed',
|
||||
task: 'task_changed',
|
||||
|
|
@ -58,7 +62,8 @@ function parseMemberTurnSettled(detail: string | undefined): MemberTurnSettledEv
|
|||
export class MemberWorkSyncTeamChangeRouter {
|
||||
constructor(
|
||||
private readonly rosterSource: MemberWorkSyncRosterSource,
|
||||
private readonly queue: MemberWorkSyncEventQueue
|
||||
private readonly queue: MemberWorkSyncEventQueue,
|
||||
private readonly materializer?: MemberWorkSyncMemberStorageMaterializer
|
||||
) {}
|
||||
|
||||
async enqueueStartupScan(teamNames: string[]): Promise<void> {
|
||||
|
|
@ -137,6 +142,13 @@ export class MemberWorkSyncTeamChangeRouter {
|
|||
runAfterMs?: number
|
||||
): Promise<void> {
|
||||
const activeMembers = await this.rosterSource.loadActiveMemberNames(teamName);
|
||||
if (this.materializer) {
|
||||
await Promise.allSettled(
|
||||
activeMembers.map((memberName) =>
|
||||
this.materializer?.materializeMember(teamName, memberName)
|
||||
)
|
||||
);
|
||||
}
|
||||
for (const memberName of activeMembers) {
|
||||
this.queue.enqueue({ teamName, memberName, triggerReason, runAfterMs });
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ import { ClaudeStopHookPayloadNormalizer } from '../infrastructure/ClaudeStopHoo
|
|||
import { CodexNativeTurnSettledPayloadNormalizer } from '../infrastructure/CodexNativeTurnSettledPayloadNormalizer';
|
||||
import { CompositeRuntimeTurnSettledPayloadNormalizer } from '../infrastructure/CompositeRuntimeTurnSettledPayloadNormalizer';
|
||||
import { FileRuntimeTurnSettledEventStore } from '../infrastructure/FileRuntimeTurnSettledEventStore';
|
||||
import { FileMemberWorkSyncAuditJournal } from '../infrastructure/FileMemberWorkSyncAuditJournal';
|
||||
import { HmacMemberWorkSyncReportTokenAdapter } from '../infrastructure/HmacMemberWorkSyncReportTokenAdapter';
|
||||
import { JsonMemberWorkSyncStore } from '../infrastructure/JsonMemberWorkSyncStore';
|
||||
import {
|
||||
|
|
@ -132,7 +133,11 @@ export function createMemberWorkSyncFeature(deps: {
|
|||
clock,
|
||||
});
|
||||
const storePaths = new MemberWorkSyncStorePaths(deps.teamsBasePath);
|
||||
const store = new JsonMemberWorkSyncStore(storePaths);
|
||||
const auditJournal = new FileMemberWorkSyncAuditJournal(storePaths, deps.logger);
|
||||
const store = new JsonMemberWorkSyncStore(storePaths, {
|
||||
auditJournal,
|
||||
logger: deps.logger,
|
||||
});
|
||||
const runtimeTurnSettledSpool = new RuntimeTurnSettledSpoolInitializer(deps.teamsBasePath);
|
||||
const runtimeTurnSettledStore = new FileRuntimeTurnSettledEventStore({
|
||||
paths: runtimeTurnSettledSpool.getPaths(),
|
||||
|
|
@ -165,6 +170,7 @@ export function createMemberWorkSyncFeature(deps: {
|
|||
watchdogCooldown,
|
||||
busySignal,
|
||||
reportToken,
|
||||
auditJournal,
|
||||
...(deps.isTeamActive ? { lifecycle: { isTeamActive: deps.isTeamActive } } : {}),
|
||||
logger: deps.logger,
|
||||
};
|
||||
|
|
@ -186,9 +192,13 @@ export function createMemberWorkSyncFeature(deps: {
|
|||
},
|
||||
isTeamActive: deps.isTeamActive ?? (() => true),
|
||||
...(deps.queueQuietWindowMs != null ? { quietWindowMs: deps.queueQuietWindowMs } : {}),
|
||||
auditJournal,
|
||||
logger: deps.logger,
|
||||
});
|
||||
const router = new MemberWorkSyncTeamChangeRouter(agendaSource, queue);
|
||||
const router = new MemberWorkSyncTeamChangeRouter(agendaSource, queue, {
|
||||
materializeMember: (teamName, memberName) =>
|
||||
storePaths.ensureMemberWorkSyncDir(teamName, memberName),
|
||||
});
|
||||
const runtimeTurnSettledIngestor = new RuntimeTurnSettledIngestor({
|
||||
eventStore: runtimeTurnSettledStore,
|
||||
normalizer: runtimeTurnSettledNormalizer,
|
||||
|
|
@ -207,6 +217,7 @@ export function createMemberWorkSyncFeature(deps: {
|
|||
},
|
||||
},
|
||||
clock,
|
||||
auditJournal,
|
||||
logger: deps.logger,
|
||||
});
|
||||
const runtimeTurnSettledDrainScheduler = new RuntimeTurnSettledDrainScheduler({
|
||||
|
|
|
|||
|
|
@ -0,0 +1,157 @@
|
|||
import { appendFile, mkdir, rename, rm, stat } from 'fs/promises';
|
||||
import { dirname } from 'path';
|
||||
|
||||
import { withFileLock } from '@main/services/team/fileLock';
|
||||
|
||||
import type {
|
||||
MemberWorkSyncAuditEvent,
|
||||
MemberWorkSyncAuditJournalPort,
|
||||
MemberWorkSyncLoggerPort,
|
||||
} from '../../core/application';
|
||||
import type { MemberWorkSyncStorePaths } from './MemberWorkSyncStorePaths';
|
||||
|
||||
const DEFAULT_MAX_BYTES = 5 * 1024 * 1024;
|
||||
const DEFAULT_ROTATED_FILE_COUNT = 5;
|
||||
const MAX_PREVIEW_CHARS = 240;
|
||||
const MAX_DIAGNOSTICS = 20;
|
||||
const MAX_TRIGGER_REASONS = 20;
|
||||
const MAX_TASK_REFS = 20;
|
||||
const MAX_SHORT_FIELD_CHARS = 240;
|
||||
|
||||
interface PersistedAuditEvent extends MemberWorkSyncAuditEvent {
|
||||
schemaVersion: 1;
|
||||
}
|
||||
|
||||
export interface FileMemberWorkSyncAuditJournalOptions {
|
||||
maxBytes?: number;
|
||||
rotatedFileCount?: number;
|
||||
}
|
||||
|
||||
function truncateText(value: string, maxChars: number): string {
|
||||
return value.length <= maxChars ? value : `${value.slice(0, maxChars)}...`;
|
||||
}
|
||||
|
||||
function sanitizeMetadata(
|
||||
metadata: MemberWorkSyncAuditEvent['metadata']
|
||||
): MemberWorkSyncAuditEvent['metadata'] {
|
||||
if (!metadata) {
|
||||
return undefined;
|
||||
}
|
||||
const sanitized = Object.create(null) as NonNullable<MemberWorkSyncAuditEvent['metadata']>;
|
||||
for (const [key, value] of Object.entries(metadata)) {
|
||||
sanitized[truncateText(key, MAX_SHORT_FIELD_CHARS)] =
|
||||
typeof value === 'string' ? truncateText(value, MAX_SHORT_FIELD_CHARS) : value;
|
||||
}
|
||||
return sanitized;
|
||||
}
|
||||
|
||||
function sanitizeTaskRefs(
|
||||
taskRefs: MemberWorkSyncAuditEvent['taskRefs']
|
||||
): MemberWorkSyncAuditEvent['taskRefs'] {
|
||||
return taskRefs?.slice(0, MAX_TASK_REFS).map((taskRef) => ({
|
||||
taskId: truncateText(taskRef.taskId, MAX_SHORT_FIELD_CHARS),
|
||||
...(taskRef.displayId
|
||||
? { displayId: truncateText(taskRef.displayId, MAX_SHORT_FIELD_CHARS) }
|
||||
: {}),
|
||||
...(taskRef.teamName
|
||||
? { teamName: truncateText(taskRef.teamName, MAX_SHORT_FIELD_CHARS) }
|
||||
: {}),
|
||||
}));
|
||||
}
|
||||
|
||||
function sanitizeEvent(event: MemberWorkSyncAuditEvent): PersistedAuditEvent {
|
||||
return {
|
||||
...event,
|
||||
schemaVersion: 1,
|
||||
source: truncateText(event.source, MAX_SHORT_FIELD_CHARS),
|
||||
...(event.reason ? { reason: truncateText(event.reason, MAX_SHORT_FIELD_CHARS) } : {}),
|
||||
...(event.providerId
|
||||
? { providerId: truncateText(event.providerId, MAX_SHORT_FIELD_CHARS) }
|
||||
: {}),
|
||||
...(event.state ? { state: truncateText(event.state, MAX_SHORT_FIELD_CHARS) } : {}),
|
||||
...(event.agendaFingerprint
|
||||
? { agendaFingerprint: truncateText(event.agendaFingerprint, MAX_SHORT_FIELD_CHARS) }
|
||||
: {}),
|
||||
...(typeof event.messagePreview === 'string'
|
||||
? { messagePreview: truncateText(event.messagePreview, MAX_PREVIEW_CHARS) }
|
||||
: {}),
|
||||
...(event.diagnostics
|
||||
? {
|
||||
diagnostics: event.diagnostics
|
||||
.slice(0, MAX_DIAGNOSTICS)
|
||||
.map((diagnostic) => truncateText(diagnostic, MAX_SHORT_FIELD_CHARS)),
|
||||
}
|
||||
: {}),
|
||||
...(event.triggerReasons
|
||||
? {
|
||||
triggerReasons: event.triggerReasons
|
||||
.slice(0, MAX_TRIGGER_REASONS)
|
||||
.map((reason) => truncateText(reason, MAX_SHORT_FIELD_CHARS)),
|
||||
}
|
||||
: {}),
|
||||
...(event.taskRefs ? { taskRefs: sanitizeTaskRefs(event.taskRefs) } : {}),
|
||||
...(event.metadata ? { metadata: sanitizeMetadata(event.metadata) } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function rotatedPath(filePath: string, index: number): string {
|
||||
return `${filePath}.${index}`;
|
||||
}
|
||||
|
||||
async function rotateIfNeeded(
|
||||
filePath: string,
|
||||
maxBytes: number,
|
||||
rotatedFileCount: number
|
||||
): Promise<void> {
|
||||
const current = await stat(filePath).catch(() => null);
|
||||
if (!current?.isFile() || current.size < maxBytes) {
|
||||
return;
|
||||
}
|
||||
|
||||
await rm(rotatedPath(filePath, rotatedFileCount), { force: true }).catch(() => undefined);
|
||||
for (let index = rotatedFileCount - 1; index >= 1; index -= 1) {
|
||||
await rename(rotatedPath(filePath, index), rotatedPath(filePath, index + 1)).catch(
|
||||
() => undefined
|
||||
);
|
||||
}
|
||||
await rename(filePath, rotatedPath(filePath, 1)).catch(() => undefined);
|
||||
}
|
||||
|
||||
export class NoopMemberWorkSyncAuditJournal implements MemberWorkSyncAuditJournalPort {
|
||||
async append(): Promise<void> {
|
||||
// Intentionally empty.
|
||||
}
|
||||
}
|
||||
|
||||
export class FileMemberWorkSyncAuditJournal implements MemberWorkSyncAuditJournalPort {
|
||||
private readonly maxBytes: number;
|
||||
private readonly rotatedFileCount: number;
|
||||
|
||||
constructor(
|
||||
private readonly paths: MemberWorkSyncStorePaths,
|
||||
private readonly logger?: MemberWorkSyncLoggerPort,
|
||||
options: FileMemberWorkSyncAuditJournalOptions = {}
|
||||
) {
|
||||
this.maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES;
|
||||
this.rotatedFileCount = options.rotatedFileCount ?? DEFAULT_ROTATED_FILE_COUNT;
|
||||
}
|
||||
|
||||
async append(event: MemberWorkSyncAuditEvent): Promise<void> {
|
||||
try {
|
||||
await this.paths.ensureMemberWorkSyncDir(event.teamName, event.memberName);
|
||||
const filePath = this.paths.getMemberJournalPath(event.teamName, event.memberName);
|
||||
await mkdir(dirname(filePath), { recursive: true });
|
||||
await withFileLock(filePath, async () => {
|
||||
await rotateIfNeeded(filePath, this.maxBytes, this.rotatedFileCount);
|
||||
await appendFile(filePath, `${JSON.stringify(sanitizeEvent(event))}\n`, 'utf8');
|
||||
});
|
||||
} catch (error) {
|
||||
this.logger?.warn('member work sync audit journal append failed', {
|
||||
teamName: event.teamName,
|
||||
memberName: event.memberName,
|
||||
event: event.event,
|
||||
error: String(error),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -1,4 +1,8 @@
|
|||
import type { MemberWorkSyncLoggerPort } from '../../core/application';
|
||||
import type {
|
||||
MemberWorkSyncAuditEvent,
|
||||
MemberWorkSyncAuditJournalPort,
|
||||
MemberWorkSyncLoggerPort,
|
||||
} from '../../core/application';
|
||||
import type { MemberWorkSyncReconcileContext } from '../../core/application/MemberWorkSyncReconciler';
|
||||
|
||||
export type MemberWorkSyncTriggerReason =
|
||||
|
|
@ -43,6 +47,8 @@ export interface MemberWorkSyncEventQueueDeps {
|
|||
quietWindowMs?: number;
|
||||
concurrency?: number;
|
||||
now?: () => number;
|
||||
nowIso?: () => string;
|
||||
auditJournal?: MemberWorkSyncAuditJournalPort;
|
||||
logger?: MemberWorkSyncLoggerPort;
|
||||
}
|
||||
|
||||
|
|
@ -61,6 +67,7 @@ export class MemberWorkSyncEventQueue {
|
|||
private readonly quietWindowMs: number;
|
||||
private readonly concurrency: number;
|
||||
private readonly now: () => number;
|
||||
private readonly nowIso: () => string;
|
||||
private timer: ReturnType<typeof setTimeout> | null = null;
|
||||
private stopped = false;
|
||||
private counters = {
|
||||
|
|
@ -75,6 +82,7 @@ export class MemberWorkSyncEventQueue {
|
|||
this.quietWindowMs = deps.quietWindowMs ?? 90_000;
|
||||
this.concurrency = Math.max(1, deps.concurrency ?? 2);
|
||||
this.now = deps.now ?? Date.now;
|
||||
this.nowIso = deps.nowIso ?? (() => new Date().toISOString());
|
||||
}
|
||||
|
||||
enqueue(input: {
|
||||
|
|
@ -87,19 +95,27 @@ export class MemberWorkSyncEventQueue {
|
|||
return;
|
||||
}
|
||||
|
||||
const teamName = input.teamName.trim();
|
||||
const memberName = input.memberName.trim();
|
||||
if (!input.teamName.trim() || !memberName) {
|
||||
if (!teamName || !memberName) {
|
||||
this.counters.dropped += 1;
|
||||
return;
|
||||
}
|
||||
|
||||
const key = keyOf(input.teamName, memberName);
|
||||
const key = keyOf(teamName, memberName);
|
||||
const runAt = this.now() + (input.runAfterMs ?? this.quietWindowMs);
|
||||
const running = this.running.get(key);
|
||||
if (running) {
|
||||
running.rerunRequested = true;
|
||||
running.triggerReasons.add(input.triggerReason);
|
||||
this.counters.coalesced += 1;
|
||||
this.appendAudit({
|
||||
teamName,
|
||||
memberName,
|
||||
event: 'queue_coalesced',
|
||||
source: 'event_queue',
|
||||
reason: input.triggerReason,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -108,17 +124,31 @@ export class MemberWorkSyncEventQueue {
|
|||
existing.triggerReasons.add(input.triggerReason);
|
||||
existing.runAt = Math.max(existing.runAt, runAt);
|
||||
this.counters.coalesced += 1;
|
||||
this.appendAudit({
|
||||
teamName,
|
||||
memberName,
|
||||
event: 'queue_coalesced',
|
||||
source: 'event_queue',
|
||||
reason: input.triggerReason,
|
||||
});
|
||||
this.schedule();
|
||||
return;
|
||||
}
|
||||
|
||||
this.items.set(key, {
|
||||
teamName: input.teamName,
|
||||
teamName,
|
||||
memberName,
|
||||
runAt,
|
||||
triggerReasons: new Set([input.triggerReason]),
|
||||
});
|
||||
this.counters.enqueued += 1;
|
||||
this.appendAudit({
|
||||
teamName,
|
||||
memberName,
|
||||
event: 'queue_enqueued',
|
||||
source: 'event_queue',
|
||||
reason: input.triggerReason,
|
||||
});
|
||||
this.schedule();
|
||||
}
|
||||
|
||||
|
|
@ -231,6 +261,13 @@ export class MemberWorkSyncEventQueue {
|
|||
private async executeItem(_key: string, item: QueueItem, running: RunningItem): Promise<void> {
|
||||
if (!(await this.deps.isTeamActive(item.teamName))) {
|
||||
this.counters.dropped += 1;
|
||||
this.appendAudit({
|
||||
teamName: item.teamName,
|
||||
memberName: item.memberName,
|
||||
event: 'queue_dropped',
|
||||
source: 'event_queue',
|
||||
reason: 'team_inactive',
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -242,5 +279,31 @@ export class MemberWorkSyncEventQueue {
|
|||
}
|
||||
);
|
||||
this.counters.reconciled += 1;
|
||||
this.appendAudit({
|
||||
teamName: item.teamName,
|
||||
memberName: item.memberName,
|
||||
event: 'queue_reconciled',
|
||||
source: 'event_queue',
|
||||
triggerReasons: [...running.triggerReasons].sort(),
|
||||
});
|
||||
}
|
||||
|
||||
private appendAudit(input: Omit<MemberWorkSyncAuditEvent, 'timestamp'>): void {
|
||||
if (!this.deps.auditJournal) {
|
||||
return;
|
||||
}
|
||||
void this.deps.auditJournal
|
||||
.append({
|
||||
...input,
|
||||
timestamp: this.nowIso(),
|
||||
})
|
||||
.catch((error: unknown) => {
|
||||
this.deps.logger?.warn('member work sync queue audit append failed', {
|
||||
teamName: input.teamName,
|
||||
memberName: input.memberName,
|
||||
event: input.event,
|
||||
error: String(error),
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,17 @@
|
|||
import { join } from 'path';
|
||||
|
||||
import { TeamMemberStoragePaths } from '@main/services/team/TeamMemberStoragePaths';
|
||||
|
||||
export class MemberWorkSyncStorePaths {
|
||||
constructor(private readonly teamsBasePath: string) {}
|
||||
private readonly memberStorage: TeamMemberStoragePaths;
|
||||
|
||||
constructor(private readonly teamsBasePath: string) {
|
||||
this.memberStorage = new TeamMemberStoragePaths(teamsBasePath);
|
||||
}
|
||||
|
||||
getTeamRootDir(teamName: string): string {
|
||||
return join(this.teamsBasePath, teamName);
|
||||
}
|
||||
|
||||
getTeamDir(teamName: string): string {
|
||||
return join(this.teamsBasePath, teamName, '.member-work-sync');
|
||||
|
|
@ -22,4 +32,64 @@ export class MemberWorkSyncStorePaths {
|
|||
getReportTokenSecretPath(teamName: string): string {
|
||||
return join(this.getTeamDir(teamName), 'report-token-secret.json');
|
||||
}
|
||||
|
||||
getIndexesDir(teamName: string): string {
|
||||
return join(this.getTeamDir(teamName), 'indexes');
|
||||
}
|
||||
|
||||
getMetricsIndexPath(teamName: string): string {
|
||||
return join(this.getIndexesDir(teamName), 'metrics.json');
|
||||
}
|
||||
|
||||
getOutboxIndexPath(teamName: string): string {
|
||||
return join(this.getIndexesDir(teamName), 'outbox-index.json');
|
||||
}
|
||||
|
||||
getPendingReportsIndexPath(teamName: string): string {
|
||||
return join(this.getIndexesDir(teamName), 'pending-reports-index.json');
|
||||
}
|
||||
|
||||
getLegacyStatusPath(teamName: string): string {
|
||||
return this.getStatusPath(teamName);
|
||||
}
|
||||
|
||||
getLegacyPendingReportsPath(teamName: string): string {
|
||||
return this.getPendingReportsPath(teamName);
|
||||
}
|
||||
|
||||
getLegacyOutboxPath(teamName: string): string {
|
||||
return this.getOutboxPath(teamName);
|
||||
}
|
||||
|
||||
getMemberKey(memberName: string): string {
|
||||
return this.memberStorage.getMemberKey(memberName);
|
||||
}
|
||||
|
||||
getMemberDir(teamName: string, memberName: string): string {
|
||||
return this.memberStorage.getMemberDir(teamName, memberName);
|
||||
}
|
||||
|
||||
getMemberWorkSyncDir(teamName: string, memberName: string): string {
|
||||
return this.memberStorage.getMemberFeatureDir(teamName, memberName, '.member-work-sync');
|
||||
}
|
||||
|
||||
getMemberStatusPath(teamName: string, memberName: string): string {
|
||||
return join(this.getMemberWorkSyncDir(teamName, memberName), 'status.json');
|
||||
}
|
||||
|
||||
getMemberReportsPath(teamName: string, memberName: string): string {
|
||||
return join(this.getMemberWorkSyncDir(teamName, memberName), 'reports.json');
|
||||
}
|
||||
|
||||
getMemberOutboxPath(teamName: string, memberName: string): string {
|
||||
return join(this.getMemberWorkSyncDir(teamName, memberName), 'outbox.json');
|
||||
}
|
||||
|
||||
getMemberJournalPath(teamName: string, memberName: string): string {
|
||||
return join(this.getMemberWorkSyncDir(teamName, memberName), 'journal.jsonl');
|
||||
}
|
||||
|
||||
async ensureMemberWorkSyncDir(teamName: string, memberName: string): Promise<void> {
|
||||
await this.memberStorage.ensureMemberMeta(teamName, memberName);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,14 +4,24 @@ import { createLogger } from '@shared/utils/logger';
|
|||
|
||||
const logger = createLogger('Perf:EventLoop');
|
||||
|
||||
const DEFAULT_MAX_STALL_THRESHOLD_MS = 750;
|
||||
const DEFAULT_REPORT_INTERVAL_MS = 30_000;
|
||||
|
||||
let started = false;
|
||||
let currentOp: string | null = null;
|
||||
let lastReportAt = 0;
|
||||
|
||||
function isEnabled(): boolean {
|
||||
const raw = process.env.CLAUDE_TEAM_EVENT_LOOP_LAG_MONITOR_ENABLED?.trim().toLowerCase();
|
||||
return raw === '1' || raw === 'true' || raw === 'yes' || raw === 'on';
|
||||
}
|
||||
|
||||
export function setCurrentMainOp(op: string | null): void {
|
||||
currentOp = op;
|
||||
}
|
||||
|
||||
export function startEventLoopLagMonitor(): void {
|
||||
if (!isEnabled()) return;
|
||||
if (started) return;
|
||||
started = true;
|
||||
|
||||
|
|
@ -24,14 +34,19 @@ export function startEventLoopLagMonitor(): void {
|
|||
// Reset first so next window is clean even if logging throws
|
||||
h.reset();
|
||||
|
||||
// Only report meaningful stalls
|
||||
if (maxMs < 250) return;
|
||||
// Only report severe stalls. Sub-second blips are common during expected
|
||||
// Electron/main-process IO and are too noisy for default development logs.
|
||||
if (maxMs < DEFAULT_MAX_STALL_THRESHOLD_MS) return;
|
||||
|
||||
// For known IPC/main-thread operations we already emit operation-specific
|
||||
// timing diagnostics. Suppress the generic event-loop warning to avoid
|
||||
// duplicate noisy logs that do not add new debugging value.
|
||||
if (currentOp) return;
|
||||
|
||||
const now = Date.now();
|
||||
if (now - lastReportAt < DEFAULT_REPORT_INTERVAL_MS) return;
|
||||
lastReportAt = now;
|
||||
|
||||
logger.warn(
|
||||
`Event loop stall detected: p95=${p95Ms.toFixed(1)}ms max=${maxMs.toFixed(1)}ms` +
|
||||
(currentOp ? ` op=${currentOp}` : '')
|
||||
|
|
|
|||
|
|
@ -70,8 +70,9 @@ const TEAM_ROOT_FILES = [
|
|||
|
||||
// Subdirs under ~/.claude/teams/{teamName}/
|
||||
const TEAM_SUBDIRS = ['inboxes', 'review-decisions'];
|
||||
const TEAM_RECURSIVE_SUBDIRS = ['.opencode-runtime'];
|
||||
const TEAM_RECURSIVE_SUBDIRS = ['.opencode-runtime', 'members'];
|
||||
const ATOMIC_WRITE_TEMP_FILE_PREFIX = '.tmp.';
|
||||
const FILE_LOCK_SUFFIX = '.lock';
|
||||
const QUARANTINED_OPENCODE_LANE_INDEX_RE = /^lanes\.invalid\.\d+\.json$/;
|
||||
// Subdirs under getAppDataPath() (our own storage, not in ~/.claude/)
|
||||
const APP_DATA_SUBDIRS = ['attachments'];
|
||||
|
|
@ -112,6 +113,9 @@ function shouldCollectRecursiveBackupFile(relPath: string): boolean {
|
|||
if (fileName.startsWith(ATOMIC_WRITE_TEMP_FILE_PREFIX)) {
|
||||
return false;
|
||||
}
|
||||
if (fileName.endsWith(FILE_LOCK_SUFFIX)) {
|
||||
return false;
|
||||
}
|
||||
// Runtime quarantine files are diagnostic snapshots of invalid JSON.
|
||||
if (QUARANTINED_OPENCODE_LANE_INDEX_RE.test(fileName)) {
|
||||
return false;
|
||||
|
|
|
|||
|
|
@ -480,6 +480,7 @@ function normalizePersistedMemberState(
|
|||
parsed.pendingPermissionRequestIds
|
||||
),
|
||||
runtimePid: normalizeRuntimePid(parsed.runtimePid),
|
||||
runtimeRunId: normalizeOptionalString(parsed.runtimeRunId),
|
||||
runtimeSessionId: normalizeOptionalString(parsed.runtimeSessionId),
|
||||
livenessKind,
|
||||
pidSource: normalizePidSource(parsed.pidSource),
|
||||
|
|
|
|||
109
src/main/services/team/TeamMemberStoragePaths.ts
Normal file
109
src/main/services/team/TeamMemberStoragePaths.ts
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
import { mkdir, readFile } from 'fs/promises';
|
||||
import { join } from 'path';
|
||||
|
||||
import { atomicWriteAsync } from '@main/utils/atomicWrite';
|
||||
|
||||
export interface TeamMemberStorageMetaFile {
|
||||
schemaVersion: 1;
|
||||
memberName: string;
|
||||
memberKey: string;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
export function normalizeTeamMemberStorageName(memberName: string): string {
|
||||
return memberName.trim().toLowerCase();
|
||||
}
|
||||
|
||||
export function encodeTeamMemberStorageKey(memberName: string): string {
|
||||
const normalized = normalizeTeamMemberStorageName(memberName);
|
||||
if (!normalized) {
|
||||
throw new Error('memberName is required for member-scoped storage');
|
||||
}
|
||||
const encoded = encodeURIComponent(normalized);
|
||||
if (encoded === '.') {
|
||||
return '%2E';
|
||||
}
|
||||
if (encoded === '..') {
|
||||
return '%2E%2E';
|
||||
}
|
||||
return encoded;
|
||||
}
|
||||
|
||||
function isMetaFile(value: unknown): value is TeamMemberStorageMetaFile {
|
||||
return (
|
||||
value != null &&
|
||||
typeof value === 'object' &&
|
||||
(value as TeamMemberStorageMetaFile).schemaVersion === 1 &&
|
||||
typeof (value as TeamMemberStorageMetaFile).memberName === 'string' &&
|
||||
typeof (value as TeamMemberStorageMetaFile).memberKey === 'string' &&
|
||||
typeof (value as TeamMemberStorageMetaFile).updatedAt === 'string'
|
||||
);
|
||||
}
|
||||
|
||||
export class TeamMemberStoragePaths {
|
||||
constructor(private readonly teamsBasePath: string) {}
|
||||
|
||||
getTeamDir(teamName: string): string {
|
||||
return join(this.teamsBasePath, teamName);
|
||||
}
|
||||
|
||||
getMembersDir(teamName: string): string {
|
||||
return join(this.getTeamDir(teamName), 'members');
|
||||
}
|
||||
|
||||
getMemberKey(memberName: string): string {
|
||||
return encodeTeamMemberStorageKey(memberName);
|
||||
}
|
||||
|
||||
getMemberDir(teamName: string, memberName: string): string {
|
||||
return join(this.getMembersDir(teamName), this.getMemberKey(memberName));
|
||||
}
|
||||
|
||||
getMemberMetaPath(teamName: string, memberName: string): string {
|
||||
return join(this.getMemberDir(teamName, memberName), 'member.meta.json');
|
||||
}
|
||||
|
||||
getMemberFeatureDir(teamName: string, memberName: string, featureDirName: string): string {
|
||||
const featureDirSegment = featureDirName.trim();
|
||||
if (
|
||||
!featureDirSegment ||
|
||||
featureDirSegment === '.' ||
|
||||
featureDirSegment === '..' ||
|
||||
featureDirSegment.includes('/') ||
|
||||
featureDirSegment.includes('\\')
|
||||
) {
|
||||
throw new Error('featureDirName must be a single path segment');
|
||||
}
|
||||
return join(this.getMemberDir(teamName, memberName), featureDirSegment);
|
||||
}
|
||||
|
||||
async ensureMemberMeta(teamName: string, memberName: string): Promise<TeamMemberStorageMetaFile> {
|
||||
const canonicalMemberName = memberName.trim();
|
||||
const memberKey = this.getMemberKey(canonicalMemberName);
|
||||
const metaPath = this.getMemberMetaPath(teamName, canonicalMemberName);
|
||||
const existing = await this.readMeta(metaPath);
|
||||
if (existing?.memberName === canonicalMemberName && existing.memberKey === memberKey) {
|
||||
return existing;
|
||||
}
|
||||
|
||||
const next: TeamMemberStorageMetaFile = {
|
||||
schemaVersion: 1,
|
||||
memberName: canonicalMemberName,
|
||||
memberKey,
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
await mkdir(this.getMemberDir(teamName, canonicalMemberName), { recursive: true });
|
||||
await atomicWriteAsync(metaPath, `${JSON.stringify(next, null, 2)}\n`);
|
||||
return next;
|
||||
}
|
||||
|
||||
private async readMeta(filePath: string): Promise<TeamMemberStorageMetaFile | null> {
|
||||
try {
|
||||
const raw = await readFile(filePath, 'utf8');
|
||||
const parsed = JSON.parse(raw);
|
||||
return isMetaFile(parsed) ? parsed : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -167,6 +167,7 @@ import {
|
|||
} from './opencode/store/OpenCodeRuntimeManifestEvidenceReader';
|
||||
import {
|
||||
createRuntimeRunTombstoneStore,
|
||||
RuntimeStaleEvidenceError,
|
||||
type RuntimeEvidenceKind,
|
||||
} from './opencode/store/RuntimeRunTombstoneStore';
|
||||
import { OpenCodeTaskLogAttributionStore } from './taskLogs/stream/OpenCodeTaskLogAttributionStore';
|
||||
|
|
@ -1411,6 +1412,11 @@ interface ProvisioningRun {
|
|||
effectiveMembers: TeamCreateRequest['members'];
|
||||
launchIdentity: ProviderModelLaunchIdentity | null;
|
||||
mixedSecondaryLanes: MixedSecondaryRuntimeLaneState[];
|
||||
/**
|
||||
* OpenCode secondary lanes share bridge state files. Launch them sequentially
|
||||
* per team run to avoid file-lock contention while keeping launch non-blocking.
|
||||
*/
|
||||
mixedSecondaryLaneLaunchQueue?: Promise<void>;
|
||||
lastLogProgressAt: number;
|
||||
/** Monotonic ms timestamp of last stdout/stderr data. For stall detection. */
|
||||
lastDataReceivedAt: number;
|
||||
|
|
@ -1619,7 +1625,7 @@ interface PromptSizeSummary {
|
|||
lines: number;
|
||||
}
|
||||
|
||||
const MEMBER_LAUNCH_GRACE_MS = 90_000;
|
||||
const MEMBER_LAUNCH_GRACE_MS = 150_000;
|
||||
const MEMBER_BOOTSTRAP_STALL_MS = 5 * 60_000;
|
||||
|
||||
export function shouldWarnOnUnreadableMemberAuditConfig(params: {
|
||||
|
|
@ -1809,6 +1815,18 @@ function isRecoverablePersistedOpenCodeRuntimeCandidate(
|
|||
);
|
||||
}
|
||||
|
||||
function isPersistedOpenCodeSecondaryLaneMember(
|
||||
member: PersistedTeamLaunchMemberState | undefined | null
|
||||
): boolean {
|
||||
return (
|
||||
member?.providerId === 'opencode' &&
|
||||
member.laneKind === 'secondary' &&
|
||||
member.laneOwnerProviderId === 'opencode' &&
|
||||
typeof member.laneId === 'string' &&
|
||||
member.laneId.trim().length > 0
|
||||
);
|
||||
}
|
||||
|
||||
function isRecoverablePersistedOpenCodeTerminalRuntimeCandidate(
|
||||
member: PersistedTeamLaunchMemberState | undefined | null
|
||||
): boolean {
|
||||
|
|
@ -6081,6 +6099,26 @@ export class TeamProvisioningService {
|
|||
runtimeActive = true;
|
||||
}
|
||||
}
|
||||
if (
|
||||
runtimeActive &&
|
||||
laneIdentity.laneKind === 'secondary' &&
|
||||
laneIdentity.laneOwnerProviderId === 'opencode' &&
|
||||
!liveSecondaryLaneRunId
|
||||
) {
|
||||
const staleLane = await recoverStaleOpenCodeRuntimeLaneIndexEntry({
|
||||
teamsBasePath: getTeamsBasePath(),
|
||||
teamName,
|
||||
laneId: laneIdentity.laneId,
|
||||
});
|
||||
if (staleLane.stale) {
|
||||
this.deleteSecondaryRuntimeRun(teamName, laneIdentity.laneId);
|
||||
return {
|
||||
delivered: false,
|
||||
reason: 'opencode_runtime_not_active',
|
||||
diagnostics: staleLane.diagnostics,
|
||||
};
|
||||
}
|
||||
}
|
||||
if (!runtimeActive) {
|
||||
this.cleanupStoppedTeamOpenCodeRuntimeLanesInBackground(teamName);
|
||||
return { delivered: false, reason: 'opencode_runtime_not_active' };
|
||||
|
|
@ -8193,6 +8231,38 @@ export class TeamProvisioningService {
|
|||
laneId,
|
||||
evidenceKind: 'bootstrap_checkin',
|
||||
});
|
||||
const idempotent = await this.resolveOpenCodeRuntimeBootstrapCheckinIdempotency({
|
||||
teamName,
|
||||
runId,
|
||||
memberName,
|
||||
runtimeSessionId,
|
||||
});
|
||||
await this.assertOpenCodeRuntimeMemberCheckinAllowed({
|
||||
teamName,
|
||||
memberName,
|
||||
previousMember: idempotent.previousMember,
|
||||
});
|
||||
if (idempotent.state === 'duplicate') {
|
||||
return {
|
||||
ok: true,
|
||||
providerId: 'opencode',
|
||||
teamName,
|
||||
runId,
|
||||
state: 'accepted',
|
||||
memberName,
|
||||
runtimeSessionId,
|
||||
diagnostics: ['opencode_bootstrap_checkin_duplicate_accepted'],
|
||||
observedAt,
|
||||
};
|
||||
}
|
||||
if (idempotent.state === 'conflict') {
|
||||
throw new RuntimeStaleEvidenceError(
|
||||
`opencode_bootstrap_checkin_session_conflict: existing runtime session ${idempotent.existingRuntimeSessionId}, received ${runtimeSessionId} for ${memberName}`,
|
||||
'run_mismatch',
|
||||
'bootstrap_checkin',
|
||||
runId
|
||||
);
|
||||
}
|
||||
await this.updateOpenCodeRuntimeMemberLiveness({
|
||||
teamName,
|
||||
runId,
|
||||
|
|
@ -8217,6 +8287,95 @@ export class TeamProvisioningService {
|
|||
};
|
||||
}
|
||||
|
||||
private async resolveOpenCodeRuntimeBootstrapCheckinIdempotency(input: {
|
||||
teamName: string;
|
||||
runId: string;
|
||||
memberName: string;
|
||||
runtimeSessionId: string;
|
||||
}): Promise<
|
||||
| {
|
||||
state: 'new';
|
||||
previousMember?: PersistedTeamLaunchMemberState;
|
||||
}
|
||||
| {
|
||||
state: 'duplicate';
|
||||
previousMember: PersistedTeamLaunchMemberState;
|
||||
}
|
||||
| {
|
||||
state: 'conflict';
|
||||
previousMember: PersistedTeamLaunchMemberState;
|
||||
existingRuntimeSessionId: string;
|
||||
}
|
||||
> {
|
||||
const snapshot = await this.launchStateStore.read(input.teamName);
|
||||
const previousMember = snapshot?.members[input.memberName];
|
||||
if (!previousMember) {
|
||||
return { state: 'new' };
|
||||
}
|
||||
|
||||
const existingRuntimeSessionId = previousMember.runtimeSessionId?.trim();
|
||||
const existingRuntimeRunId =
|
||||
typeof previousMember.runtimeRunId === 'string' ? previousMember.runtimeRunId.trim() : '';
|
||||
const hasAcceptedBootstrap =
|
||||
previousMember.bootstrapConfirmed === true ||
|
||||
previousMember.livenessKind === 'confirmed_bootstrap' ||
|
||||
previousMember.launchState === 'confirmed_alive';
|
||||
|
||||
if (!hasAcceptedBootstrap || !existingRuntimeSessionId) {
|
||||
return { state: 'new', previousMember };
|
||||
}
|
||||
|
||||
if (existingRuntimeRunId && existingRuntimeRunId !== input.runId) {
|
||||
return { state: 'new', previousMember };
|
||||
}
|
||||
|
||||
if (existingRuntimeSessionId === input.runtimeSessionId) {
|
||||
return { state: 'duplicate', previousMember };
|
||||
}
|
||||
|
||||
if (!existingRuntimeRunId) {
|
||||
return { state: 'new', previousMember };
|
||||
}
|
||||
|
||||
return {
|
||||
state: 'conflict',
|
||||
previousMember,
|
||||
existingRuntimeSessionId,
|
||||
};
|
||||
}
|
||||
|
||||
private async assertOpenCodeRuntimeMemberCheckinAllowed(input: {
|
||||
teamName: string;
|
||||
memberName: string;
|
||||
previousMember?: PersistedTeamLaunchMemberState;
|
||||
}): Promise<void> {
|
||||
const config = await this.configReader.getConfig(input.teamName).catch(() => null);
|
||||
const metaMembers = await this.membersMetaStore.getMembers(input.teamName).catch(() => []);
|
||||
const configuredMember = this.resolveEffectiveConfiguredMember(
|
||||
config?.members ?? [],
|
||||
metaMembers,
|
||||
input.memberName
|
||||
);
|
||||
|
||||
if (configuredMember?.removedAt != null) {
|
||||
throw new RuntimeStaleEvidenceError(
|
||||
`Rejected OpenCode bootstrap check-in for removed member "${input.memberName}"`,
|
||||
'run_mismatch',
|
||||
'bootstrap_checkin',
|
||||
null
|
||||
);
|
||||
}
|
||||
|
||||
if (!configuredMember && !input.previousMember) {
|
||||
throw new RuntimeStaleEvidenceError(
|
||||
`Rejected OpenCode bootstrap check-in for unconfigured member "${input.memberName}"`,
|
||||
'run_mismatch',
|
||||
'bootstrap_checkin',
|
||||
null
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async deliverOpenCodeRuntimeMessage(raw: unknown): Promise<OpenCodeRuntimeControlAck> {
|
||||
const payload = asRuntimeRecord(raw);
|
||||
const teamName = requireRuntimeString(payload.teamName, 'teamName');
|
||||
|
|
@ -8386,6 +8545,16 @@ export class TeamProvisioningService {
|
|||
.map((member) => (typeof member.name === 'string' ? member.name.trim() : ''))
|
||||
.filter((name) => name.length > 0 && name !== 'user' && !isLeadMember({ name }));
|
||||
const previousMember = previous?.members[input.memberName];
|
||||
const previousRuntimeRunId =
|
||||
typeof previousMember?.runtimeRunId === 'string' ? previousMember.runtimeRunId.trim() : '';
|
||||
const sameRuntimeRun = previousRuntimeRunId.length > 0 && previousRuntimeRunId === input.runId;
|
||||
const runtimePid =
|
||||
input.metadata?.runtimePid ?? (sameRuntimeRun ? previousMember?.runtimePid : undefined);
|
||||
const pidSource = input.metadata?.runtimePid
|
||||
? ('runtime_bootstrap' as const)
|
||||
: sameRuntimeRun
|
||||
? previousMember?.pidSource
|
||||
: undefined;
|
||||
const persistedIdentity = this.resolvePersistedRuntimeMemberIdentity({
|
||||
teamName: input.teamName,
|
||||
memberName: input.memberName,
|
||||
|
|
@ -8400,10 +8569,11 @@ export class TeamProvisioningService {
|
|||
runtimeAlive: true,
|
||||
bootstrapConfirmed: true,
|
||||
hardFailure: false,
|
||||
...(input.metadata?.runtimePid ? { runtimePid: input.metadata.runtimePid } : {}),
|
||||
runtimePid,
|
||||
runtimeRunId: input.runId,
|
||||
runtimeSessionId: input.runtimeSessionId,
|
||||
livenessKind: 'confirmed_bootstrap',
|
||||
...(input.metadata?.runtimePid ? { pidSource: 'runtime_bootstrap' as const } : {}),
|
||||
pidSource,
|
||||
runtimeDiagnostic: input.reason,
|
||||
runtimeDiagnosticSeverity: 'info',
|
||||
runtimeLastSeenAt: input.observedAt,
|
||||
|
|
@ -10854,6 +11024,7 @@ export class TeamProvisioningService {
|
|||
return;
|
||||
}
|
||||
await this.reconcileBootstrapTranscriptFailures(run);
|
||||
await this.reconcileBootstrapTranscriptSuccesses(run);
|
||||
if (this.shouldSkipMemberSpawnAudit(run)) {
|
||||
return;
|
||||
}
|
||||
|
|
@ -10899,9 +11070,17 @@ export class TeamProvisioningService {
|
|||
private async reconcileBootstrapTranscriptSuccesses(run: ProvisioningRun): Promise<void> {
|
||||
for (const memberName of run.expectedMembers ?? []) {
|
||||
const current = run.memberSpawnStatuses.get(memberName);
|
||||
if (this.isOpenCodeSecondaryLaneMemberInRun(run, memberName)) {
|
||||
continue;
|
||||
}
|
||||
const failureReason = current?.hardFailureReason ?? current?.error;
|
||||
const canClearFailedBootstrap =
|
||||
current?.launchState === 'failed_to_start' &&
|
||||
current.agentToolAccepted === true &&
|
||||
isAutoClearableLaunchFailureReason(failureReason);
|
||||
if (
|
||||
!current ||
|
||||
current.launchState === 'failed_to_start' ||
|
||||
(current.launchState === 'failed_to_start' && !canClearFailedBootstrap) ||
|
||||
current.launchState === 'confirmed_alive' ||
|
||||
current.bootstrapConfirmed === true ||
|
||||
current.agentToolAccepted !== true
|
||||
|
|
@ -10922,6 +11101,11 @@ export class TeamProvisioningService {
|
|||
}
|
||||
}
|
||||
|
||||
private isOpenCodeSecondaryLaneMemberInRun(run: ProvisioningRun, memberName: string): boolean {
|
||||
const lanes = Array.isArray(run.mixedSecondaryLanes) ? run.mixedSecondaryLanes : [];
|
||||
return lanes.some((lane) => lane.providerId === 'opencode' && lane.member.name === memberName);
|
||||
}
|
||||
|
||||
private static readonly CONTEXT_EMIT_THROTTLE_MS = 2000;
|
||||
private static readonly LEAD_TEXT_EMIT_THROTTLE_MS = 2000;
|
||||
|
||||
|
|
@ -17848,8 +18032,13 @@ export class TeamProvisioningService {
|
|||
lane.state = 'launching';
|
||||
lane.runId = lane.runId ?? randomUUID();
|
||||
|
||||
void (async () => {
|
||||
const launch = async () => {
|
||||
try {
|
||||
if (run.cancelRequested || run.processKilled) {
|
||||
this.deleteSecondaryRuntimeRun(run.teamName, lane.laneId);
|
||||
lane.state = 'finished';
|
||||
return;
|
||||
}
|
||||
await this.launchSingleMixedSecondaryLane(run, lane);
|
||||
} catch (error) {
|
||||
if (run.cancelRequested || run.processKilled) {
|
||||
|
|
@ -17879,7 +18068,18 @@ export class TeamProvisioningService {
|
|||
await this.publishMixedSecondaryLaneStatusChange(run, lane).catch(() => undefined);
|
||||
lane.state = 'finished';
|
||||
}
|
||||
})();
|
||||
};
|
||||
|
||||
const previousLaunch = run.mixedSecondaryLaneLaunchQueue ?? Promise.resolve();
|
||||
const nextLaunch = previousLaunch.catch(() => undefined).then(launch);
|
||||
run.mixedSecondaryLaneLaunchQueue = nextLaunch.catch((error) => {
|
||||
logger.warn(
|
||||
`[${run.teamName}] OpenCode secondary lane launch queue failed: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}`
|
||||
);
|
||||
});
|
||||
void run.mixedSecondaryLaneLaunchQueue;
|
||||
}
|
||||
|
||||
private async launchMixedSecondaryLaneIfNeeded(
|
||||
|
|
@ -18085,7 +18285,7 @@ export class TeamProvisioningService {
|
|||
projectPath,
|
||||
previousLaunchState: persistedSnapshot ?? bootstrapSnapshot,
|
||||
});
|
||||
if (runtimeEvidence) {
|
||||
if (isRecoverableOpenCodeRuntimeEvidence(runtimeEvidence)) {
|
||||
recoveredAny = true;
|
||||
secondaryMembers.push({
|
||||
laneId: laneIdentity.laneId,
|
||||
|
|
@ -18340,7 +18540,10 @@ export class TeamProvisioningService {
|
|||
expected,
|
||||
Number.isFinite(acceptedAtMs) ? acceptedAtMs : null
|
||||
);
|
||||
if (transcriptOutcome) {
|
||||
if (
|
||||
transcriptOutcome &&
|
||||
(transcriptOutcome.kind !== 'success' || !isPersistedOpenCodeSecondaryLaneMember(current))
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
|
@ -18481,12 +18684,17 @@ export class TeamProvisioningService {
|
|||
hardFailure: false,
|
||||
lastEvaluatedAt: now,
|
||||
};
|
||||
const isOpenCodeSecondaryLaneMember = isPersistedOpenCodeSecondaryLaneMember(current);
|
||||
if (bootstrapMember?.agentToolAccepted && !current.agentToolAccepted) {
|
||||
current.agentToolAccepted = true;
|
||||
current.firstSpawnAcceptedAt =
|
||||
current.firstSpawnAcceptedAt ?? bootstrapMember.firstSpawnAcceptedAt;
|
||||
}
|
||||
if (bootstrapMember?.bootstrapConfirmed && !current.bootstrapConfirmed) {
|
||||
if (
|
||||
bootstrapMember?.bootstrapConfirmed &&
|
||||
!current.bootstrapConfirmed &&
|
||||
!isOpenCodeSecondaryLaneMember
|
||||
) {
|
||||
current.bootstrapConfirmed = true;
|
||||
current.lastHeartbeatAt = current.lastHeartbeatAt ?? bootstrapMember.lastHeartbeatAt;
|
||||
}
|
||||
|
|
@ -18546,7 +18754,7 @@ export class TeamProvisioningService {
|
|||
current.hardFailure = true;
|
||||
current.hardFailureReason = heartbeatReason;
|
||||
current.sources.hardFailureSignal = true;
|
||||
} else if (heartbeatMessage) {
|
||||
} else if (heartbeatMessage && !isOpenCodeSecondaryLaneMember) {
|
||||
current.bootstrapConfirmed = true;
|
||||
current.lastHeartbeatAt = heartbeatMessage.timestamp;
|
||||
current.hardFailure = false;
|
||||
|
|
@ -18558,7 +18766,7 @@ export class TeamProvisioningService {
|
|||
expected,
|
||||
Number.isFinite(acceptedAtMs) ? acceptedAtMs : null
|
||||
);
|
||||
if (transcriptOutcome?.kind === 'success') {
|
||||
if (transcriptOutcome?.kind === 'success' && !isOpenCodeSecondaryLaneMember) {
|
||||
current.bootstrapConfirmed = true;
|
||||
current.lastHeartbeatAt = current.lastHeartbeatAt ?? transcriptOutcome.observedAt;
|
||||
current.hardFailure = false;
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import { withFileLock } from '../../fileLock';
|
|||
import {
|
||||
createDefaultRuntimeStoreManifest,
|
||||
createRuntimeStoreManifestStore,
|
||||
OPENCODE_RUNTIME_STORE_DESCRIPTORS,
|
||||
OPENCODE_RUNTIME_STORE_MANIFEST_SCHEMA_VERSION,
|
||||
validateRuntimeStoreManifest,
|
||||
} from './RuntimeStoreManifest';
|
||||
|
|
@ -28,11 +29,19 @@ const OPENCODE_TEAM_RUNTIME_LANES_DIR = 'lanes';
|
|||
const OPENCODE_TEAM_RUNTIME_LANES_INDEX_FILE = 'lanes.json';
|
||||
const OPENCODE_RUNTIME_MANIFEST_FILE = 'manifest.json';
|
||||
const OPENCODE_RUNTIME_RUN_TOMBSTONES_FILE = 'opencode-run-tombstones.json';
|
||||
const OPENCODE_ACTIVE_EMPTY_LANE_STALE_MS = 150_000;
|
||||
const OPENCODE_LANE_INDEX_LOCK_OPTIONS = {
|
||||
acquireTimeoutMs: 30_000,
|
||||
staleTimeoutMs: 25_000,
|
||||
retryIntervalMs: 25,
|
||||
} as const;
|
||||
const OPENCODE_RUNTIME_EVIDENCE_FILES = new Set(
|
||||
OPENCODE_RUNTIME_STORE_DESCRIPTORS.filter(
|
||||
(descriptor) =>
|
||||
descriptor.schemaName !== 'opencode.promptDeliveryLedger' &&
|
||||
descriptor.schemaName !== 'opencode.deliveryJournal'
|
||||
).map((descriptor) => descriptor.relativePath)
|
||||
);
|
||||
|
||||
export interface OpenCodeRuntimeLaneIndexEntry {
|
||||
laneId: string;
|
||||
|
|
@ -318,6 +327,9 @@ export async function inspectOpenCodeRuntimeLaneStorage(params: {
|
|||
}): Promise<{
|
||||
laneDirectoryExists: boolean;
|
||||
hasStateOnDisk: boolean;
|
||||
hasRuntimeEvidenceOnDisk: boolean;
|
||||
manifestEntryCount: number | null;
|
||||
manifestUpdatedAt: string | null;
|
||||
fileNames: string[];
|
||||
}> {
|
||||
const laneDir = getOpenCodeTeamRuntimeLaneDirectory(
|
||||
|
|
@ -330,14 +342,38 @@ export async function inspectOpenCodeRuntimeLaneStorage(params: {
|
|||
return {
|
||||
laneDirectoryExists: false,
|
||||
hasStateOnDisk: false,
|
||||
hasRuntimeEvidenceOnDisk: false,
|
||||
manifestEntryCount: null,
|
||||
manifestUpdatedAt: null,
|
||||
fileNames: [],
|
||||
};
|
||||
}
|
||||
|
||||
const fileNames = (await readdir(laneDir).catch(() => [] as string[])).sort();
|
||||
const manifestPath = getOpenCodeRuntimeManifestPath(
|
||||
params.teamsBasePath,
|
||||
params.teamName,
|
||||
params.laneId
|
||||
);
|
||||
const manifest = (await fileExists(manifestPath))
|
||||
? await readRuntimeStoreManifestEvidenceData(
|
||||
manifestPath,
|
||||
params.teamName,
|
||||
() => new Date()
|
||||
).catch(() => null)
|
||||
: null;
|
||||
const hasRuntimeEvidenceFile = fileNames.some((fileName) =>
|
||||
OPENCODE_RUNTIME_EVIDENCE_FILES.has(fileName)
|
||||
);
|
||||
const hasRuntimeEvidenceManifestEntry =
|
||||
manifest?.entries.some((entry) => OPENCODE_RUNTIME_EVIDENCE_FILES.has(entry.relativePath)) ??
|
||||
false;
|
||||
return {
|
||||
laneDirectoryExists: true,
|
||||
hasStateOnDisk: fileNames.length > 0,
|
||||
hasRuntimeEvidenceOnDisk: hasRuntimeEvidenceFile || hasRuntimeEvidenceManifestEntry,
|
||||
manifestEntryCount: manifest ? manifest.entries.length : null,
|
||||
manifestUpdatedAt: manifest?.updatedAt ?? null,
|
||||
fileNames,
|
||||
};
|
||||
}
|
||||
|
|
@ -513,6 +549,8 @@ export async function recoverStaleOpenCodeRuntimeLaneIndexEntry(params: {
|
|||
teamsBasePath: string;
|
||||
teamName: string;
|
||||
laneId: string;
|
||||
clock?: () => Date;
|
||||
emptyLaneStaleAfterMs?: number;
|
||||
}): Promise<{
|
||||
stale: boolean;
|
||||
degraded: boolean;
|
||||
|
|
@ -529,7 +567,7 @@ export async function recoverStaleOpenCodeRuntimeLaneIndexEntry(params: {
|
|||
}
|
||||
|
||||
const storage = await inspectOpenCodeRuntimeLaneStorage(params);
|
||||
if (storage.hasStateOnDisk) {
|
||||
if (storage.hasRuntimeEvidenceOnDisk) {
|
||||
return {
|
||||
stale: false,
|
||||
degraded: false,
|
||||
|
|
@ -537,9 +575,26 @@ export async function recoverStaleOpenCodeRuntimeLaneIndexEntry(params: {
|
|||
};
|
||||
}
|
||||
|
||||
const diagnostics = [
|
||||
`OpenCode lane ${params.laneId} is marked active in lanes.json, but no lane state exists on disk.`,
|
||||
];
|
||||
const now = params.clock?.() ?? new Date();
|
||||
const staleAfterMs = params.emptyLaneStaleAfterMs ?? OPENCODE_ACTIVE_EMPTY_LANE_STALE_MS;
|
||||
const lastTouchedAt =
|
||||
Date.parse(storage.manifestUpdatedAt ?? '') || Date.parse(entry.updatedAt) || NaN;
|
||||
const laneAgeMs = Number.isFinite(lastTouchedAt) ? now.getTime() - lastTouchedAt : Infinity;
|
||||
if (storage.hasStateOnDisk && laneAgeMs < staleAfterMs) {
|
||||
return {
|
||||
stale: false,
|
||||
degraded: false,
|
||||
diagnostics: [],
|
||||
};
|
||||
}
|
||||
|
||||
const diagnostics = storage.hasStateOnDisk
|
||||
? [
|
||||
`OpenCode lane ${params.laneId} is marked active in lanes.json, but its runtime manifest has no committed runtime evidence after launch grace.`,
|
||||
]
|
||||
: [
|
||||
`OpenCode lane ${params.laneId} is marked active in lanes.json, but no lane state exists on disk.`,
|
||||
];
|
||||
await upsertOpenCodeRuntimeLaneIndexEntry({
|
||||
teamsBasePath: params.teamsBasePath,
|
||||
teamName: params.teamName,
|
||||
|
|
|
|||
|
|
@ -429,6 +429,7 @@ function mapOpenCodeLaunchDataToRuntimeResult(
|
|||
data: OpenCodeLaunchTeamCommandData,
|
||||
prepareWarnings: string[]
|
||||
): TeamRuntimeLaunchResult {
|
||||
const bridgeDiagnostics = data.diagnostics.map(formatOpenCodeBridgeDiagnostic);
|
||||
const checkpointNames = extractCheckpointNames(data);
|
||||
const readyCheckpointsPresent = [...REQUIRED_READY_CHECKPOINTS].every((name) =>
|
||||
checkpointNames.has(name)
|
||||
|
|
@ -491,6 +492,7 @@ function mapOpenCodeLaunchDataToRuntimeResult(
|
|||
...(bridgeMember?.evidence ?? []).map(
|
||||
(evidence) => `${evidence.kind} at ${evidence.observedAt}`
|
||||
),
|
||||
...bridgeDiagnostics,
|
||||
...checkpointDiagnostic,
|
||||
...(missingExpectedMembers.includes(member.name) ? incompleteReadyDiagnostic : []),
|
||||
]
|
||||
|
|
@ -518,11 +520,7 @@ function mapOpenCodeLaunchDataToRuntimeResult(
|
|||
: 'partial_failure',
|
||||
members,
|
||||
warnings: [...prepareWarnings, ...data.warnings.map((warning) => warning.message)],
|
||||
diagnostics: [
|
||||
...data.diagnostics.map(formatOpenCodeBridgeDiagnostic),
|
||||
...checkpointDiagnostic,
|
||||
...incompleteReadyDiagnostic,
|
||||
],
|
||||
diagnostics: [...bridgeDiagnostics, ...checkpointDiagnostic, ...incompleteReadyDiagnostic],
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -539,29 +537,28 @@ function mapBridgeMemberToRuntimeEvidence(
|
|||
const failed = launchState === 'failed';
|
||||
const hasRuntimePid =
|
||||
typeof runtimePid === 'number' && Number.isFinite(runtimePid) && runtimePid > 0;
|
||||
const pendingRuntimeObserved = launchState === 'created' && hasRuntimePid;
|
||||
const hasSessionId = typeof sessionId === 'string' && sessionId.trim().length > 0;
|
||||
const hasRuntimeHandle = hasRuntimePid || hasSessionId;
|
||||
const pendingRuntimeObserved = launchState === 'created' && hasRuntimeHandle;
|
||||
const livenessKind = confirmed
|
||||
? 'confirmed_bootstrap'
|
||||
: pendingRuntimeObserved
|
||||
? 'runtime_process_candidate'
|
||||
: launchState === 'permission_blocked'
|
||||
? 'permission_blocked'
|
||||
: runtimeMaterialized || sessionId
|
||||
? 'runtime_process_candidate'
|
||||
: 'registered_only';
|
||||
: 'registered_only';
|
||||
const runtimeDiagnostic = pendingRuntimeObserved
|
||||
? 'OpenCode runtime pid reported by bridge without local process verification'
|
||||
? hasRuntimePid
|
||||
? 'OpenCode runtime pid reported by bridge without local process verification'
|
||||
: 'OpenCode session exists without verified runtime pid'
|
||||
: launchState === 'permission_blocked'
|
||||
? 'OpenCode runtime is waiting for permission approval'
|
||||
: runtimeMaterialized || sessionId
|
||||
? 'OpenCode session exists without verified runtime pid'
|
||||
: runtimeMaterialized
|
||||
? 'OpenCode bridge did not report a runtime session or pid for this member'
|
||||
: undefined;
|
||||
const runtimeDiagnosticSeverity = failed
|
||||
? 'error'
|
||||
: pendingRuntimeObserved ||
|
||||
launchState === 'permission_blocked' ||
|
||||
runtimeMaterialized ||
|
||||
sessionId
|
||||
: pendingRuntimeObserved || launchState === 'permission_blocked' || runtimeMaterialized
|
||||
? 'warning'
|
||||
: undefined;
|
||||
return {
|
||||
|
|
@ -578,8 +575,7 @@ function mapBridgeMemberToRuntimeEvidence(
|
|||
confirmed ||
|
||||
pendingRuntimeObserved ||
|
||||
launchState === 'permission_blocked' ||
|
||||
runtimeMaterialized ||
|
||||
Boolean(sessionId),
|
||||
hasRuntimeHandle,
|
||||
runtimeAlive: confirmed,
|
||||
bootstrapConfirmed: confirmed,
|
||||
hardFailure: failed,
|
||||
|
|
|
|||
|
|
@ -376,6 +376,10 @@ export const CliLogsRichView = ({
|
|||
|
||||
const groups = useMemo(() => parseStreamJsonToGroups(cliLogsTail), [cliLogsTail]);
|
||||
const entries = useMemo(() => groupBySubagent(groups), [groups]);
|
||||
const emptyMessage =
|
||||
cliLogsTail.trim().length > 0
|
||||
? 'No displayable assistant/runtime logs yet.'
|
||||
: 'Waiting for response...';
|
||||
|
||||
// Derive expanded state: all groups expanded unless manually collapsed
|
||||
const expandedGroupIds = useMemo(() => {
|
||||
|
|
@ -578,9 +582,7 @@ export const CliLogsRichView = ({
|
|||
<span className="absolute inline-flex size-full animate-ping rounded-full bg-[var(--color-text-muted)] opacity-40" />
|
||||
<span className="relative inline-flex size-2 rounded-full bg-[var(--color-text-muted)]" />
|
||||
</span>
|
||||
<span className="text-[11px] text-[var(--color-text-muted)]">
|
||||
Waiting for response...
|
||||
</span>
|
||||
<span className="text-[11px] text-[var(--color-text-muted)]">{emptyMessage}</span>
|
||||
</div>
|
||||
{footer}
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { useEffect, useMemo, useState } from 'react';
|
||||
|
||||
import { MemberWorkSyncStatusPanel } from '@features/member-work-sync/renderer';
|
||||
// import { MemberWorkSyncStatusPanel } from '@features/member-work-sync/renderer';
|
||||
import { Button } from '@renderer/components/ui/button';
|
||||
import { Dialog, DialogContent, DialogFooter, DialogHeader } from '@renderer/components/ui/dialog';
|
||||
import { Tabs, TabsContent, TabsList, TabsTrigger } from '@renderer/components/ui/tabs';
|
||||
|
|
@ -293,11 +293,13 @@ export const MemberDetailDialog = ({
|
|||
</TabsList>
|
||||
<TabsContent value="tasks">
|
||||
<div className="space-y-3">
|
||||
{/*
|
||||
<MemberWorkSyncStatusPanel
|
||||
teamName={teamName}
|
||||
memberName={member.name}
|
||||
enabled={open && !member.removedAt}
|
||||
/>
|
||||
*/}
|
||||
<MemberTasksTab tasks={memberTasks} onTaskClick={onTaskClick} />
|
||||
</div>
|
||||
</TabsContent>
|
||||
|
|
|
|||
|
|
@ -1002,6 +1002,8 @@ export interface PersistedTeamLaunchMemberState {
|
|||
hardFailureReason?: string;
|
||||
pendingPermissionRequestIds?: string[];
|
||||
runtimePid?: number;
|
||||
/** OpenCode runtime run id that produced the current runtimeSessionId/liveness evidence. */
|
||||
runtimeRunId?: string;
|
||||
runtimeSessionId?: string;
|
||||
livenessKind?: TeamAgentRuntimeLivenessKind;
|
||||
pidSource?: TeamAgentRuntimePidSource;
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import {
|
|||
MemberWorkSyncReconciler,
|
||||
MemberWorkSyncReporter,
|
||||
type MemberWorkSyncAgendaSourceResult,
|
||||
type MemberWorkSyncAuditEvent,
|
||||
type MemberWorkSyncInboxNudgePort,
|
||||
type MemberWorkSyncOutboxStorePort,
|
||||
type MemberWorkSyncStatusStorePort,
|
||||
|
|
@ -247,6 +248,7 @@ function createDeps(options?: {
|
|||
}) {
|
||||
const clock = new MutableClock();
|
||||
const store = new InMemoryStatusStore();
|
||||
const auditEvents: MemberWorkSyncAuditEvent[] = [];
|
||||
const source: MemberWorkSyncAgendaSourceResult = {
|
||||
agenda: {
|
||||
teamName: 'team-a',
|
||||
|
|
@ -286,13 +288,18 @@ function createDeps(options?: {
|
|||
lifecycle: {
|
||||
isTeamActive: () => options?.teamActive ?? true,
|
||||
},
|
||||
auditJournal: {
|
||||
append: async (event) => {
|
||||
auditEvents.push(event);
|
||||
},
|
||||
},
|
||||
};
|
||||
return { clock, deps, source, store };
|
||||
return { auditEvents, clock, deps, source, store };
|
||||
}
|
||||
|
||||
describe('MemberWorkSync use cases', () => {
|
||||
it('reconciles actionable work into needs_sync without side effects', async () => {
|
||||
const { deps, store } = createDeps();
|
||||
const { auditEvents, deps, store } = createDeps();
|
||||
const status = await new MemberWorkSyncDiagnosticsReader(deps).execute({
|
||||
teamName: 'team-a',
|
||||
memberName: 'bob',
|
||||
|
|
@ -308,10 +315,15 @@ describe('MemberWorkSync use cases', () => {
|
|||
fingerprintChanged: false,
|
||||
});
|
||||
expect(store.pendingReports).toEqual([]);
|
||||
expect(auditEvents.map((event) => event.event)).toEqual([
|
||||
'reconcile_started',
|
||||
'agenda_loaded',
|
||||
'decision_made',
|
||||
]);
|
||||
});
|
||||
|
||||
it('accepts still_working as a bounded lease for the current fingerprint', async () => {
|
||||
const { clock, deps } = createDeps();
|
||||
const { auditEvents, clock, deps } = createDeps();
|
||||
const reader = new MemberWorkSyncDiagnosticsReader(deps);
|
||||
const reporter = new MemberWorkSyncReporter(deps);
|
||||
const current = await reader.execute({ teamName: 'team-a', memberName: 'bob' });
|
||||
|
|
@ -340,6 +352,7 @@ describe('MemberWorkSync use cases', () => {
|
|||
const expired = await reader.execute({ teamName: 'team-a', memberName: 'bob' });
|
||||
expect(expired.state).toBe('needs_sync');
|
||||
expect(expired.diagnostics).toContain('report_lease_expired');
|
||||
expect(auditEvents.map((event) => event.event)).toContain('report_accepted');
|
||||
});
|
||||
|
||||
it('uses app clock instead of model supplied reportedAt for lease timing', async () => {
|
||||
|
|
@ -365,7 +378,7 @@ describe('MemberWorkSync use cases', () => {
|
|||
});
|
||||
|
||||
it('rejects stale reports without turning app-side validation failures into pending intents', async () => {
|
||||
const { deps, store } = createDeps();
|
||||
const { auditEvents, deps, store } = createDeps();
|
||||
const result = await new MemberWorkSyncReporter(deps).execute({
|
||||
teamName: 'team-a',
|
||||
memberName: 'bob',
|
||||
|
|
@ -384,6 +397,14 @@ describe('MemberWorkSync use cases', () => {
|
|||
});
|
||||
expect(store.writes.at(-1)?.diagnostics).toContain('report_rejected:stale_fingerprint');
|
||||
expect(store.pendingReports).toHaveLength(0);
|
||||
expect(auditEvents).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
event: 'report_rejected',
|
||||
reason: 'stale_fingerprint',
|
||||
}),
|
||||
])
|
||||
);
|
||||
});
|
||||
|
||||
it('accepts caught_up only when the app-side agenda is empty', async () => {
|
||||
|
|
@ -619,7 +640,7 @@ describe('MemberWorkSync use cases', () => {
|
|||
it('defers nudge dispatch while the member has active or recent tool activity', async () => {
|
||||
const outbox = new InMemoryOutboxStore();
|
||||
const inbox = new InMemoryInboxNudge();
|
||||
const { deps, store } = createDeps({
|
||||
const { auditEvents, deps, store } = createDeps({
|
||||
outboxStore: outbox,
|
||||
inboxNudge: inbox,
|
||||
busySignal: {
|
||||
|
|
@ -651,6 +672,14 @@ describe('MemberWorkSync use cases', () => {
|
|||
lastError: 'member_busy:active_tool_activity',
|
||||
nextAttemptAt: '2026-04-29T00:02:00.000Z',
|
||||
});
|
||||
expect(auditEvents).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
event: 'member_busy',
|
||||
reason: 'member_busy:active_tool_activity',
|
||||
}),
|
||||
])
|
||||
);
|
||||
});
|
||||
|
||||
it('uses bounded retry backoff when inbox delivery fails', async () => {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,113 @@
|
|||
import { mkdtemp, readFile, readdir, rm } from 'fs/promises';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { FileMemberWorkSyncAuditJournal } from '@features/member-work-sync/main/infrastructure/FileMemberWorkSyncAuditJournal';
|
||||
import { MemberWorkSyncStorePaths } from '@features/member-work-sync/main/infrastructure/MemberWorkSyncStorePaths';
|
||||
|
||||
function journalPath(root: string): string {
|
||||
return join(root, 'team-a', 'members', 'bob', '.member-work-sync', 'journal.jsonl');
|
||||
}
|
||||
|
||||
describe('FileMemberWorkSyncAuditJournal', () => {
|
||||
let root: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
root = await mkdtemp(join(tmpdir(), 'member-work-sync-audit-'));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('appends per-member JSONL audit events in order', async () => {
|
||||
const journal = new FileMemberWorkSyncAuditJournal(new MemberWorkSyncStorePaths(root));
|
||||
|
||||
await journal.append({
|
||||
timestamp: '2026-04-30T00:00:00.000Z',
|
||||
teamName: 'team-a',
|
||||
memberName: 'bob',
|
||||
event: 'reconcile_started',
|
||||
source: 'test',
|
||||
});
|
||||
await journal.append({
|
||||
timestamp: '2026-04-30T00:00:01.000Z',
|
||||
teamName: 'team-a',
|
||||
memberName: 'bob',
|
||||
event: 'status_written',
|
||||
source: 'test',
|
||||
agendaFingerprint: 'agenda:v1:abc',
|
||||
actionableCount: 1,
|
||||
});
|
||||
|
||||
const lines = (await readFile(journalPath(root), 'utf8')).trim().split('\n');
|
||||
expect(lines.map((line) => JSON.parse(line).event)).toEqual([
|
||||
'reconcile_started',
|
||||
'status_written',
|
||||
]);
|
||||
expect(JSON.parse(lines[1])).toMatchObject({
|
||||
schemaVersion: 1,
|
||||
teamName: 'team-a',
|
||||
memberName: 'bob',
|
||||
agendaFingerprint: 'agenda:v1:abc',
|
||||
});
|
||||
});
|
||||
|
||||
it('truncates previews and rotates bounded journals', async () => {
|
||||
const journal = new FileMemberWorkSyncAuditJournal(
|
||||
new MemberWorkSyncStorePaths(root),
|
||||
undefined,
|
||||
{ maxBytes: 200, rotatedFileCount: 2 }
|
||||
);
|
||||
|
||||
for (let index = 0; index < 8; index += 1) {
|
||||
await journal.append({
|
||||
timestamp: `2026-04-30T00:00:0${index}.000Z`,
|
||||
teamName: 'team-a',
|
||||
memberName: 'bob',
|
||||
event: 'nudge_skipped',
|
||||
source: 'test',
|
||||
reason: 'r'.repeat(500),
|
||||
diagnostics: ['d'.repeat(500)],
|
||||
metadata: { long: 'm'.repeat(500), ['__proto__']: 'safe' },
|
||||
taskRefs: [{ taskId: 't'.repeat(500), displayId: 'x'.repeat(500) }],
|
||||
messagePreview: 'x'.repeat(500),
|
||||
});
|
||||
}
|
||||
|
||||
const dirEntries = await readdir(join(root, 'team-a', 'members', 'bob', '.member-work-sync'));
|
||||
expect(dirEntries).toEqual(expect.arrayContaining(['journal.jsonl', 'journal.jsonl.1']));
|
||||
expect(dirEntries).not.toContain('journal.jsonl.3');
|
||||
|
||||
const latestLine = (await readFile(journalPath(root), 'utf8')).trim().split('\n').at(-1);
|
||||
const latest = JSON.parse(latestLine ?? '{}');
|
||||
expect(latest.messagePreview).toHaveLength(243);
|
||||
expect(latest.reason).toHaveLength(243);
|
||||
expect(latest.diagnostics[0]).toHaveLength(243);
|
||||
expect(latest.metadata.long).toHaveLength(243);
|
||||
expect(latest.metadata.__proto__).toBe('safe');
|
||||
expect(latest.taskRefs[0].taskId).toHaveLength(243);
|
||||
});
|
||||
|
||||
it('logs and swallows append failures', async () => {
|
||||
const logger = { debug: vi.fn(), warn: vi.fn(), error: vi.fn() };
|
||||
const paths = new MemberWorkSyncStorePaths(root);
|
||||
vi.spyOn(paths, 'ensureMemberWorkSyncDir').mockRejectedValue(new Error('boom'));
|
||||
const journal = new FileMemberWorkSyncAuditJournal(paths, logger);
|
||||
|
||||
await expect(
|
||||
journal.append({
|
||||
timestamp: '2026-04-30T00:00:00.000Z',
|
||||
teamName: 'team-a',
|
||||
memberName: 'bob',
|
||||
event: 'reconcile_started',
|
||||
source: 'test',
|
||||
})
|
||||
).resolves.toBeUndefined();
|
||||
expect(logger.warn).toHaveBeenCalledWith(
|
||||
'member work sync audit journal append failed',
|
||||
expect.objectContaining({ error: 'Error: boom' })
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
@ -9,6 +9,7 @@ import type {
|
|||
MemberWorkSyncNudgePayload,
|
||||
MemberWorkSyncStatus,
|
||||
} from '@features/member-work-sync/contracts';
|
||||
import type { MemberWorkSyncAuditEvent } from '@features/member-work-sync/core/application';
|
||||
|
||||
function makeStatus(overrides: Partial<MemberWorkSyncStatus>): MemberWorkSyncStatus {
|
||||
return {
|
||||
|
|
@ -58,6 +59,16 @@ function makeNudgePayload(overrides: Partial<MemberWorkSyncNudgePayload> = {}):
|
|||
};
|
||||
}
|
||||
|
||||
function memberWorkSyncDir(root: string, teamName: string, memberName: string): string {
|
||||
return join(
|
||||
root,
|
||||
teamName,
|
||||
'members',
|
||||
encodeURIComponent(memberName.trim().toLowerCase()),
|
||||
'.member-work-sync'
|
||||
);
|
||||
}
|
||||
|
||||
describe('JsonMemberWorkSyncStore', () => {
|
||||
let root: string;
|
||||
let store: JsonMemberWorkSyncStore;
|
||||
|
|
@ -83,6 +94,56 @@ describe('JsonMemberWorkSyncStore', () => {
|
|||
expect(entries.some((entry) => entry.startsWith('status.json.invalid.'))).toBe(true);
|
||||
});
|
||||
|
||||
it('writes status into member-scoped storage and keeps team metrics in an index', async () => {
|
||||
await store.write(makeStatus({ providerId: 'opencode' }));
|
||||
|
||||
const statusFile = JSON.parse(
|
||||
await readFile(join(memberWorkSyncDir(root, 'team-a', 'bob'), 'status.json'), 'utf8')
|
||||
);
|
||||
expect(statusFile).toMatchObject({
|
||||
schemaVersion: 2,
|
||||
status: {
|
||||
teamName: 'team-a',
|
||||
memberName: 'bob',
|
||||
providerId: 'opencode',
|
||||
},
|
||||
});
|
||||
|
||||
const metaFile = JSON.parse(
|
||||
await readFile(join(root, 'team-a', 'members', 'bob', 'member.meta.json'), 'utf8')
|
||||
);
|
||||
expect(metaFile).toMatchObject({
|
||||
schemaVersion: 1,
|
||||
memberName: 'bob',
|
||||
memberKey: 'bob',
|
||||
});
|
||||
|
||||
const metricsIndex = JSON.parse(
|
||||
await readFile(join(root, 'team-a', '.member-work-sync', 'indexes', 'metrics.json'), 'utf8')
|
||||
);
|
||||
expect(metricsIndex.members.bob).toMatchObject({
|
||||
memberName: 'bob',
|
||||
state: 'needs_sync',
|
||||
actionableCount: 1,
|
||||
});
|
||||
});
|
||||
|
||||
it('prefers member-scoped v2 status over legacy v1 status', async () => {
|
||||
await store.write(makeStatus({ state: 'caught_up', agenda: { ...makeStatus({}).agenda, items: [] } }));
|
||||
|
||||
const legacyStatusPath = join(root, 'team-a', '.member-work-sync', 'status.json');
|
||||
await mkdir(join(root, 'team-a', '.member-work-sync'), { recursive: true });
|
||||
await writeFile(
|
||||
legacyStatusPath,
|
||||
JSON.stringify({ schemaVersion: 1, members: { bob: makeStatus({ state: 'needs_sync' }) } }),
|
||||
'utf8'
|
||||
);
|
||||
|
||||
await expect(store.read({ teamName: 'team-a', memberName: 'bob' })).resolves.toMatchObject({
|
||||
state: 'caught_up',
|
||||
});
|
||||
});
|
||||
|
||||
it('deduplicates pending report intents and marks them processed', async () => {
|
||||
const request = {
|
||||
teamName: 'team-a',
|
||||
|
|
@ -114,12 +175,129 @@ describe('JsonMemberWorkSyncStore', () => {
|
|||
|
||||
expect(await store.listPendingReports('team-a')).toEqual([]);
|
||||
const file = JSON.parse(
|
||||
await readFile(join(root, 'team-a', '.member-work-sync', 'pending-reports.json'), 'utf8')
|
||||
await readFile(join(memberWorkSyncDir(root, 'team-a', 'bob'), 'reports.json'), 'utf8')
|
||||
);
|
||||
expect(file.intents[pending[0].id]).toMatchObject({
|
||||
status: 'accepted',
|
||||
resultCode: 'accepted',
|
||||
});
|
||||
const index = JSON.parse(
|
||||
await readFile(
|
||||
join(root, 'team-a', '.member-work-sync', 'indexes', 'pending-reports-index.json'),
|
||||
'utf8'
|
||||
)
|
||||
);
|
||||
expect(index.items[pending[0].id]).toMatchObject({
|
||||
memberName: 'bob',
|
||||
status: 'accepted',
|
||||
});
|
||||
});
|
||||
|
||||
it('repairs a missing pending-report index from member-scoped report files', async () => {
|
||||
const request = {
|
||||
teamName: 'team-a',
|
||||
memberName: 'bob',
|
||||
state: 'still_working' as const,
|
||||
agendaFingerprint: 'agenda:v1:abc',
|
||||
reportToken: 'wrs:v1.test',
|
||||
source: 'mcp' as const,
|
||||
};
|
||||
|
||||
await store.appendPendingReport(request, 'control_api_unavailable');
|
||||
await rm(join(root, 'team-a', '.member-work-sync', 'indexes', 'pending-reports-index.json'), {
|
||||
force: true,
|
||||
});
|
||||
|
||||
await expect(store.listPendingReports('team-a')).resolves.toHaveLength(1);
|
||||
const repaired = JSON.parse(
|
||||
await readFile(
|
||||
join(root, 'team-a', '.member-work-sync', 'indexes', 'pending-reports-index.json'),
|
||||
'utf8'
|
||||
)
|
||||
);
|
||||
expect(Object.values(repaired.items)).toEqual([
|
||||
expect.objectContaining({ memberName: 'bob', status: 'pending' }),
|
||||
]);
|
||||
});
|
||||
|
||||
it('repairs a stale pending-report index route from member-scoped report files', async () => {
|
||||
const bobRequest = {
|
||||
teamName: 'team-a',
|
||||
memberName: 'bob',
|
||||
state: 'still_working' as const,
|
||||
agendaFingerprint: 'agenda:v1:bob',
|
||||
reportToken: 'wrs:v1.bob',
|
||||
source: 'mcp' as const,
|
||||
};
|
||||
const tomRequest = {
|
||||
...bobRequest,
|
||||
memberName: 'tom',
|
||||
agendaFingerprint: 'agenda:v1:tom',
|
||||
reportToken: 'wrs:v1.tom',
|
||||
};
|
||||
|
||||
await store.appendPendingReport(bobRequest, 'control_api_unavailable');
|
||||
await store.appendPendingReport(tomRequest, 'control_api_unavailable');
|
||||
await writeFile(
|
||||
join(root, 'team-a', 'members', 'bob', '.member-work-sync', 'reports.json'),
|
||||
JSON.stringify({ schemaVersion: 2, intents: {} }),
|
||||
'utf8'
|
||||
);
|
||||
|
||||
const pending = await store.listPendingReports('team-a');
|
||||
expect(pending.map((intent) => intent.memberName)).toEqual(['tom']);
|
||||
const repaired = JSON.parse(
|
||||
await readFile(
|
||||
join(root, 'team-a', '.member-work-sync', 'indexes', 'pending-reports-index.json'),
|
||||
'utf8'
|
||||
)
|
||||
);
|
||||
expect(Object.values(repaired.items).map((item) => (item as { memberName: string }).memberName)).toEqual([
|
||||
'tom',
|
||||
]);
|
||||
});
|
||||
|
||||
it('repairs a partially missing pending-report index route from member-scoped report files', async () => {
|
||||
const bobRequest = {
|
||||
teamName: 'team-a',
|
||||
memberName: 'bob',
|
||||
state: 'still_working' as const,
|
||||
agendaFingerprint: 'agenda:v1:bob',
|
||||
reportToken: 'wrs:v1.bob',
|
||||
source: 'mcp' as const,
|
||||
};
|
||||
const tomRequest = {
|
||||
...bobRequest,
|
||||
memberName: 'tom',
|
||||
agendaFingerprint: 'agenda:v1:tom',
|
||||
reportToken: 'wrs:v1.tom',
|
||||
};
|
||||
|
||||
await store.appendPendingReport(bobRequest, 'control_api_unavailable');
|
||||
await store.appendPendingReport(tomRequest, 'control_api_unavailable');
|
||||
const indexPath = join(
|
||||
root,
|
||||
'team-a',
|
||||
'.member-work-sync',
|
||||
'indexes',
|
||||
'pending-reports-index.json'
|
||||
);
|
||||
const index = JSON.parse(await readFile(indexPath, 'utf8'));
|
||||
for (const [id, route] of Object.entries(index.items)) {
|
||||
if ((route as { memberName: string }).memberName === 'tom') {
|
||||
delete index.items[id];
|
||||
}
|
||||
}
|
||||
await writeFile(indexPath, JSON.stringify(index), 'utf8');
|
||||
|
||||
const pending = await store.listPendingReports('team-a');
|
||||
expect(pending.map((intent) => intent.memberName).sort()).toEqual(['bob', 'tom']);
|
||||
const repaired = JSON.parse(await readFile(indexPath, 'utf8'));
|
||||
expect(
|
||||
Object.values(repaired.items)
|
||||
.map((item) => (item as { memberName: string }).memberName)
|
||||
.sort()
|
||||
).toEqual(['bob', 'tom']);
|
||||
});
|
||||
|
||||
it('records bounded shadow metrics from status writes', async () => {
|
||||
|
|
@ -263,7 +441,7 @@ describe('JsonMemberWorkSyncStore', () => {
|
|||
teamName: 'team-a',
|
||||
claimedBy: 'dispatcher-a',
|
||||
nowIso: '2026-04-29T00:01:00.000Z',
|
||||
limit: 5,
|
||||
limit: 1,
|
||||
});
|
||||
expect(claimed).toHaveLength(1);
|
||||
expect(claimed[0]).toMatchObject({
|
||||
|
|
@ -305,12 +483,287 @@ describe('JsonMemberWorkSyncStore', () => {
|
|||
});
|
||||
|
||||
const file = JSON.parse(
|
||||
await readFile(join(root, 'team-a', '.member-work-sync', 'outbox.json'), 'utf8')
|
||||
await readFile(join(memberWorkSyncDir(root, 'team-a', 'bob'), 'outbox.json'), 'utf8')
|
||||
);
|
||||
expect(file.items[input.id]).toMatchObject({
|
||||
status: 'delivered',
|
||||
deliveredMessageId: 'message-1',
|
||||
attemptGeneration: 2,
|
||||
});
|
||||
const index = JSON.parse(
|
||||
await readFile(join(root, 'team-a', '.member-work-sync', 'indexes', 'outbox-index.json'), 'utf8')
|
||||
);
|
||||
expect(index.items[input.id]).toMatchObject({
|
||||
memberName: 'bob',
|
||||
status: 'delivered',
|
||||
});
|
||||
});
|
||||
|
||||
it('claims due outbox items from the index without scanning unrelated member outboxes', async () => {
|
||||
const bobInput = {
|
||||
id: 'member-work-sync:team-a:bob:agenda:v1:abc',
|
||||
teamName: 'team-a',
|
||||
memberName: 'bob',
|
||||
agendaFingerprint: 'agenda:v1:abc',
|
||||
payloadHash: 'hash-a',
|
||||
payload: makeNudgePayload(),
|
||||
nowIso: '2026-04-29T00:00:00.000Z',
|
||||
};
|
||||
await store.ensurePending(bobInput);
|
||||
|
||||
await mkdir(join(root, 'team-a', 'members', 'tom', '.member-work-sync'), { recursive: true });
|
||||
await writeFile(
|
||||
join(root, 'team-a', 'members', 'tom', 'member.meta.json'),
|
||||
JSON.stringify({
|
||||
schemaVersion: 1,
|
||||
memberName: 'tom',
|
||||
memberKey: 'tom',
|
||||
updatedAt: '2026-04-29T00:00:00.000Z',
|
||||
}),
|
||||
'utf8'
|
||||
);
|
||||
await writeFile(
|
||||
join(root, 'team-a', 'members', 'tom', '.member-work-sync', 'outbox.json'),
|
||||
JSON.stringify({
|
||||
schemaVersion: 2,
|
||||
items: {
|
||||
'member-work-sync:team-a:tom:agenda:v1:other': {
|
||||
...bobInput,
|
||||
id: 'member-work-sync:team-a:tom:agenda:v1:other',
|
||||
memberName: 'tom',
|
||||
status: 'pending',
|
||||
attemptGeneration: 0,
|
||||
createdAt: '2026-04-29T00:00:00.000Z',
|
||||
updatedAt: '2026-04-29T00:00:00.000Z',
|
||||
},
|
||||
},
|
||||
}),
|
||||
'utf8'
|
||||
);
|
||||
|
||||
const claimed = await store.claimDue({
|
||||
teamName: 'team-a',
|
||||
claimedBy: 'dispatcher-a',
|
||||
nowIso: '2026-04-29T00:01:00.000Z',
|
||||
limit: 1,
|
||||
});
|
||||
expect(claimed.map((item) => item.memberName)).toEqual(['bob']);
|
||||
});
|
||||
|
||||
it('repairs a missing outbox index from member-scoped outbox files for delivered counts', async () => {
|
||||
const input = {
|
||||
id: 'member-work-sync:team-a:bob:agenda:v1:abc',
|
||||
teamName: 'team-a',
|
||||
memberName: 'bob',
|
||||
agendaFingerprint: 'agenda:v1:abc',
|
||||
payloadHash: 'hash-a',
|
||||
payload: makeNudgePayload(),
|
||||
nowIso: '2026-04-29T00:00:00.000Z',
|
||||
};
|
||||
await store.ensurePending(input);
|
||||
const [claimed] = await store.claimDue({
|
||||
teamName: 'team-a',
|
||||
claimedBy: 'dispatcher-a',
|
||||
nowIso: '2026-04-29T00:01:00.000Z',
|
||||
limit: 1,
|
||||
});
|
||||
await store.markDelivered({
|
||||
teamName: 'team-a',
|
||||
id: input.id,
|
||||
attemptGeneration: claimed.attemptGeneration,
|
||||
deliveredMessageId: 'message-1',
|
||||
nowIso: '2026-04-29T00:02:00.000Z',
|
||||
});
|
||||
await rm(join(root, 'team-a', '.member-work-sync', 'indexes', 'outbox-index.json'), {
|
||||
force: true,
|
||||
});
|
||||
|
||||
await expect(
|
||||
store.countRecentDelivered({
|
||||
teamName: 'team-a',
|
||||
memberName: 'bob',
|
||||
sinceIso: '2026-04-29T00:00:00.000Z',
|
||||
})
|
||||
).resolves.toBe(1);
|
||||
});
|
||||
|
||||
it('counts delivered nudges from the member outbox when the outbox index is partially stale', async () => {
|
||||
const bobInput = {
|
||||
id: 'member-work-sync:team-a:bob:agenda:v1:abc',
|
||||
teamName: 'team-a',
|
||||
memberName: 'bob',
|
||||
agendaFingerprint: 'agenda:v1:abc',
|
||||
payloadHash: 'hash-a',
|
||||
payload: makeNudgePayload(),
|
||||
nowIso: '2026-04-29T00:00:00.000Z',
|
||||
};
|
||||
const tomInput = {
|
||||
...bobInput,
|
||||
id: 'member-work-sync:team-a:tom:agenda:v1:def',
|
||||
memberName: 'tom',
|
||||
payload: makeNudgePayload({ to: 'tom' }),
|
||||
};
|
||||
await store.ensurePending(bobInput);
|
||||
await store.ensurePending(tomInput);
|
||||
const [claimedBob] = await store.claimDue({
|
||||
teamName: 'team-a',
|
||||
claimedBy: 'dispatcher-a',
|
||||
nowIso: '2026-04-29T00:01:00.000Z',
|
||||
limit: 1,
|
||||
});
|
||||
await store.markDelivered({
|
||||
teamName: 'team-a',
|
||||
id: bobInput.id,
|
||||
attemptGeneration: claimedBob.attemptGeneration,
|
||||
deliveredMessageId: 'message-1',
|
||||
nowIso: '2026-04-29T00:02:00.000Z',
|
||||
});
|
||||
|
||||
const indexPath = join(root, 'team-a', '.member-work-sync', 'indexes', 'outbox-index.json');
|
||||
const index = JSON.parse(await readFile(indexPath, 'utf8'));
|
||||
delete index.items[bobInput.id];
|
||||
await writeFile(indexPath, JSON.stringify(index), 'utf8');
|
||||
|
||||
await expect(
|
||||
store.countRecentDelivered({
|
||||
teamName: 'team-a',
|
||||
memberName: 'bob',
|
||||
sinceIso: '2026-04-29T00:00:00.000Z',
|
||||
})
|
||||
).resolves.toBe(1);
|
||||
const repaired = JSON.parse(await readFile(indexPath, 'utf8'));
|
||||
expect(repaired.items[bobInput.id]).toMatchObject({ memberName: 'bob', status: 'delivered' });
|
||||
});
|
||||
|
||||
it('repairs stale due outbox index routes before persisting claim results', async () => {
|
||||
const bobInput = {
|
||||
id: 'member-work-sync:team-a:bob:agenda:v1:abc',
|
||||
teamName: 'team-a',
|
||||
memberName: 'bob',
|
||||
agendaFingerprint: 'agenda:v1:abc',
|
||||
payloadHash: 'hash-a',
|
||||
payload: makeNudgePayload(),
|
||||
nowIso: '2026-04-29T00:00:00.000Z',
|
||||
};
|
||||
const tomInput = {
|
||||
...bobInput,
|
||||
id: 'member-work-sync:team-a:tom:agenda:v1:def',
|
||||
memberName: 'tom',
|
||||
payload: makeNudgePayload({ to: 'tom' }),
|
||||
};
|
||||
await store.ensurePending(bobInput);
|
||||
await store.ensurePending(tomInput);
|
||||
await writeFile(
|
||||
join(root, 'team-a', 'members', 'bob', '.member-work-sync', 'outbox.json'),
|
||||
JSON.stringify({ schemaVersion: 2, items: {} }),
|
||||
'utf8'
|
||||
);
|
||||
|
||||
const claimed = await store.claimDue({
|
||||
teamName: 'team-a',
|
||||
claimedBy: 'dispatcher-a',
|
||||
nowIso: '2026-04-29T00:01:00.000Z',
|
||||
limit: 5,
|
||||
});
|
||||
expect(claimed.map((item) => item.memberName)).toEqual(['tom']);
|
||||
const repaired = JSON.parse(
|
||||
await readFile(join(root, 'team-a', '.member-work-sync', 'indexes', 'outbox-index.json'), 'utf8')
|
||||
);
|
||||
expect(Object.values(repaired.items).map((item) => (item as { memberName: string }).memberName)).toEqual([
|
||||
'tom',
|
||||
]);
|
||||
});
|
||||
|
||||
it('repairs partially missing due outbox index routes before claiming', async () => {
|
||||
const bobInput = {
|
||||
id: 'member-work-sync:team-a:bob:agenda:v1:abc',
|
||||
teamName: 'team-a',
|
||||
memberName: 'bob',
|
||||
agendaFingerprint: 'agenda:v1:abc',
|
||||
payloadHash: 'hash-a',
|
||||
payload: makeNudgePayload(),
|
||||
nowIso: '2026-04-29T00:00:00.000Z',
|
||||
};
|
||||
const tomInput = {
|
||||
...bobInput,
|
||||
id: 'member-work-sync:team-a:tom:agenda:v1:def',
|
||||
memberName: 'tom',
|
||||
payload: makeNudgePayload({ to: 'tom' }),
|
||||
};
|
||||
await store.ensurePending(bobInput);
|
||||
await store.ensurePending(tomInput);
|
||||
const indexPath = join(root, 'team-a', '.member-work-sync', 'indexes', 'outbox-index.json');
|
||||
const index = JSON.parse(await readFile(indexPath, 'utf8'));
|
||||
delete index.items[tomInput.id];
|
||||
await writeFile(indexPath, JSON.stringify(index), 'utf8');
|
||||
|
||||
const claimed = await store.claimDue({
|
||||
teamName: 'team-a',
|
||||
claimedBy: 'dispatcher-a',
|
||||
nowIso: '2026-04-29T00:01:00.000Z',
|
||||
limit: 5,
|
||||
});
|
||||
expect(claimed.map((item) => item.memberName).sort()).toEqual(['bob', 'tom']);
|
||||
});
|
||||
|
||||
it('falls back to legacy v1 status and materializes legacy outbox during claim', async () => {
|
||||
const auditEvents: MemberWorkSyncAuditEvent[] = [];
|
||||
store = new JsonMemberWorkSyncStore(new MemberWorkSyncStorePaths(root), {
|
||||
auditJournal: {
|
||||
append: async (event) => {
|
||||
auditEvents.push(event);
|
||||
},
|
||||
},
|
||||
now: () => new Date('2026-04-29T00:02:00.000Z'),
|
||||
});
|
||||
const legacyStatusPath = join(root, 'team-a', '.member-work-sync', 'status.json');
|
||||
await mkdir(join(root, 'team-a', '.member-work-sync'), { recursive: true });
|
||||
await writeFile(
|
||||
legacyStatusPath,
|
||||
JSON.stringify({ schemaVersion: 1, members: { bob: makeStatus({}) } }),
|
||||
'utf8'
|
||||
);
|
||||
|
||||
await expect(store.read({ teamName: 'team-a', memberName: 'bob' })).resolves.toMatchObject({
|
||||
memberName: 'bob',
|
||||
state: 'needs_sync',
|
||||
});
|
||||
|
||||
const input = {
|
||||
id: 'member-work-sync:team-a:bob:agenda:v1:legacy',
|
||||
teamName: 'team-a',
|
||||
memberName: 'bob',
|
||||
agendaFingerprint: 'agenda:v1:legacy',
|
||||
payloadHash: 'hash-a',
|
||||
payload: makeNudgePayload(),
|
||||
status: 'pending' as const,
|
||||
attemptGeneration: 0,
|
||||
createdAt: '2026-04-29T00:00:00.000Z',
|
||||
updatedAt: '2026-04-29T00:00:00.000Z',
|
||||
};
|
||||
await writeFile(
|
||||
join(root, 'team-a', '.member-work-sync', 'outbox.json'),
|
||||
JSON.stringify({ schemaVersion: 1, items: { [input.id]: input } }),
|
||||
'utf8'
|
||||
);
|
||||
|
||||
const claimed = await store.claimDue({
|
||||
teamName: 'team-a',
|
||||
claimedBy: 'dispatcher-a',
|
||||
nowIso: '2026-04-29T00:01:00.000Z',
|
||||
limit: 1,
|
||||
});
|
||||
expect(claimed).toHaveLength(1);
|
||||
expect(
|
||||
JSON.parse(await readFile(join(memberWorkSyncDir(root, 'team-a', 'bob'), 'outbox.json'), 'utf8'))
|
||||
.items[input.id]
|
||||
).toMatchObject({ status: 'claimed' });
|
||||
expect(auditEvents.map((event) => `${event.event}:${event.reason}`)).toEqual(
|
||||
expect.arrayContaining([
|
||||
'legacy_fallback_used:status_v1',
|
||||
'index_repaired:outbox',
|
||||
'legacy_fallback_used:outbox_v1',
|
||||
])
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -13,12 +13,18 @@ describe('MemberWorkSyncEventQueue', () => {
|
|||
|
||||
it('coalesces duplicate member events into one queue reconcile', async () => {
|
||||
const reconciles: unknown[] = [];
|
||||
const auditEvents: string[] = [];
|
||||
const queue = new MemberWorkSyncEventQueue({
|
||||
quietWindowMs: 100,
|
||||
reconcile: async (request, context) => {
|
||||
reconciles.push({ request, context });
|
||||
},
|
||||
isTeamActive: () => true,
|
||||
auditJournal: {
|
||||
append: async (event) => {
|
||||
auditEvents.push(event.event);
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
queue.enqueue({ teamName: 'team-a', memberName: 'bob', triggerReason: 'task_changed' });
|
||||
|
|
@ -35,6 +41,7 @@ describe('MemberWorkSyncEventQueue', () => {
|
|||
},
|
||||
});
|
||||
expect(queue.getDiagnostics()).toMatchObject({ reconciled: 1, coalesced: 1 });
|
||||
expect(auditEvents).toEqual(['queue_enqueued', 'queue_coalesced', 'queue_reconciled']);
|
||||
await queue.stop();
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import { NodeHashAdapter } from '@features/member-work-sync/main/infrastructure/
|
|||
import { OpenCodeTurnSettledPayloadNormalizer } from '@features/member-work-sync/main/infrastructure/OpenCodeTurnSettledPayloadNormalizer';
|
||||
|
||||
import type {
|
||||
MemberWorkSyncAuditEvent,
|
||||
RuntimeTurnSettledClaimedPayload,
|
||||
RuntimeTurnSettledEventStorePort,
|
||||
RuntimeTurnSettledInvalidResult,
|
||||
|
|
@ -56,6 +57,7 @@ describe('RuntimeTurnSettledIngestor', () => {
|
|||
resolve: vi.fn(async () => ({ ok: true as const, teamName: 'team-a', memberName: 'alice' })),
|
||||
};
|
||||
const enqueueRuntimeTurnSettled = vi.fn();
|
||||
const auditEvents: MemberWorkSyncAuditEvent[] = [];
|
||||
|
||||
const ingestor = new RuntimeTurnSettledIngestor({
|
||||
eventStore: store,
|
||||
|
|
@ -63,6 +65,11 @@ describe('RuntimeTurnSettledIngestor', () => {
|
|||
targetResolver: resolver,
|
||||
reconcileQueue: { enqueueRuntimeTurnSettled },
|
||||
clock: { now: () => new Date('2026-04-29T12:01:00.000Z') },
|
||||
auditJournal: {
|
||||
append: async (event) => {
|
||||
auditEvents.push(event);
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
await expect(ingestor.drainPending()).resolves.toEqual({
|
||||
|
|
@ -185,6 +192,7 @@ describe('RuntimeTurnSettledIngestor', () => {
|
|||
resolve: vi.fn(async () => ({ ok: true as const, teamName: 'team-a', memberName: 'jack' })),
|
||||
};
|
||||
const enqueueRuntimeTurnSettled = vi.fn();
|
||||
const auditEvents: MemberWorkSyncAuditEvent[] = [];
|
||||
|
||||
const ingestor = new RuntimeTurnSettledIngestor({
|
||||
eventStore: store,
|
||||
|
|
@ -192,6 +200,11 @@ describe('RuntimeTurnSettledIngestor', () => {
|
|||
targetResolver: resolver,
|
||||
reconcileQueue: { enqueueRuntimeTurnSettled },
|
||||
clock: { now: () => new Date('2026-04-29T12:01:00.000Z') },
|
||||
auditJournal: {
|
||||
append: async (event) => {
|
||||
auditEvents.push(event);
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
await expect(ingestor.drainPending()).resolves.toMatchObject({
|
||||
|
|
@ -220,6 +233,10 @@ describe('RuntimeTurnSettledIngestor', () => {
|
|||
teamName: 'team-a',
|
||||
memberName: 'jack',
|
||||
});
|
||||
expect(auditEvents.map((event) => event.event)).toEqual([
|
||||
'turn_settled_claimed',
|
||||
'turn_settled_resolved',
|
||||
]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
|
|
|
|||
|
|
@ -264,6 +264,9 @@ describe('OpenCodeRuntimeManifestEvidenceReader migration', () => {
|
|||
).resolves.toEqual({
|
||||
laneDirectoryExists: false,
|
||||
hasStateOnDisk: false,
|
||||
hasRuntimeEvidenceOnDisk: false,
|
||||
manifestEntryCount: null,
|
||||
manifestUpdatedAt: null,
|
||||
fileNames: [],
|
||||
});
|
||||
|
||||
|
|
@ -293,6 +296,117 @@ describe('OpenCodeRuntimeManifestEvidenceReader migration', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('degrades an active lane that only has a stale empty runtime manifest', async () => {
|
||||
const teamName = 'team-empty-manifest';
|
||||
const laneId = 'secondary:opencode:bob';
|
||||
|
||||
await upsertOpenCodeRuntimeLaneIndexEntry({
|
||||
teamsBasePath: tempDir,
|
||||
teamName,
|
||||
laneId,
|
||||
state: 'active',
|
||||
});
|
||||
await setOpenCodeRuntimeActiveRunManifest({
|
||||
teamsBasePath: tempDir,
|
||||
teamName,
|
||||
laneId,
|
||||
runId: 'run-empty',
|
||||
clock: () => new Date('2026-04-22T09:55:00.000Z'),
|
||||
});
|
||||
await fs.writeFile(
|
||||
getOpenCodeLaneScopedRuntimeFilePath({
|
||||
teamsBasePath: tempDir,
|
||||
teamName,
|
||||
laneId,
|
||||
fileName: 'opencode-prompt-delivery-ledger.json',
|
||||
}),
|
||||
JSON.stringify({ records: [] }),
|
||||
'utf8'
|
||||
);
|
||||
|
||||
await expect(
|
||||
inspectOpenCodeRuntimeLaneStorage({
|
||||
teamsBasePath: tempDir,
|
||||
teamName,
|
||||
laneId,
|
||||
})
|
||||
).resolves.toMatchObject({
|
||||
laneDirectoryExists: true,
|
||||
hasStateOnDisk: true,
|
||||
hasRuntimeEvidenceOnDisk: false,
|
||||
manifestEntryCount: 0,
|
||||
fileNames: ['manifest.json', 'opencode-prompt-delivery-ledger.json'],
|
||||
});
|
||||
|
||||
const result = await recoverStaleOpenCodeRuntimeLaneIndexEntry({
|
||||
teamsBasePath: tempDir,
|
||||
teamName,
|
||||
laneId,
|
||||
clock: () => now,
|
||||
emptyLaneStaleAfterMs: 150_000,
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
stale: true,
|
||||
degraded: true,
|
||||
diagnostics: [
|
||||
`OpenCode lane ${laneId} is marked active in lanes.json, but its runtime manifest has no committed runtime evidence after launch grace.`,
|
||||
],
|
||||
});
|
||||
await expect(readOpenCodeRuntimeLaneIndex(tempDir, teamName)).resolves.toMatchObject({
|
||||
lanes: {
|
||||
[laneId]: {
|
||||
laneId,
|
||||
state: 'degraded',
|
||||
diagnostics: [
|
||||
`OpenCode lane ${laneId} is marked active in lanes.json, but its runtime manifest has no committed runtime evidence after launch grace.`,
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('does not degrade a fresh active lane while the empty runtime manifest is still inside launch grace', async () => {
|
||||
const teamName = 'team-fresh-empty-manifest';
|
||||
const laneId = 'secondary:opencode:bob';
|
||||
|
||||
await upsertOpenCodeRuntimeLaneIndexEntry({
|
||||
teamsBasePath: tempDir,
|
||||
teamName,
|
||||
laneId,
|
||||
state: 'active',
|
||||
});
|
||||
await setOpenCodeRuntimeActiveRunManifest({
|
||||
teamsBasePath: tempDir,
|
||||
teamName,
|
||||
laneId,
|
||||
runId: 'run-fresh',
|
||||
clock: () => new Date('2026-04-22T09:59:00.000Z'),
|
||||
});
|
||||
|
||||
const result = await recoverStaleOpenCodeRuntimeLaneIndexEntry({
|
||||
teamsBasePath: tempDir,
|
||||
teamName,
|
||||
laneId,
|
||||
clock: () => now,
|
||||
emptyLaneStaleAfterMs: 150_000,
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
stale: false,
|
||||
degraded: false,
|
||||
diagnostics: [],
|
||||
});
|
||||
await expect(readOpenCodeRuntimeLaneIndex(tempDir, teamName)).resolves.toMatchObject({
|
||||
lanes: {
|
||||
[laneId]: {
|
||||
laneId,
|
||||
state: 'active',
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('quarantines malformed lanes.json and falls back to an empty index', async () => {
|
||||
const teamName = 'team-zeta';
|
||||
const runtimeDir = getOpenCodeTeamRuntimeDirectory(tempDir, teamName);
|
||||
|
|
|
|||
|
|
@ -620,7 +620,7 @@ describe('OpenCodeTeamRuntimeAdapter', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('treats materialized bridge members without session or pid as accepted but not alive', async () => {
|
||||
it('does not treat bridge members without session or pid as runtime candidates', async () => {
|
||||
const launchOpenCodeTeam = vi.fn(
|
||||
async () =>
|
||||
({
|
||||
|
|
@ -648,10 +648,10 @@ describe('OpenCodeTeamRuntimeAdapter', () => {
|
|||
|
||||
expect(result.members.alice).toMatchObject({
|
||||
launchState: 'runtime_pending_bootstrap',
|
||||
agentToolAccepted: true,
|
||||
agentToolAccepted: false,
|
||||
runtimeAlive: false,
|
||||
livenessKind: 'runtime_process_candidate',
|
||||
runtimeDiagnostic: 'OpenCode session exists without verified runtime pid',
|
||||
livenessKind: 'registered_only',
|
||||
runtimeDiagnostic: 'OpenCode bridge did not report a runtime session or pid for this member',
|
||||
});
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -780,7 +780,7 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
const second = await secondPromise;
|
||||
expect(second.runId).toBeTruthy();
|
||||
expect(second.runId).not.toBe(firstRunId);
|
||||
await waitForCondition(() => adapter.launchInputs.length === 2);
|
||||
await waitForCondition(() => adapter.launchInputs.length === 1);
|
||||
|
||||
expect(svc.isTeamAlive(teamName)).toBe(true);
|
||||
expect(svc.getAliveTeams()).toEqual([teamName]);
|
||||
|
|
@ -1372,7 +1372,7 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
() => undefined
|
||||
);
|
||||
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 2);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 1);
|
||||
const cancelledRunId = adapter.pendingLaunchInputs.find(
|
||||
(input) => input.teamName === cancelledTeamName
|
||||
)?.runId;
|
||||
|
|
@ -1461,7 +1461,7 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
() => undefined
|
||||
);
|
||||
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 2);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 1);
|
||||
const cancelledRunId = adapter.pendingLaunchInputs.find(
|
||||
(input) => input.teamName === cancelledTeamName
|
||||
)?.runId;
|
||||
|
|
@ -1805,7 +1805,7 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
},
|
||||
() => undefined
|
||||
);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 2);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 1);
|
||||
const firstRunId = adapter.pendingLaunchInputs.find(
|
||||
(input) => input.teamName === firstTeamName
|
||||
)?.runId;
|
||||
|
|
@ -1817,7 +1817,7 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
|
||||
svc.stopAllTeams();
|
||||
|
||||
await waitForCondition(() => adapter.stopInputs.length === 2);
|
||||
await waitForCondition(() => adapter.stopInputs.length === 1);
|
||||
expect(adapter.stopInputs.map((input) => input.teamName).sort()).toEqual([
|
||||
firstTeamName,
|
||||
secondTeamName,
|
||||
|
|
@ -1867,7 +1867,7 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
expect(svc.getAliveTeams()).toEqual([]);
|
||||
|
||||
adapter.releaseStops();
|
||||
await waitForCondition(() => adapter.stopInputs.length === 2);
|
||||
await waitForCondition(() => adapter.stopInputs.length === 1);
|
||||
await expect(
|
||||
readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), firstTeamName)
|
||||
).resolves.toMatchObject({
|
||||
|
|
@ -1909,7 +1909,7 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
},
|
||||
() => undefined
|
||||
);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 2);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 1);
|
||||
const firstRunId = adapter.pendingLaunchInputs.find(
|
||||
(input) => input.teamName === firstTeamName
|
||||
)?.runId;
|
||||
|
|
@ -1921,7 +1921,7 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
|
||||
svc.stopAllTeams();
|
||||
|
||||
await waitForCondition(() => adapter.stopInputs.length === 2);
|
||||
await waitForCondition(() => adapter.stopInputs.length === 1);
|
||||
expect(adapter.stopInputs.map((input) => input.teamName).sort()).toEqual([
|
||||
firstTeamName,
|
||||
secondTeamName,
|
||||
|
|
@ -1941,7 +1941,7 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
adapter.releaseLaunches();
|
||||
await expect(firstPromise).resolves.toEqual({ runId: firstRunId });
|
||||
await expect(secondPromise).resolves.toEqual({ runId: secondRunId });
|
||||
await waitForCondition(() => adapter.launchInputs.length === 2);
|
||||
await waitForCondition(() => adapter.launchInputs.length === 1);
|
||||
|
||||
expect(svc.getAliveTeams()).toEqual([]);
|
||||
await expect(
|
||||
|
|
@ -2070,19 +2070,18 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
trackLiveRun(svc, run);
|
||||
|
||||
await (svc as any).launchMixedSecondaryLaneIfNeeded(run);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 2);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 1);
|
||||
|
||||
svc.stopAllTeams();
|
||||
|
||||
await waitForCondition(() => adapter.stopInputs.length === 2);
|
||||
await waitForCondition(() => adapter.stopInputs.length === 1);
|
||||
expect(adapter.stopInputs.map((input) => input.laneId).sort()).toEqual([
|
||||
'secondary:opencode:bob',
|
||||
'secondary:opencode:tom',
|
||||
]);
|
||||
expect(svc.isTeamAlive(teamName)).toBe(false);
|
||||
|
||||
adapter.releaseLaunches();
|
||||
await waitForCondition(() => adapter.rejectedLaunchCount === 2);
|
||||
await waitForCondition(() => adapter.rejectedLaunchCount === 1);
|
||||
|
||||
await expect(readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName)).resolves.toMatchObject(
|
||||
{
|
||||
|
|
@ -2113,19 +2112,18 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
trackLiveRun(svc, cancelledRun);
|
||||
|
||||
await (svc as any).launchMixedSecondaryLaneIfNeeded(cancelledRun);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 2);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 1);
|
||||
|
||||
svc.stopAllTeams();
|
||||
|
||||
await waitForCondition(() => adapter.stopInputs.length === 2);
|
||||
await waitForCondition(() => adapter.stopInputs.length === 1);
|
||||
expect(adapter.stopInputs.map((input) => input.laneId).sort()).toEqual([
|
||||
'secondary:opencode:bob',
|
||||
'secondary:opencode:tom',
|
||||
]);
|
||||
expect(svc.isTeamAlive(teamName)).toBe(false);
|
||||
|
||||
adapter.releaseLaunches();
|
||||
await waitForCondition(() => adapter.launchInputs.length === 2);
|
||||
await waitForCondition(() => adapter.launchInputs.length === 1);
|
||||
await expect(readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName)).resolves.toMatchObject(
|
||||
{
|
||||
lanes: {},
|
||||
|
|
@ -2139,7 +2137,7 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
trackLiveRun(svc, freshRun);
|
||||
|
||||
await (svc as any).launchMixedSecondaryLaneIfNeeded(freshRun);
|
||||
await waitForCondition(() => adapter.launchInputs.length === 4);
|
||||
await waitForCondition(() => adapter.launchInputs.length === 3);
|
||||
await waitForCondition(() =>
|
||||
freshRun.mixedSecondaryLanes.every((lane: { state: string }) => lane.state === 'finished')
|
||||
);
|
||||
|
|
@ -2200,27 +2198,23 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
|
||||
await (svc as any).launchMixedSecondaryLaneIfNeeded(firstRun);
|
||||
await (svc as any).launchMixedSecondaryLaneIfNeeded(secondRun);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 4);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 1);
|
||||
|
||||
svc.stopAllTeams();
|
||||
|
||||
await waitForCondition(() => adapter.stopInputs.length === 4);
|
||||
await waitForCondition(() => adapter.stopInputs.length === 1);
|
||||
expect(adapter.stopInputs.map((input) => input.teamName).sort()).toEqual([
|
||||
firstTeamName,
|
||||
firstTeamName,
|
||||
secondTeamName,
|
||||
secondTeamName,
|
||||
]);
|
||||
expect(adapter.stopInputs.map((input) => input.laneId).sort()).toEqual([
|
||||
'secondary:opencode:bob',
|
||||
'secondary:opencode:bob',
|
||||
'secondary:opencode:tom',
|
||||
'secondary:opencode:tom',
|
||||
]);
|
||||
expect(svc.getAliveTeams()).toEqual([]);
|
||||
|
||||
adapter.releaseLaunches();
|
||||
await waitForCondition(() => adapter.rejectedLaunchCount === 4);
|
||||
await waitForCondition(() => adapter.rejectedLaunchCount === 1);
|
||||
|
||||
await expect(
|
||||
readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), firstTeamName)
|
||||
|
|
@ -5619,14 +5613,14 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
const statuses = await svc.getMemberSpawnStatuses(teamName);
|
||||
|
||||
expect(statuses.expectedMembers).toEqual(['alice', 'bob']);
|
||||
expect(statuses.teamLaunchState).toBe('clean_success');
|
||||
expect(statuses.teamLaunchState).toBe('partial_pending');
|
||||
expect(statuses.statuses.bob).toMatchObject({
|
||||
status: 'online',
|
||||
launchState: 'confirmed_alive',
|
||||
bootstrapConfirmed: true,
|
||||
status: 'spawning',
|
||||
launchState: 'starting',
|
||||
bootstrapConfirmed: false,
|
||||
hardFailure: false,
|
||||
livenessSource: 'heartbeat',
|
||||
});
|
||||
expect(statuses.statuses.bob?.livenessSource).not.toBe('heartbeat');
|
||||
expect(statuses.statuses['bob-2']).toBeUndefined();
|
||||
});
|
||||
|
||||
|
|
@ -6387,14 +6381,14 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
const statuses = await new TeamProvisioningService().getMemberSpawnStatuses(teamName);
|
||||
|
||||
expect(statuses.expectedMembers).toEqual(['alice', 'bob']);
|
||||
expect(statuses.teamLaunchState).toBe('clean_success');
|
||||
expect(statuses.teamLaunchState).toBe('partial_pending');
|
||||
expect(statuses.statuses.bob).toMatchObject({
|
||||
status: 'online',
|
||||
launchState: 'confirmed_alive',
|
||||
bootstrapConfirmed: true,
|
||||
status: 'waiting',
|
||||
launchState: 'runtime_pending_bootstrap',
|
||||
bootstrapConfirmed: false,
|
||||
hardFailure: false,
|
||||
lastHeartbeatAt: newerSignalAt,
|
||||
});
|
||||
expect(statuses.statuses.bob?.lastHeartbeatAt).not.toBe(newerSignalAt);
|
||||
expect(statuses.statuses['bob-2']).toBeUndefined();
|
||||
});
|
||||
|
||||
|
|
@ -6672,14 +6666,14 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
const statuses = await new TeamProvisioningService().getMemberSpawnStatuses(teamName);
|
||||
|
||||
expect(statuses.expectedMembers).toEqual(['alice', 'bob']);
|
||||
expect(statuses.teamLaunchState).toBe('clean_success');
|
||||
expect(statuses.teamLaunchState).toBe('partial_pending');
|
||||
expect(statuses.statuses.bob).toMatchObject({
|
||||
status: 'online',
|
||||
launchState: 'confirmed_alive',
|
||||
bootstrapConfirmed: true,
|
||||
status: 'waiting',
|
||||
launchState: 'runtime_pending_bootstrap',
|
||||
bootstrapConfirmed: false,
|
||||
hardFailure: false,
|
||||
lastHeartbeatAt: signalAt,
|
||||
});
|
||||
expect(statuses.statuses.bob?.lastHeartbeatAt).not.toBe(signalAt);
|
||||
expect(statuses.statuses['bob-2']).toBeUndefined();
|
||||
});
|
||||
|
||||
|
|
@ -10076,7 +10070,7 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
|
||||
await (svc as any).launchMixedSecondaryLaneIfNeeded(cancelledRun);
|
||||
await (svc as any).launchMixedSecondaryLaneIfNeeded(survivingRun);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 4);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 1);
|
||||
|
||||
await svc.cancelProvisioning(cancelledRun.runId);
|
||||
|
||||
|
|
@ -10086,7 +10080,7 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
expect(adapter.stopInputs.some((input) => input.teamName === survivingTeamName)).toBe(false);
|
||||
|
||||
adapter.releaseLaunches();
|
||||
await waitForCondition(() => adapter.launchInputs.length === 4);
|
||||
await waitForCondition(() => adapter.launchInputs.length === 3);
|
||||
await waitForCondition(() =>
|
||||
survivingRun.mixedSecondaryLanes.every((lane: { state: string }) => lane.state === 'finished')
|
||||
);
|
||||
|
|
@ -10103,7 +10097,7 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
trackLiveRun(svc, freshRun);
|
||||
|
||||
await (svc as any).launchMixedSecondaryLaneIfNeeded(freshRun);
|
||||
await waitForCondition(() => adapter.launchInputs.length === 6);
|
||||
await waitForCondition(() => adapter.launchInputs.length === 5);
|
||||
await waitForCondition(() =>
|
||||
freshRun.mixedSecondaryLanes.every((lane: { state: string }) => lane.state === 'finished')
|
||||
);
|
||||
|
|
@ -10201,19 +10195,19 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
|
||||
await (svc as any).launchMixedSecondaryLaneIfNeeded(cancelledRun);
|
||||
await (svc as any).launchMixedSecondaryLaneIfNeeded(survivingRun);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 4);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 2);
|
||||
|
||||
await svc.cancelProvisioning(cancelledRun.runId);
|
||||
|
||||
await waitForCondition(
|
||||
() => adapter.stopInputs.filter((input) => input.teamName === cancelledTeamName).length === 2
|
||||
() => adapter.stopInputs.filter((input) => input.teamName === cancelledTeamName).length === 1
|
||||
);
|
||||
expect(adapter.stopInputs.some((input) => input.teamName === survivingTeamName)).toBe(false);
|
||||
expect(svc.isTeamAlive(cancelledTeamName)).toBe(false);
|
||||
expect(svc.isTeamAlive(survivingTeamName)).toBe(true);
|
||||
|
||||
adapter.releaseLaunches();
|
||||
await waitForCondition(() => adapter.launchInputs.length === 4);
|
||||
await waitForCondition(() => adapter.launchInputs.length === 3);
|
||||
await waitForCondition(() =>
|
||||
survivingRun.mixedSecondaryLanes.every((lane: { state: string }) => lane.state === 'finished')
|
||||
);
|
||||
|
|
@ -11004,7 +10998,7 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
await waitForCondition(() => adapter.launchInputs.length === 2);
|
||||
|
||||
svc.stopTeam(teamName);
|
||||
await waitForCondition(() => adapter.stopInputs.length === 2);
|
||||
await waitForCondition(() => adapter.stopInputs.length === 1);
|
||||
await waitForCondition(() => !svc.isTeamAlive(teamName));
|
||||
expect((await readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName)).lanes).toEqual({});
|
||||
|
||||
|
|
@ -12777,18 +12771,17 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
trackLiveRun(svc, currentRun);
|
||||
|
||||
await (svc as any).launchMixedSecondaryLaneIfNeeded(currentRun);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 2);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 1);
|
||||
|
||||
svc.stopTeam(teamName);
|
||||
|
||||
await waitForCondition(() => adapter.stopInputs.length === 2);
|
||||
await waitForCondition(() => adapter.stopInputs.length === 1);
|
||||
expectDirectChildKillCount(staleKillCount, 0);
|
||||
expectDirectChildKillCount(currentKillCount, 1);
|
||||
expect(staleRun.cancelRequested).toBe(false);
|
||||
expect(currentRun.cancelRequested).toBe(true);
|
||||
expect(adapter.stopInputs.map((input) => input.laneId).sort()).toEqual([
|
||||
'secondary:opencode:bob',
|
||||
'secondary:opencode:tom',
|
||||
]);
|
||||
expect(await svc.getRuntimeState(teamName)).toMatchObject({
|
||||
teamName,
|
||||
|
|
@ -12835,7 +12828,7 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
trackLiveRun(svc, currentRun);
|
||||
|
||||
await (svc as any).launchMixedSecondaryLaneIfNeeded(currentRun);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 2);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 1);
|
||||
|
||||
await svc.cancelProvisioning(staleRun.runId);
|
||||
|
||||
|
|
@ -14691,7 +14684,7 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
const initialSnapshot = await (svc as any).launchMixedSecondaryLaneIfNeeded(run);
|
||||
|
||||
expect(initialSnapshot.teamLaunchState).toBe('partial_pending');
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 2);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 1);
|
||||
|
||||
const inFlightStatuses = await svc.getMemberSpawnStatuses(teamName);
|
||||
expect(inFlightStatuses.teamLaunchState).toBe('partial_pending');
|
||||
|
|
@ -14747,7 +14740,7 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
const initialSnapshot = await (svc as any).launchMixedSecondaryLaneIfNeeded(run);
|
||||
|
||||
expect(initialSnapshot.teamLaunchState).toBe('partial_pending');
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 2);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 1);
|
||||
|
||||
const inFlightStatuses = await svc.getMemberSpawnStatuses(teamName);
|
||||
expect(inFlightStatuses.teamLaunchState).toBe('partial_pending');
|
||||
|
|
@ -14806,14 +14799,14 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
trackLiveRun(svc, run);
|
||||
|
||||
await (svc as any).launchMixedSecondaryLaneIfNeeded(run);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 2);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 1);
|
||||
const firstLaneRunIds = run.mixedSecondaryLanes.map(
|
||||
(lane: { runId: string | null }) => lane.runId
|
||||
);
|
||||
|
||||
await (svc as any).launchMixedSecondaryLaneIfNeeded(run);
|
||||
|
||||
expect(adapter.pendingLaunchInputs).toHaveLength(2);
|
||||
expect(adapter.pendingLaunchInputs).toHaveLength(1);
|
||||
expect(adapter.launchInputs).toHaveLength(0);
|
||||
expect(run.mixedSecondaryLanes.map((lane: { state: string }) => lane.state)).toEqual([
|
||||
'launching',
|
||||
|
|
@ -14857,14 +14850,14 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
trackLiveRun(svc, run);
|
||||
|
||||
await (svc as any).launchMixedSecondaryLaneIfNeeded(run);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 2);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 1);
|
||||
const firstLaneRunIds = run.mixedSecondaryLanes.map(
|
||||
(lane: { runId: string | null }) => lane.runId
|
||||
);
|
||||
|
||||
await (svc as any).launchMixedSecondaryLaneIfNeeded(run);
|
||||
|
||||
expect(adapter.pendingLaunchInputs).toHaveLength(2);
|
||||
expect(adapter.pendingLaunchInputs).toHaveLength(1);
|
||||
expect(adapter.launchInputs).toHaveLength(0);
|
||||
expect(run.mixedSecondaryLanes.map((lane: { state: string }) => lane.state)).toEqual([
|
||||
'launching',
|
||||
|
|
@ -15011,19 +15004,18 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
trackLiveRun(svc, run);
|
||||
|
||||
await (svc as any).launchMixedSecondaryLaneIfNeeded(run);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 2);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 1);
|
||||
|
||||
svc.stopTeam(teamName);
|
||||
|
||||
await waitForCondition(() => !svc.isTeamAlive(teamName));
|
||||
await waitForCondition(() => adapter.stopInputs.length === 2);
|
||||
await waitForCondition(() => adapter.stopInputs.length === 1);
|
||||
expect(adapter.stopInputs.map((input) => input.laneId).sort()).toEqual([
|
||||
'secondary:opencode:bob',
|
||||
'secondary:opencode:tom',
|
||||
]);
|
||||
|
||||
adapter.releaseLaunches();
|
||||
await waitForCondition(() => adapter.launchInputs.length === 2);
|
||||
await waitForCondition(() => adapter.launchInputs.length === 1);
|
||||
|
||||
const statuses = await svc.getMemberSpawnStatuses(teamName);
|
||||
expect(svc.isTeamAlive(teamName)).toBe(false);
|
||||
|
|
@ -15045,19 +15037,18 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
trackLiveRun(svc, run);
|
||||
|
||||
await (svc as any).launchMixedSecondaryLaneIfNeeded(run);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 2);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 1);
|
||||
|
||||
svc.stopTeam(teamName);
|
||||
|
||||
await waitForCondition(() => !svc.isTeamAlive(teamName));
|
||||
await waitForCondition(() => adapter.stopInputs.length === 2);
|
||||
await waitForCondition(() => adapter.stopInputs.length === 1);
|
||||
expect(adapter.stopInputs.map((input) => input.laneId).sort()).toEqual([
|
||||
'secondary:opencode:bob',
|
||||
'secondary:opencode:tom',
|
||||
]);
|
||||
|
||||
adapter.releaseLaunches();
|
||||
await waitForCondition(() => adapter.launchInputs.length === 2);
|
||||
await waitForCondition(() => adapter.launchInputs.length === 1);
|
||||
|
||||
const statuses = await svc.getMemberSpawnStatuses(teamName);
|
||||
expect(svc.isTeamAlive(teamName)).toBe(false);
|
||||
|
|
@ -15124,19 +15115,18 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
trackLiveRun(svc, run);
|
||||
|
||||
await (svc as any).launchMixedSecondaryLaneIfNeeded(run);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 2);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 1);
|
||||
|
||||
svc.stopTeam(teamName);
|
||||
|
||||
await waitForCondition(() => !svc.isTeamAlive(teamName));
|
||||
await waitForCondition(() => adapter.stopInputs.length === 2);
|
||||
await waitForCondition(() => adapter.stopInputs.length === 1);
|
||||
expect(adapter.stopInputs.map((input) => input.laneId).sort()).toEqual([
|
||||
'secondary:opencode:bob',
|
||||
'secondary:opencode:tom',
|
||||
]);
|
||||
|
||||
adapter.releaseLaunches();
|
||||
await waitForCondition(() => adapter.launchInputs.length === 2);
|
||||
await waitForCondition(() => adapter.launchInputs.length === 1);
|
||||
|
||||
const statuses = await svc.getMemberSpawnStatuses(teamName);
|
||||
expect(svc.isTeamAlive(teamName)).toBe(false);
|
||||
|
|
@ -15175,19 +15165,19 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
|
||||
await (svc as any).launchMixedSecondaryLaneIfNeeded(stoppedRun);
|
||||
await (svc as any).launchMixedSecondaryLaneIfNeeded(survivingRun);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 4);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 2);
|
||||
|
||||
svc.stopTeam(stoppedTeamName);
|
||||
|
||||
await waitForCondition(
|
||||
() => adapter.stopInputs.filter((input) => input.teamName === stoppedTeamName).length === 2
|
||||
() => adapter.stopInputs.filter((input) => input.teamName === stoppedTeamName).length === 1
|
||||
);
|
||||
expect(adapter.stopInputs.some((input) => input.teamName === survivingTeamName)).toBe(false);
|
||||
expect(svc.isTeamAlive(stoppedTeamName)).toBe(false);
|
||||
expect(svc.isTeamAlive(survivingTeamName)).toBe(true);
|
||||
|
||||
adapter.releaseLaunches();
|
||||
await waitForCondition(() => adapter.launchInputs.length === 4);
|
||||
await waitForCondition(() => adapter.launchInputs.length === 3);
|
||||
await waitForCondition(() =>
|
||||
survivingRun.mixedSecondaryLanes.every((lane: { state: string }) => lane.state === 'finished')
|
||||
);
|
||||
|
|
@ -15223,11 +15213,11 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
trackLiveRun(svc, oldRun);
|
||||
|
||||
await (svc as any).launchMixedSecondaryLaneIfNeeded(oldRun);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 2);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 1);
|
||||
|
||||
svc.stopTeam(teamName);
|
||||
await waitForCondition(() => !svc.isTeamAlive(teamName));
|
||||
await waitForCondition(() => adapter.stopInputs.length === 2);
|
||||
await waitForCondition(() => adapter.stopInputs.length === 1);
|
||||
|
||||
await writeMixedTeamLaunchState({
|
||||
teamName,
|
||||
|
|
@ -15275,7 +15265,7 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
});
|
||||
|
||||
adapter.releaseLaunches();
|
||||
await waitForCondition(() => adapter.launchInputs.length === 2);
|
||||
await waitForCondition(() => adapter.launchInputs.length === 1);
|
||||
|
||||
const statuses = await svc.getMemberSpawnStatuses(teamName);
|
||||
expect(statuses.teamLaunchState).toBe('partial_failure');
|
||||
|
|
@ -15307,11 +15297,11 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
trackLiveRun(svc, oldRun);
|
||||
|
||||
await (svc as any).launchMixedSecondaryLaneIfNeeded(oldRun);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 2);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 1);
|
||||
|
||||
svc.stopTeam(teamName);
|
||||
await waitForCondition(() => !svc.isTeamAlive(teamName));
|
||||
await waitForCondition(() => adapter.stopInputs.length === 2);
|
||||
await waitForCondition(() => adapter.stopInputs.length === 1);
|
||||
|
||||
await writeMixedTeamLaunchState({
|
||||
teamName,
|
||||
|
|
@ -15358,7 +15348,7 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
});
|
||||
|
||||
adapter.releaseLaunches();
|
||||
await waitForCondition(() => adapter.launchInputs.length === 2);
|
||||
await waitForCondition(() => adapter.launchInputs.length === 1);
|
||||
|
||||
const statuses = await svc.getMemberSpawnStatuses(teamName);
|
||||
expect(statuses.teamLaunchState).toBe('partial_failure');
|
||||
|
|
@ -15428,11 +15418,11 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
trackLiveRun(svc, oldRun);
|
||||
|
||||
await (svc as any).launchMixedSecondaryLaneIfNeeded(oldRun);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 2);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 1);
|
||||
|
||||
svc.stopTeam(teamName);
|
||||
await waitForCondition(() => !svc.isTeamAlive(teamName));
|
||||
await waitForCondition(() => adapter.stopInputs.length === 2);
|
||||
await waitForCondition(() => adapter.stopInputs.length === 1);
|
||||
|
||||
await writeMixedTeamLaunchState({
|
||||
teamName,
|
||||
|
|
@ -15526,14 +15516,14 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
trackLiveRun(svc, run);
|
||||
|
||||
await (svc as any).launchMixedSecondaryLaneIfNeeded(run);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 2);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 1);
|
||||
|
||||
svc.stopTeam(teamName);
|
||||
await waitForCondition(() => !svc.isTeamAlive(teamName));
|
||||
await waitForCondition(() => adapter.stopInputs.length === 2);
|
||||
await waitForCondition(() => adapter.stopInputs.length === 1);
|
||||
|
||||
adapter.releaseLaunches();
|
||||
await waitForCondition(() => adapter.rejectedLaunchCount === 2);
|
||||
await waitForCondition(() => adapter.rejectedLaunchCount === 1);
|
||||
|
||||
await expect(readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName)).resolves.toMatchObject(
|
||||
{
|
||||
|
|
@ -15564,14 +15554,14 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
trackLiveRun(svc, run);
|
||||
|
||||
await (svc as any).launchMixedSecondaryLaneIfNeeded(run);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 2);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 1);
|
||||
|
||||
svc.stopTeam(teamName);
|
||||
await waitForCondition(() => !svc.isTeamAlive(teamName));
|
||||
await waitForCondition(() => adapter.stopInputs.length === 2);
|
||||
await waitForCondition(() => adapter.stopInputs.length === 1);
|
||||
|
||||
adapter.releaseLaunches();
|
||||
await waitForCondition(() => adapter.rejectedLaunchCount === 2);
|
||||
await waitForCondition(() => adapter.rejectedLaunchCount === 1);
|
||||
|
||||
await expect(readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName)).resolves.toMatchObject(
|
||||
{
|
||||
|
|
@ -15642,14 +15632,14 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
trackLiveRun(svc, run);
|
||||
|
||||
await (svc as any).launchMixedSecondaryLaneIfNeeded(run);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 2);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 1);
|
||||
|
||||
svc.stopTeam(teamName);
|
||||
await waitForCondition(() => !svc.isTeamAlive(teamName));
|
||||
await waitForCondition(() => adapter.stopInputs.length === 2);
|
||||
await waitForCondition(() => adapter.stopInputs.length === 1);
|
||||
|
||||
adapter.releaseLaunches();
|
||||
await waitForCondition(() => adapter.rejectedLaunchCount === 2);
|
||||
await waitForCondition(() => adapter.rejectedLaunchCount === 1);
|
||||
|
||||
await expect(readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName)).resolves.toMatchObject(
|
||||
{
|
||||
|
|
@ -15694,7 +15684,6 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
await waitForCondition(() => adapter.stopInputs.length === 2);
|
||||
expect(adapter.stopInputs.map((input) => input.laneId).sort()).toEqual([
|
||||
'secondary:opencode:bob',
|
||||
'secondary:opencode:tom',
|
||||
]);
|
||||
expect(svc.isTeamAlive(teamName)).toBe(false);
|
||||
|
||||
|
|
@ -15727,12 +15716,11 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
await waitForCondition(() => adapter.stopInputs.length === 2);
|
||||
expect(adapter.stopInputs.map((input) => input.laneId).sort()).toEqual([
|
||||
'secondary:opencode:bob',
|
||||
'secondary:opencode:tom',
|
||||
]);
|
||||
expect(svc.isTeamAlive(teamName)).toBe(false);
|
||||
|
||||
adapter.releaseLaunches();
|
||||
await waitForCondition(() => adapter.launchInputs.length === 2);
|
||||
await waitForCondition(() => adapter.launchInputs.length === 1);
|
||||
|
||||
const statuses = await svc.getMemberSpawnStatuses(teamName);
|
||||
expect(statuses.teamLaunchState).not.toBe('clean_success');
|
||||
|
|
@ -15767,19 +15755,18 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
trackLiveRun(svc, run);
|
||||
|
||||
await (svc as any).launchMixedSecondaryLaneIfNeeded(run);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 2);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 1);
|
||||
|
||||
await svc.cancelProvisioning(run.runId);
|
||||
|
||||
await waitForCondition(() => adapter.stopInputs.length === 2);
|
||||
await waitForCondition(() => adapter.stopInputs.length === 1);
|
||||
expect(adapter.stopInputs.map((input) => input.laneId).sort()).toEqual([
|
||||
'secondary:opencode:bob',
|
||||
'secondary:opencode:tom',
|
||||
]);
|
||||
expect(svc.isTeamAlive(teamName)).toBe(false);
|
||||
|
||||
adapter.releaseLaunches();
|
||||
await waitForCondition(() => adapter.launchInputs.length === 2);
|
||||
await waitForCondition(() => adapter.launchInputs.length === 1);
|
||||
|
||||
const statuses = await svc.getMemberSpawnStatuses(teamName);
|
||||
expect(statuses.teamLaunchState).not.toBe('clean_success');
|
||||
|
|
@ -15825,15 +15812,14 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
|
||||
await svc.cancelProvisioning(cancelledRun.runId);
|
||||
|
||||
await waitForCondition(() => adapter.stopInputs.length === 2);
|
||||
await waitForCondition(() => adapter.stopInputs.length === 1);
|
||||
expect(adapter.stopInputs.map((input) => input.laneId).sort()).toEqual([
|
||||
'secondary:opencode:bob',
|
||||
'secondary:opencode:tom',
|
||||
]);
|
||||
expect(svc.isTeamAlive(teamName)).toBe(false);
|
||||
|
||||
adapter.releaseLaunches();
|
||||
await waitForCondition(() => adapter.launchInputs.length === 2);
|
||||
await waitForCondition(() => adapter.launchInputs.length === 1);
|
||||
|
||||
const cancelledStatuses = await svc.getMemberSpawnStatuses(teamName);
|
||||
expect(cancelledStatuses.teamLaunchState).not.toBe('clean_success');
|
||||
|
|
@ -15849,7 +15835,7 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
trackLiveRun(svc, freshRun);
|
||||
|
||||
await (svc as any).launchMixedSecondaryLaneIfNeeded(freshRun);
|
||||
await waitForCondition(() => adapter.launchInputs.length === 4);
|
||||
await waitForCondition(() => adapter.launchInputs.length === 3);
|
||||
await waitForCondition(() =>
|
||||
freshRun.mixedSecondaryLanes.every((lane: { state: string }) => lane.state === 'finished')
|
||||
);
|
||||
|
|
@ -15902,19 +15888,19 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
|
||||
await (svc as any).launchMixedSecondaryLaneIfNeeded(cancelledRun);
|
||||
await (svc as any).launchMixedSecondaryLaneIfNeeded(survivingRun);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 4);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 2);
|
||||
|
||||
await svc.cancelProvisioning(cancelledRun.runId);
|
||||
|
||||
await waitForCondition(
|
||||
() => adapter.stopInputs.filter((input) => input.teamName === cancelledTeamName).length === 2
|
||||
() => adapter.stopInputs.filter((input) => input.teamName === cancelledTeamName).length === 1
|
||||
);
|
||||
expect(adapter.stopInputs.some((input) => input.teamName === survivingTeamName)).toBe(false);
|
||||
expect(svc.isTeamAlive(cancelledTeamName)).toBe(false);
|
||||
expect(svc.isTeamAlive(survivingTeamName)).toBe(true);
|
||||
|
||||
adapter.releaseLaunches();
|
||||
await waitForCondition(() => adapter.launchInputs.length === 4);
|
||||
await waitForCondition(() => adapter.launchInputs.length === 3);
|
||||
await waitForCondition(() =>
|
||||
survivingRun.mixedSecondaryLanes.every((lane: { state: string }) => lane.state === 'finished')
|
||||
);
|
||||
|
|
@ -15987,19 +15973,19 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
|
||||
await (svc as any).launchMixedSecondaryLaneIfNeeded(cancelledRun);
|
||||
await (svc as any).launchMixedSecondaryLaneIfNeeded(survivingRun);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 4);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 2);
|
||||
|
||||
await svc.cancelProvisioning(cancelledRun.runId);
|
||||
|
||||
await waitForCondition(
|
||||
() => adapter.stopInputs.filter((input) => input.teamName === cancelledTeamName).length === 2
|
||||
() => adapter.stopInputs.filter((input) => input.teamName === cancelledTeamName).length === 1
|
||||
);
|
||||
expect(adapter.stopInputs.some((input) => input.teamName === survivingTeamName)).toBe(false);
|
||||
expect(svc.isTeamAlive(cancelledTeamName)).toBe(false);
|
||||
expect(svc.isTeamAlive(survivingTeamName)).toBe(true);
|
||||
|
||||
adapter.releaseLaunches();
|
||||
await waitForCondition(() => adapter.launchInputs.length === 4);
|
||||
await waitForCondition(() => adapter.launchInputs.length === 3);
|
||||
await waitForCondition(() =>
|
||||
survivingRun.mixedSecondaryLanes.every((lane: { state: string }) => lane.state === 'finished')
|
||||
);
|
||||
|
|
@ -16051,17 +16037,17 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
|
||||
await (svc as any).launchMixedSecondaryLaneIfNeeded(cancelledRun);
|
||||
await (svc as any).launchMixedSecondaryLaneIfNeeded(survivingRun);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 4);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 2);
|
||||
|
||||
await svc.cancelProvisioning(cancelledRun.runId);
|
||||
|
||||
await waitForCondition(
|
||||
() => adapter.stopInputs.filter((input) => input.teamName === cancelledTeamName).length === 2
|
||||
() => adapter.stopInputs.filter((input) => input.teamName === cancelledTeamName).length === 1
|
||||
);
|
||||
expect(adapter.stopInputs.some((input) => input.teamName === survivingTeamName)).toBe(false);
|
||||
|
||||
adapter.releaseLaunches();
|
||||
await waitForCondition(() => adapter.launchInputs.length === 4);
|
||||
await waitForCondition(() => adapter.launchInputs.length === 3);
|
||||
await waitForCondition(() =>
|
||||
survivingRun.mixedSecondaryLanes.every((lane: { state: string }) => lane.state === 'finished')
|
||||
);
|
||||
|
|
@ -16073,7 +16059,7 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
trackLiveRun(svc, freshRun);
|
||||
|
||||
await (svc as any).launchMixedSecondaryLaneIfNeeded(freshRun);
|
||||
await waitForCondition(() => adapter.launchInputs.length === 6);
|
||||
await waitForCondition(() => adapter.launchInputs.length === 5);
|
||||
await waitForCondition(() =>
|
||||
freshRun.mixedSecondaryLanes.every((lane: { state: string }) => lane.state === 'finished')
|
||||
);
|
||||
|
|
@ -16154,17 +16140,17 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
|
||||
await (svc as any).launchMixedSecondaryLaneIfNeeded(cancelledRun);
|
||||
await (svc as any).launchMixedSecondaryLaneIfNeeded(survivingRun);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 4);
|
||||
await waitForCondition(() => adapter.pendingLaunchInputs.length === 2);
|
||||
|
||||
await svc.cancelProvisioning(cancelledRun.runId);
|
||||
|
||||
await waitForCondition(
|
||||
() => adapter.stopInputs.filter((input) => input.teamName === cancelledTeamName).length === 2
|
||||
() => adapter.stopInputs.filter((input) => input.teamName === cancelledTeamName).length === 1
|
||||
);
|
||||
expect(adapter.stopInputs.some((input) => input.teamName === survivingTeamName)).toBe(false);
|
||||
|
||||
adapter.releaseLaunches();
|
||||
await waitForCondition(() => adapter.launchInputs.length === 4);
|
||||
await waitForCondition(() => adapter.launchInputs.length === 3);
|
||||
await waitForCondition(() =>
|
||||
survivingRun.mixedSecondaryLanes.every((lane: { state: string }) => lane.state === 'finished')
|
||||
);
|
||||
|
|
@ -16181,7 +16167,7 @@ describe('Team agent launch matrix safe e2e', () => {
|
|||
trackLiveRun(svc, freshRun);
|
||||
|
||||
await (svc as any).launchMixedSecondaryLaneIfNeeded(freshRun);
|
||||
await waitForCondition(() => adapter.launchInputs.length === 6);
|
||||
await waitForCondition(() => adapter.launchInputs.length === 5);
|
||||
await waitForCondition(() =>
|
||||
freshRun.mixedSecondaryLanes.every((lane: { state: string }) => lane.state === 'finished')
|
||||
);
|
||||
|
|
|
|||
|
|
@ -315,4 +315,84 @@ describe('TeamBackupService', () => {
|
|||
warnSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('backs up member-scoped work sync files', async () => {
|
||||
const service = new TeamBackupService();
|
||||
const teamName = 'member-work-sync-team';
|
||||
const teamDir = path.join(hoisted.teamsBase, teamName);
|
||||
const memberDir = path.join(teamDir, 'members', 'jack');
|
||||
const workSyncDir = path.join(memberDir, '.member-work-sync');
|
||||
const status = {
|
||||
teamName,
|
||||
memberName: 'jack',
|
||||
state: 'caught_up',
|
||||
evaluatedAt: '2026-04-30T12:00:00.000Z',
|
||||
agenda: { fingerprint: 'abc123', items: [] },
|
||||
};
|
||||
|
||||
try {
|
||||
await fs.mkdir(workSyncDir, { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(teamDir, 'config.json'),
|
||||
JSON.stringify({ name: 'Member Work Sync Team' }),
|
||||
'utf8'
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(memberDir, 'member.meta.json'),
|
||||
JSON.stringify({
|
||||
schemaVersion: 1,
|
||||
memberName: 'jack',
|
||||
memberKey: 'jack',
|
||||
updatedAt: '2026-04-30T12:00:00.000Z',
|
||||
}),
|
||||
'utf8'
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(workSyncDir, 'status.json'),
|
||||
JSON.stringify({ schemaVersion: 2, status }),
|
||||
'utf8'
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(workSyncDir, 'journal.jsonl'),
|
||||
`${JSON.stringify({
|
||||
schemaVersion: 1,
|
||||
timestamp: '2026-04-30T12:00:00.000Z',
|
||||
teamName,
|
||||
memberName: 'jack',
|
||||
event: 'status_written',
|
||||
source: 'test',
|
||||
})}\n`,
|
||||
'utf8'
|
||||
);
|
||||
await fs.writeFile(path.join(workSyncDir, '.tmp.deadbeef'), '{"partial":', 'utf8');
|
||||
await fs.writeFile(path.join(workSyncDir, 'journal.jsonl.lock'), '123\n', 'utf8');
|
||||
|
||||
await service.initialize();
|
||||
await service.backupTeam(teamName);
|
||||
|
||||
const backupMemberDir = path.join(hoisted.backupsBase, 'teams', teamName, 'members', 'jack');
|
||||
await expect(fs.readFile(path.join(backupMemberDir, 'member.meta.json'), 'utf8')).resolves.toBe(
|
||||
JSON.stringify({
|
||||
schemaVersion: 1,
|
||||
memberName: 'jack',
|
||||
memberKey: 'jack',
|
||||
updatedAt: '2026-04-30T12:00:00.000Z',
|
||||
})
|
||||
);
|
||||
await expect(
|
||||
fs.readFile(path.join(backupMemberDir, '.member-work-sync', 'status.json'), 'utf8')
|
||||
).resolves.toBe(JSON.stringify({ schemaVersion: 2, status }));
|
||||
await expect(
|
||||
fs.readFile(path.join(backupMemberDir, '.member-work-sync', 'journal.jsonl'), 'utf8')
|
||||
).resolves.toContain('"event":"status_written"');
|
||||
await expect(
|
||||
fs.stat(path.join(backupMemberDir, '.member-work-sync', '.tmp.deadbeef'))
|
||||
).rejects.toMatchObject({ code: 'ENOENT' });
|
||||
await expect(
|
||||
fs.stat(path.join(backupMemberDir, '.member-work-sync', 'journal.jsonl.lock'))
|
||||
).rejects.toMatchObject({ code: 'ENOENT' });
|
||||
} finally {
|
||||
service.dispose();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
|
|||
76
test/main/services/team/TeamMemberStoragePaths.test.ts
Normal file
76
test/main/services/team/TeamMemberStoragePaths.test.ts
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
import { mkdtemp, readFile, rm } from 'fs/promises';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
|
||||
import {
|
||||
encodeTeamMemberStorageKey,
|
||||
TeamMemberStoragePaths,
|
||||
} from '@main/services/team/TeamMemberStoragePaths';
|
||||
|
||||
describe('TeamMemberStoragePaths', () => {
|
||||
let root: string;
|
||||
let paths: TeamMemberStoragePaths;
|
||||
|
||||
beforeEach(async () => {
|
||||
root = await mkdtemp(join(tmpdir(), 'team-member-storage-paths-'));
|
||||
paths = new TeamMemberStoragePaths(root);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('builds stable path-safe keys from canonical member names', () => {
|
||||
expect(encodeTeamMemberStorageKey(' Bob ')).toBe('bob');
|
||||
expect(encodeTeamMemberStorageKey('Jack Smith')).toBe('jack%20smith');
|
||||
expect(encodeTeamMemberStorageKey('../Alice')).toBe('..%2Falice');
|
||||
expect(encodeTeamMemberStorageKey('.')).toBe('%2E');
|
||||
expect(encodeTeamMemberStorageKey('..')).toBe('%2E%2E');
|
||||
expect(encodeTeamMemberStorageKey('Том')).toBe('%D1%82%D0%BE%D0%BC');
|
||||
});
|
||||
|
||||
it('keeps member storage inside the team members directory', () => {
|
||||
expect(paths.getMemberDir('team-a', '../Alice')).toBe(
|
||||
join(root, 'team-a', 'members', '..%2Falice')
|
||||
);
|
||||
expect(paths.getMemberDir('team-a', '..')).toBe(
|
||||
join(root, 'team-a', 'members', '%2E%2E')
|
||||
);
|
||||
expect(paths.getMemberDir('team-a', '.')).toBe(
|
||||
join(root, 'team-a', 'members', '%2E')
|
||||
);
|
||||
expect(paths.getMemberFeatureDir('team-a', 'Bob', '.member-work-sync')).toBe(
|
||||
join(root, 'team-a', 'members', 'bob', '.member-work-sync')
|
||||
);
|
||||
});
|
||||
|
||||
it('rejects empty member names and nested feature directory names', () => {
|
||||
expect(() => encodeTeamMemberStorageKey(' ')).toThrow('memberName is required');
|
||||
expect(() => paths.getMemberFeatureDir('team-a', 'Bob', '../unsafe')).toThrow(
|
||||
'featureDirName must be a single path segment'
|
||||
);
|
||||
expect(() => paths.getMemberFeatureDir('team-a', 'Bob', 'nested/unsafe')).toThrow(
|
||||
'featureDirName must be a single path segment'
|
||||
);
|
||||
expect(() => paths.getMemberFeatureDir('team-a', 'Bob', '..')).toThrow(
|
||||
'featureDirName must be a single path segment'
|
||||
);
|
||||
expect(() => paths.getMemberFeatureDir('team-a', 'Bob', '.')).toThrow(
|
||||
'featureDirName must be a single path segment'
|
||||
);
|
||||
});
|
||||
|
||||
it('materializes canonical member meta without changing the path key', async () => {
|
||||
await paths.ensureMemberMeta('team-a', 'Bob');
|
||||
|
||||
const meta = JSON.parse(
|
||||
await readFile(join(root, 'team-a', 'members', 'bob', 'member.meta.json'), 'utf8')
|
||||
);
|
||||
expect(meta).toMatchObject({
|
||||
schemaVersion: 1,
|
||||
memberName: 'Bob',
|
||||
memberKey: 'bob',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -136,6 +136,7 @@ import {
|
|||
getOpenCodeRuntimeManifestPath,
|
||||
OpenCodeRuntimeManifestEvidenceReader,
|
||||
readOpenCodeRuntimeLaneIndex,
|
||||
setOpenCodeRuntimeActiveRunManifest,
|
||||
upsertOpenCodeRuntimeLaneIndexEntry,
|
||||
} from '@main/services/team/opencode/store/OpenCodeRuntimeManifestEvidenceReader';
|
||||
import { createDefaultRuntimeStoreManifest } from '@main/services/team/opencode/store/RuntimeStoreManifest';
|
||||
|
|
@ -1049,7 +1050,7 @@ describe('TeamProvisioningService', () => {
|
|||
it('does not carry stale persisted runtimeAlive through launch-state reconcile', async () => {
|
||||
const teamName = 'persisted-stale-runtime-status-team';
|
||||
const projectPath = '/Users/test/project';
|
||||
const acceptedAt = new Date(Date.now() - 120_000).toISOString();
|
||||
const acceptedAt = new Date(Date.now() - 220_000).toISOString();
|
||||
writeLaunchConfig(teamName, projectPath, 'lead-session', ['alice']);
|
||||
writeLaunchState(teamName, 'lead-session', {
|
||||
alice: {
|
||||
|
|
@ -5322,6 +5323,11 @@ describe('TeamProvisioningService', () => {
|
|||
)}\n`,
|
||||
'utf8'
|
||||
);
|
||||
await fsPromises.writeFile(
|
||||
path.join(path.dirname(manifestPath), 'opencode-sessions.json'),
|
||||
`${JSON.stringify({ sessions: [{ id: 'oc-session-bob' }] })}\n`,
|
||||
'utf8'
|
||||
);
|
||||
|
||||
await expect(
|
||||
svc.deliverOpenCodeMemberMessage(teamName, {
|
||||
|
|
@ -5346,6 +5352,97 @@ describe('TeamProvisioningService', () => {
|
|||
);
|
||||
});
|
||||
|
||||
it('rejects stale active lane manifest without runtime evidence before delivery', async () => {
|
||||
const svc = new TeamProvisioningService();
|
||||
const teamName = 'team-a';
|
||||
const laneId = 'secondary:opencode:bob';
|
||||
const sendMessageToMember = vi.fn(async (input: Record<string, unknown>) => ({
|
||||
ok: true,
|
||||
providerId: 'opencode',
|
||||
memberName: String(input.memberName),
|
||||
sessionId: 'oc-session-bob',
|
||||
diagnostics: [],
|
||||
}));
|
||||
const registry = new TeamRuntimeAdapterRegistry([
|
||||
{
|
||||
providerId: 'opencode',
|
||||
prepare: vi.fn(),
|
||||
launch: vi.fn(),
|
||||
reconcile: vi.fn(),
|
||||
stop: vi.fn(),
|
||||
sendMessageToMember,
|
||||
} as any,
|
||||
]);
|
||||
svc.setRuntimeAdapterRegistry(registry);
|
||||
|
||||
(svc as any).configReader = {
|
||||
getConfig: vi.fn(async () => ({
|
||||
projectPath: '/repo',
|
||||
members: [
|
||||
{ name: 'team-lead', providerId: 'codex', model: 'gpt-5.4' },
|
||||
{ name: 'bob', providerId: 'opencode', model: 'minimax-m2.5-free' },
|
||||
],
|
||||
})),
|
||||
};
|
||||
(svc as any).teamMetaStore = {
|
||||
getMeta: vi.fn(async () => ({
|
||||
launchIdentity: { providerId: 'codex' },
|
||||
providerId: 'codex',
|
||||
})),
|
||||
};
|
||||
(svc as any).membersMetaStore = {
|
||||
getMembers: vi.fn(async () => [
|
||||
{
|
||||
name: 'bob',
|
||||
providerId: 'opencode',
|
||||
model: 'opencode/minimax-m2.5-free',
|
||||
},
|
||||
]),
|
||||
};
|
||||
await upsertOpenCodeRuntimeLaneIndexEntry({
|
||||
teamsBasePath: tempTeamsBase,
|
||||
teamName,
|
||||
laneId,
|
||||
state: 'active',
|
||||
});
|
||||
const manifestPath = getOpenCodeRuntimeManifestPath(tempTeamsBase, teamName, laneId);
|
||||
await fsPromises.mkdir(path.dirname(manifestPath), { recursive: true });
|
||||
await fsPromises.writeFile(
|
||||
manifestPath,
|
||||
`${JSON.stringify(
|
||||
{
|
||||
...createDefaultRuntimeStoreManifest(teamName, '2026-04-22T12:00:00.000Z'),
|
||||
activeRunId: 'opencode-run-stale-empty',
|
||||
},
|
||||
null,
|
||||
2
|
||||
)}\n`,
|
||||
'utf8'
|
||||
);
|
||||
|
||||
await expect(
|
||||
svc.deliverOpenCodeMemberMessage(teamName, {
|
||||
memberName: 'bob',
|
||||
text: 'must not deliver to empty durable lane',
|
||||
messageId: 'msg-stale-empty-manifest',
|
||||
})
|
||||
).resolves.toMatchObject({
|
||||
delivered: false,
|
||||
reason: 'opencode_runtime_not_active',
|
||||
diagnostics: [
|
||||
expect.stringContaining('runtime manifest has no committed runtime evidence'),
|
||||
],
|
||||
});
|
||||
expect(sendMessageToMember).not.toHaveBeenCalled();
|
||||
await expect(readOpenCodeRuntimeLaneIndex(tempTeamsBase, teamName)).resolves.toMatchObject({
|
||||
lanes: {
|
||||
[laneId]: {
|
||||
state: 'degraded',
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('falls back to lane manifest when a tracked primary run lacks the secondary lane snapshot', async () => {
|
||||
const svc = new TeamProvisioningService();
|
||||
const teamName = 'team-a';
|
||||
|
|
@ -5417,6 +5514,11 @@ describe('TeamProvisioningService', () => {
|
|||
)}\n`,
|
||||
'utf8'
|
||||
);
|
||||
await fsPromises.writeFile(
|
||||
path.join(path.dirname(manifestPath), 'opencode-sessions.json'),
|
||||
`${JSON.stringify({ sessions: [{ id: 'oc-session-bob' }] })}\n`,
|
||||
'utf8'
|
||||
);
|
||||
|
||||
await expect(
|
||||
svc.deliverOpenCodeMemberMessage(teamName, {
|
||||
|
|
@ -5578,7 +5680,7 @@ describe('TeamProvisioningService', () => {
|
|||
);
|
||||
});
|
||||
|
||||
it('starts all queued OpenCode secondary lanes without letting the first in-flight lane block its siblings', async () => {
|
||||
it('starts queued OpenCode secondary lanes sequentially without blocking launch progress', async () => {
|
||||
const svc = new TeamProvisioningService();
|
||||
const registry = new TeamRuntimeAdapterRegistry([
|
||||
{
|
||||
|
|
@ -5684,7 +5786,7 @@ describe('TeamProvisioningService', () => {
|
|||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
|
||||
expect(launchSingleMixedSecondaryLane).toHaveBeenCalledTimes(3);
|
||||
expect(launchSingleMixedSecondaryLane).toHaveBeenCalledTimes(1);
|
||||
expect(run.mixedSecondaryLanes.map((lane: { state: string }) => lane.state)).toEqual([
|
||||
'launching',
|
||||
'launching',
|
||||
|
|
@ -5696,6 +5798,7 @@ describe('TeamProvisioningService', () => {
|
|||
|
||||
resolveFirstLaunch();
|
||||
await Promise.resolve();
|
||||
await vi.waitFor(() => expect(launchSingleMixedSecondaryLane).toHaveBeenCalledTimes(3));
|
||||
});
|
||||
|
||||
it('preserves mixed lane metadata when OpenCode runtime liveness updates a secondary lane member', async () => {
|
||||
|
|
@ -5797,6 +5900,8 @@ describe('TeamProvisioningService', () => {
|
|||
launchState: 'confirmed_alive',
|
||||
runtimeAlive: true,
|
||||
bootstrapConfirmed: true,
|
||||
runtimeRunId: 'run-member-spawn-1',
|
||||
runtimeSessionId: 'session-bob',
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -5876,6 +5981,71 @@ describe('TeamProvisioningService', () => {
|
|||
expect(diagnostics.join('\n')).not.toContain('super-secret');
|
||||
});
|
||||
|
||||
it('does not carry a stale OpenCode runtime pid into a fresh runtime run check-in', async () => {
|
||||
const svc = new TeamProvisioningService();
|
||||
const previousSnapshot = {
|
||||
version: 2 as const,
|
||||
teamName: 'mixed-team',
|
||||
updatedAt: '2026-04-22T12:00:00.000Z',
|
||||
launchPhase: 'active' as const,
|
||||
expectedMembers: ['bob'],
|
||||
members: {
|
||||
bob: {
|
||||
name: 'bob',
|
||||
providerId: 'opencode' as const,
|
||||
laneId: 'secondary:opencode:bob',
|
||||
laneKind: 'secondary' as const,
|
||||
laneOwnerProviderId: 'opencode' as const,
|
||||
launchState: 'confirmed_alive' as const,
|
||||
agentToolAccepted: true,
|
||||
runtimeAlive: true,
|
||||
bootstrapConfirmed: true,
|
||||
hardFailure: false,
|
||||
runtimePid: 1111,
|
||||
runtimeRunId: 'opencode-run-old',
|
||||
runtimeSessionId: 'session-bob-old',
|
||||
livenessKind: 'confirmed_bootstrap' as const,
|
||||
pidSource: 'runtime_bootstrap' as const,
|
||||
lastEvaluatedAt: '2026-04-22T12:00:00.000Z',
|
||||
},
|
||||
},
|
||||
summary: {
|
||||
confirmedCount: 1,
|
||||
pendingCount: 0,
|
||||
failedCount: 0,
|
||||
runtimeAlivePendingCount: 0,
|
||||
},
|
||||
teamLaunchState: 'ready' as const,
|
||||
};
|
||||
const write = vi.fn(async () => {});
|
||||
|
||||
(svc as any).launchStateStore = {
|
||||
read: vi.fn(async () => previousSnapshot),
|
||||
write,
|
||||
};
|
||||
|
||||
await (svc as any).updateOpenCodeRuntimeMemberLiveness({
|
||||
teamName: 'mixed-team',
|
||||
runId: 'opencode-run-new',
|
||||
memberName: 'bob',
|
||||
runtimeSessionId: 'session-bob-new',
|
||||
observedAt: '2026-04-22T12:05:00.000Z',
|
||||
diagnostics: [],
|
||||
reason: 'OpenCode runtime bootstrap check-in accepted',
|
||||
});
|
||||
|
||||
const writtenSnapshot = (
|
||||
write.mock.calls[0] as unknown as [string, Record<string, unknown>] | undefined
|
||||
)?.[1] as { members?: Record<string, Record<string, unknown>> } | undefined;
|
||||
expect(writtenSnapshot?.members?.bob).toMatchObject({
|
||||
runtimeRunId: 'opencode-run-new',
|
||||
runtimeSessionId: 'session-bob-new',
|
||||
launchState: 'confirmed_alive',
|
||||
});
|
||||
expect(writtenSnapshot?.members?.bob?.runtimePid).toBeUndefined();
|
||||
expect(writtenSnapshot?.members?.bob?.pidSource).toBeUndefined();
|
||||
});
|
||||
|
||||
it('preserves richer persisted expectedMembers when OpenCode runtime liveness updates a stale snapshot', async () => {
|
||||
const svc = new TeamProvisioningService();
|
||||
const previousSnapshot = {
|
||||
|
|
@ -5940,6 +6110,298 @@ describe('TeamProvisioningService', () => {
|
|||
expect(writtenSnapshot?.expectedMembers).toEqual(['bob', 'alice']);
|
||||
});
|
||||
|
||||
it('accepts duplicate OpenCode bootstrap check-ins for the same runtime session without rewriting liveness', async () => {
|
||||
const svc = new TeamProvisioningService();
|
||||
const previousSnapshot = {
|
||||
version: 2 as const,
|
||||
teamName: 'mixed-team',
|
||||
updatedAt: '2026-04-22T12:00:00.000Z',
|
||||
launchPhase: 'active' as const,
|
||||
expectedMembers: ['bob'],
|
||||
members: {
|
||||
bob: {
|
||||
name: 'bob',
|
||||
providerId: 'opencode' as const,
|
||||
laneId: 'secondary:opencode:bob',
|
||||
laneKind: 'secondary' as const,
|
||||
laneOwnerProviderId: 'opencode' as const,
|
||||
launchState: 'confirmed_alive' as const,
|
||||
agentToolAccepted: true,
|
||||
runtimeAlive: true,
|
||||
bootstrapConfirmed: true,
|
||||
hardFailure: false,
|
||||
runtimeRunId: 'opencode-run-1',
|
||||
runtimeSessionId: 'session-bob',
|
||||
livenessKind: 'confirmed_bootstrap' as const,
|
||||
lastEvaluatedAt: '2026-04-22T12:00:00.000Z',
|
||||
},
|
||||
},
|
||||
summary: {
|
||||
confirmedCount: 1,
|
||||
pendingCount: 0,
|
||||
failedCount: 0,
|
||||
runtimeAlivePendingCount: 0,
|
||||
},
|
||||
teamLaunchState: 'ready' as const,
|
||||
};
|
||||
const updateLiveness = vi.spyOn(svc as any, 'updateOpenCodeRuntimeMemberLiveness');
|
||||
|
||||
(svc as any).launchStateStore = {
|
||||
read: vi.fn(async () => previousSnapshot),
|
||||
write: vi.fn(async () => {}),
|
||||
};
|
||||
(svc as any).resolveOpenCodeRuntimeLaneId = vi.fn(async () => 'secondary:opencode:bob');
|
||||
(svc as any).assertOpenCodeRuntimeEvidenceAccepted = vi.fn(async () => {});
|
||||
|
||||
const ack = await svc.recordOpenCodeRuntimeBootstrapCheckin({
|
||||
teamName: 'mixed-team',
|
||||
runId: 'opencode-run-1',
|
||||
memberName: 'bob',
|
||||
runtimeSessionId: 'session-bob',
|
||||
observedAt: '2026-04-22T12:05:00.000Z',
|
||||
});
|
||||
|
||||
expect(ack).toMatchObject({
|
||||
ok: true,
|
||||
state: 'accepted',
|
||||
diagnostics: ['opencode_bootstrap_checkin_duplicate_accepted'],
|
||||
runtimeSessionId: 'session-bob',
|
||||
});
|
||||
expect(updateLiveness).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('rejects duplicate OpenCode bootstrap check-ins for members removed after the first check-in', async () => {
|
||||
const svc = new TeamProvisioningService();
|
||||
const previousSnapshot = {
|
||||
version: 2 as const,
|
||||
teamName: 'mixed-team',
|
||||
updatedAt: '2026-04-22T12:00:00.000Z',
|
||||
launchPhase: 'active' as const,
|
||||
expectedMembers: ['bob'],
|
||||
members: {
|
||||
bob: {
|
||||
name: 'bob',
|
||||
providerId: 'opencode' as const,
|
||||
laneId: 'secondary:opencode:bob',
|
||||
laneKind: 'secondary' as const,
|
||||
laneOwnerProviderId: 'opencode' as const,
|
||||
launchState: 'confirmed_alive' as const,
|
||||
agentToolAccepted: true,
|
||||
runtimeAlive: true,
|
||||
bootstrapConfirmed: true,
|
||||
hardFailure: false,
|
||||
runtimeRunId: 'opencode-run-1',
|
||||
runtimeSessionId: 'session-bob',
|
||||
livenessKind: 'confirmed_bootstrap' as const,
|
||||
lastEvaluatedAt: '2026-04-22T12:00:00.000Z',
|
||||
},
|
||||
},
|
||||
summary: {
|
||||
confirmedCount: 1,
|
||||
pendingCount: 0,
|
||||
failedCount: 0,
|
||||
runtimeAlivePendingCount: 0,
|
||||
},
|
||||
teamLaunchState: 'ready' as const,
|
||||
};
|
||||
const updateLiveness = vi.spyOn(svc as any, 'updateOpenCodeRuntimeMemberLiveness');
|
||||
|
||||
(svc as any).launchStateStore = {
|
||||
read: vi.fn(async () => previousSnapshot),
|
||||
write: vi.fn(async () => {}),
|
||||
};
|
||||
(svc as any).resolveOpenCodeRuntimeLaneId = vi.fn(async () => 'secondary:opencode:bob');
|
||||
(svc as any).assertOpenCodeRuntimeEvidenceAccepted = vi.fn(async () => {});
|
||||
(svc as any).configReader = {
|
||||
getConfig: vi.fn(async () => ({
|
||||
teamName: 'mixed-team',
|
||||
members: [{ name: 'bob', providerId: 'opencode', removedAt: 123 }],
|
||||
})),
|
||||
};
|
||||
(svc as any).membersMetaStore = {
|
||||
getMembers: vi.fn(async () => []),
|
||||
};
|
||||
|
||||
await expect(
|
||||
svc.recordOpenCodeRuntimeBootstrapCheckin({
|
||||
teamName: 'mixed-team',
|
||||
runId: 'opencode-run-1',
|
||||
memberName: 'bob',
|
||||
runtimeSessionId: 'session-bob',
|
||||
observedAt: '2026-04-22T12:05:00.000Z',
|
||||
})
|
||||
).rejects.toMatchObject({
|
||||
name: 'RuntimeStaleEvidenceError',
|
||||
});
|
||||
expect(updateLiveness).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('rejects conflicting OpenCode bootstrap check-ins for an already confirmed runtime session', async () => {
|
||||
const svc = new TeamProvisioningService();
|
||||
const previousSnapshot = {
|
||||
version: 2 as const,
|
||||
teamName: 'mixed-team',
|
||||
updatedAt: '2026-04-22T12:00:00.000Z',
|
||||
launchPhase: 'active' as const,
|
||||
expectedMembers: ['bob'],
|
||||
members: {
|
||||
bob: {
|
||||
name: 'bob',
|
||||
providerId: 'opencode' as const,
|
||||
laneId: 'secondary:opencode:bob',
|
||||
laneKind: 'secondary' as const,
|
||||
laneOwnerProviderId: 'opencode' as const,
|
||||
launchState: 'confirmed_alive' as const,
|
||||
agentToolAccepted: true,
|
||||
runtimeAlive: true,
|
||||
bootstrapConfirmed: true,
|
||||
hardFailure: false,
|
||||
runtimeRunId: 'opencode-run-1',
|
||||
runtimeSessionId: 'session-bob-1',
|
||||
livenessKind: 'confirmed_bootstrap' as const,
|
||||
lastEvaluatedAt: '2026-04-22T12:00:00.000Z',
|
||||
},
|
||||
},
|
||||
summary: {
|
||||
confirmedCount: 1,
|
||||
pendingCount: 0,
|
||||
failedCount: 0,
|
||||
runtimeAlivePendingCount: 0,
|
||||
},
|
||||
teamLaunchState: 'ready' as const,
|
||||
};
|
||||
const updateLiveness = vi.spyOn(svc as any, 'updateOpenCodeRuntimeMemberLiveness');
|
||||
|
||||
(svc as any).launchStateStore = {
|
||||
read: vi.fn(async () => previousSnapshot),
|
||||
write: vi.fn(async () => {}),
|
||||
};
|
||||
(svc as any).resolveOpenCodeRuntimeLaneId = vi.fn(async () => 'secondary:opencode:bob');
|
||||
(svc as any).assertOpenCodeRuntimeEvidenceAccepted = vi.fn(async () => {});
|
||||
|
||||
await expect(
|
||||
svc.recordOpenCodeRuntimeBootstrapCheckin({
|
||||
teamName: 'mixed-team',
|
||||
runId: 'opencode-run-1',
|
||||
memberName: 'bob',
|
||||
runtimeSessionId: 'session-bob-2',
|
||||
observedAt: '2026-04-22T12:05:00.000Z',
|
||||
})
|
||||
).rejects.toMatchObject({
|
||||
name: 'RuntimeStaleEvidenceError',
|
||||
message: expect.stringContaining('opencode_bootstrap_checkin_session_conflict'),
|
||||
});
|
||||
expect(updateLiveness).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not let stale confirmed OpenCode evidence from an older run block a fresh check-in', async () => {
|
||||
const svc = new TeamProvisioningService();
|
||||
const previousSnapshot = {
|
||||
version: 2 as const,
|
||||
teamName: 'mixed-team',
|
||||
updatedAt: '2026-04-22T12:00:00.000Z',
|
||||
launchPhase: 'active' as const,
|
||||
expectedMembers: ['bob'],
|
||||
members: {
|
||||
bob: {
|
||||
name: 'bob',
|
||||
providerId: 'opencode' as const,
|
||||
laneId: 'secondary:opencode:bob',
|
||||
laneKind: 'secondary' as const,
|
||||
laneOwnerProviderId: 'opencode' as const,
|
||||
launchState: 'confirmed_alive' as const,
|
||||
agentToolAccepted: true,
|
||||
runtimeAlive: true,
|
||||
bootstrapConfirmed: true,
|
||||
hardFailure: false,
|
||||
runtimeRunId: 'opencode-run-old',
|
||||
runtimeSessionId: 'session-bob-old',
|
||||
livenessKind: 'confirmed_bootstrap' as const,
|
||||
lastEvaluatedAt: '2026-04-22T12:00:00.000Z',
|
||||
},
|
||||
},
|
||||
summary: {
|
||||
confirmedCount: 1,
|
||||
pendingCount: 0,
|
||||
failedCount: 0,
|
||||
runtimeAlivePendingCount: 0,
|
||||
},
|
||||
teamLaunchState: 'ready' as const,
|
||||
};
|
||||
const updateLiveness = vi
|
||||
.spyOn(svc as any, 'updateOpenCodeRuntimeMemberLiveness')
|
||||
.mockResolvedValue(undefined);
|
||||
|
||||
(svc as any).launchStateStore = {
|
||||
read: vi.fn(async () => previousSnapshot),
|
||||
write: vi.fn(async () => {}),
|
||||
};
|
||||
(svc as any).resolveOpenCodeRuntimeLaneId = vi.fn(async () => 'secondary:opencode:bob');
|
||||
(svc as any).assertOpenCodeRuntimeEvidenceAccepted = vi.fn(async () => {});
|
||||
(svc as any).configReader = {
|
||||
getConfig: vi.fn(async () => ({
|
||||
teamName: 'mixed-team',
|
||||
members: [{ name: 'bob', providerId: 'opencode' }],
|
||||
})),
|
||||
};
|
||||
(svc as any).membersMetaStore = {
|
||||
getMembers: vi.fn(async () => []),
|
||||
};
|
||||
|
||||
await expect(
|
||||
svc.recordOpenCodeRuntimeBootstrapCheckin({
|
||||
teamName: 'mixed-team',
|
||||
runId: 'opencode-run-new',
|
||||
memberName: 'bob',
|
||||
runtimeSessionId: 'session-bob-new',
|
||||
observedAt: '2026-04-22T12:05:00.000Z',
|
||||
})
|
||||
).resolves.toMatchObject({
|
||||
ok: true,
|
||||
state: 'accepted',
|
||||
runtimeSessionId: 'session-bob-new',
|
||||
});
|
||||
expect(updateLiveness).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
runId: 'opencode-run-new',
|
||||
runtimeSessionId: 'session-bob-new',
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it('rejects OpenCode bootstrap check-ins for removed members before writing runtime evidence', async () => {
|
||||
const svc = new TeamProvisioningService();
|
||||
const updateLiveness = vi.spyOn(svc as any, 'updateOpenCodeRuntimeMemberLiveness');
|
||||
|
||||
(svc as any).launchStateStore = {
|
||||
read: vi.fn(async () => null),
|
||||
write: vi.fn(async () => {}),
|
||||
};
|
||||
(svc as any).resolveOpenCodeRuntimeLaneId = vi.fn(async () => 'secondary:opencode:bob');
|
||||
(svc as any).assertOpenCodeRuntimeEvidenceAccepted = vi.fn(async () => {});
|
||||
(svc as any).configReader = {
|
||||
getConfig: vi.fn(async () => ({
|
||||
teamName: 'mixed-team',
|
||||
members: [{ name: 'bob', providerId: 'opencode', removedAt: 123 }],
|
||||
})),
|
||||
};
|
||||
(svc as any).membersMetaStore = {
|
||||
getMembers: vi.fn(async () => []),
|
||||
};
|
||||
|
||||
await expect(
|
||||
svc.recordOpenCodeRuntimeBootstrapCheckin({
|
||||
teamName: 'mixed-team',
|
||||
runId: 'opencode-run-1',
|
||||
memberName: 'bob',
|
||||
runtimeSessionId: 'session-bob',
|
||||
})
|
||||
).rejects.toMatchObject({
|
||||
name: 'RuntimeStaleEvidenceError',
|
||||
});
|
||||
expect(updateLiveness).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('accepts secondary OpenCode lane evidence using the lane run id instead of the lead run id', async () => {
|
||||
const svc = new TeamProvisioningService();
|
||||
|
||||
|
|
@ -9434,6 +9896,264 @@ describe('TeamProvisioningService', () => {
|
|||
expect(run.provisioningOutputParts.join('\n')).toContain('bootstrap confirmed via transcript');
|
||||
});
|
||||
|
||||
it('clears a live grace-window failure when member transcript later shows successful member_briefing', async () => {
|
||||
allowConsoleLogs();
|
||||
const teamName = 'zz-live-bootstrap-late-transcript-success';
|
||||
const leadSessionId = 'lead-session';
|
||||
const memberSessionId = 'jack-session';
|
||||
const projectPath = '/Users/test/proj';
|
||||
const projectId = '-Users-test-proj';
|
||||
const acceptedAt = new Date(Date.now() - 170_000).toISOString();
|
||||
const successAt = new Date(Date.now() - 5_000).toISOString();
|
||||
|
||||
writeLaunchConfig(teamName, projectPath, leadSessionId, ['jack']);
|
||||
|
||||
const projectRoot = path.join(tempProjectsBase, projectId);
|
||||
fs.mkdirSync(projectRoot, { recursive: true });
|
||||
fs.writeFileSync(
|
||||
path.join(projectRoot, `${memberSessionId}.jsonl`),
|
||||
[
|
||||
JSON.stringify({
|
||||
timestamp: acceptedAt,
|
||||
teamName,
|
||||
agentName: 'jack',
|
||||
type: 'user',
|
||||
message: {
|
||||
role: 'user',
|
||||
content: `You are bootstrapping into team "${teamName}" as member "jack".`,
|
||||
},
|
||||
}),
|
||||
JSON.stringify({
|
||||
timestamp: successAt,
|
||||
teamName,
|
||||
agentName: 'jack',
|
||||
type: 'user',
|
||||
message: {
|
||||
role: 'user',
|
||||
content: [
|
||||
{
|
||||
type: 'tool_result',
|
||||
tool_use_id: 'item_1',
|
||||
content: `Member briefing for jack on team "${teamName}" (${teamName}).\nTask briefing for jack:\nNo actionable tasks.`,
|
||||
is_error: false,
|
||||
},
|
||||
],
|
||||
},
|
||||
}),
|
||||
].join('\n') + '\n',
|
||||
'utf8'
|
||||
);
|
||||
|
||||
const svc = new TeamProvisioningService();
|
||||
const run = {
|
||||
runId: 'run-live-late-success-1',
|
||||
teamName,
|
||||
startedAt: new Date(Date.now() - 220_000).toISOString(),
|
||||
request: {
|
||||
members: [],
|
||||
},
|
||||
expectedMembers: ['jack'],
|
||||
memberSpawnStatuses: new Map([
|
||||
[
|
||||
'jack',
|
||||
{
|
||||
status: 'error',
|
||||
launchState: 'failed_to_start',
|
||||
error: 'Teammate did not join within the launch grace window.',
|
||||
updatedAt: new Date(Date.now() - 10_000).toISOString(),
|
||||
runtimeAlive: false,
|
||||
livenessSource: undefined,
|
||||
bootstrapConfirmed: false,
|
||||
hardFailure: true,
|
||||
hardFailureReason: 'Teammate did not join within the launch grace window.',
|
||||
agentToolAccepted: true,
|
||||
firstSpawnAcceptedAt: acceptedAt,
|
||||
lastHeartbeatAt: undefined,
|
||||
},
|
||||
],
|
||||
]),
|
||||
lastMemberSpawnAuditAt: Date.now(),
|
||||
provisioningOutputParts: [],
|
||||
activeToolCalls: new Map(),
|
||||
isLaunch: false,
|
||||
} as any;
|
||||
|
||||
await (svc as any).maybeAuditMemberSpawnStatuses(run);
|
||||
|
||||
expect(run.memberSpawnStatuses.get('jack')).toMatchObject({
|
||||
status: 'online',
|
||||
launchState: 'confirmed_alive',
|
||||
bootstrapConfirmed: true,
|
||||
hardFailure: false,
|
||||
});
|
||||
expect(run.memberSpawnStatuses.get('jack')?.hardFailureReason).toBeUndefined();
|
||||
expect(run.provisioningOutputParts.join('\n')).toContain('bootstrap confirmed via transcript');
|
||||
});
|
||||
|
||||
it('does not treat OpenCode member_briefing transcript success as runtime bootstrap evidence', async () => {
|
||||
allowConsoleLogs();
|
||||
const teamName = 'zz-opencode-bootstrap-transcript-not-evidence';
|
||||
const leadSessionId = 'lead-session';
|
||||
const memberSessionId = 'jack-opencode-session';
|
||||
const projectPath = '/Users/test/proj';
|
||||
const projectId = '-Users-test-proj';
|
||||
const acceptedAt = new Date(Date.now() - 30_000).toISOString();
|
||||
const successAt = new Date(Date.now() - 5_000).toISOString();
|
||||
|
||||
writeLaunchConfig(teamName, projectPath, leadSessionId, ['jack']);
|
||||
|
||||
const projectRoot = path.join(tempProjectsBase, projectId);
|
||||
fs.mkdirSync(projectRoot, { recursive: true });
|
||||
fs.writeFileSync(
|
||||
path.join(projectRoot, `${memberSessionId}.jsonl`),
|
||||
[
|
||||
JSON.stringify({
|
||||
timestamp: acceptedAt,
|
||||
teamName,
|
||||
agentName: 'jack',
|
||||
type: 'user',
|
||||
message: {
|
||||
role: 'user',
|
||||
content: `You are bootstrapping into team "${teamName}" as member "jack".`,
|
||||
},
|
||||
}),
|
||||
JSON.stringify({
|
||||
timestamp: successAt,
|
||||
teamName,
|
||||
agentName: 'jack',
|
||||
type: 'user',
|
||||
message: {
|
||||
role: 'user',
|
||||
content: [
|
||||
{
|
||||
type: 'tool_result',
|
||||
tool_use_id: 'item_1',
|
||||
content: `Member briefing for jack on team "${teamName}" (${teamName}).\nTask briefing for jack:\nNo actionable tasks.`,
|
||||
is_error: false,
|
||||
},
|
||||
],
|
||||
},
|
||||
}),
|
||||
].join('\n') + '\n',
|
||||
'utf8'
|
||||
);
|
||||
|
||||
const svc = new TeamProvisioningService();
|
||||
const run = {
|
||||
runId: 'run-opencode-transcript-not-evidence',
|
||||
teamName,
|
||||
startedAt: new Date(Date.now() - 60_000).toISOString(),
|
||||
request: {
|
||||
members: [],
|
||||
},
|
||||
expectedMembers: ['jack'],
|
||||
mixedSecondaryLanes: [
|
||||
{
|
||||
laneId: 'secondary:opencode:jack',
|
||||
providerId: 'opencode',
|
||||
member: { name: 'jack', providerId: 'opencode', model: 'openrouter/qwen/qwen3-coder' },
|
||||
runId: 'opencode-run-jack',
|
||||
state: 'launching',
|
||||
result: null,
|
||||
warnings: [],
|
||||
diagnostics: [],
|
||||
},
|
||||
],
|
||||
memberSpawnStatuses: new Map([
|
||||
[
|
||||
'jack',
|
||||
{
|
||||
status: 'waiting',
|
||||
launchState: 'runtime_pending_bootstrap',
|
||||
error: undefined,
|
||||
updatedAt: acceptedAt,
|
||||
runtimeAlive: true,
|
||||
livenessSource: 'process',
|
||||
bootstrapConfirmed: false,
|
||||
hardFailure: false,
|
||||
agentToolAccepted: true,
|
||||
firstSpawnAcceptedAt: acceptedAt,
|
||||
lastHeartbeatAt: undefined,
|
||||
},
|
||||
],
|
||||
]),
|
||||
provisioningOutputParts: [],
|
||||
activeToolCalls: new Map(),
|
||||
isLaunch: false,
|
||||
} as any;
|
||||
|
||||
await (svc as any).reconcileBootstrapTranscriptSuccesses(run);
|
||||
|
||||
expect(run.memberSpawnStatuses.get('jack')).toMatchObject({
|
||||
status: 'waiting',
|
||||
launchState: 'runtime_pending_bootstrap',
|
||||
runtimeAlive: true,
|
||||
bootstrapConfirmed: false,
|
||||
});
|
||||
expect(run.provisioningOutputParts.join('\n')).not.toContain(
|
||||
'bootstrap confirmed via transcript'
|
||||
);
|
||||
});
|
||||
|
||||
it('does not copy bootstrap-state success into OpenCode secondary runtime evidence', async () => {
|
||||
const teamName = 'zz-opencode-bootstrap-state-not-evidence';
|
||||
const leadSessionId = 'lead-session';
|
||||
const acceptedAt = Date.now() - 30_000;
|
||||
const observedAt = Date.now() - 5_000;
|
||||
|
||||
writeTeamMeta(teamName, {
|
||||
providerId: 'codex',
|
||||
providerBackendId: 'codex-native',
|
||||
model: 'gpt-5.4',
|
||||
});
|
||||
writeMembersMeta(teamName, [
|
||||
{
|
||||
name: 'jack',
|
||||
providerId: 'opencode',
|
||||
model: 'openrouter/qwen/qwen3-coder',
|
||||
},
|
||||
]);
|
||||
writeLaunchConfig(teamName, '/Users/test/proj', leadSessionId, ['jack']);
|
||||
writeBootstrapState(
|
||||
teamName,
|
||||
[
|
||||
{
|
||||
name: 'jack',
|
||||
status: 'bootstrap_confirmed',
|
||||
lastAttemptAt: acceptedAt,
|
||||
lastObservedAt: observedAt,
|
||||
},
|
||||
],
|
||||
new Date(observedAt - 10_000).toISOString()
|
||||
);
|
||||
writeLaunchState(teamName, leadSessionId, {
|
||||
jack: {
|
||||
providerId: 'opencode',
|
||||
laneId: 'secondary:opencode:jack',
|
||||
laneKind: 'secondary',
|
||||
laneOwnerProviderId: 'opencode',
|
||||
launchState: 'runtime_pending_bootstrap',
|
||||
agentToolAccepted: true,
|
||||
runtimeAlive: true,
|
||||
bootstrapConfirmed: false,
|
||||
hardFailure: false,
|
||||
hardFailureReason: undefined,
|
||||
firstSpawnAcceptedAt: new Date(acceptedAt).toISOString(),
|
||||
lastRuntimeAliveAt: new Date(observedAt).toISOString(),
|
||||
lastEvaluatedAt: new Date().toISOString(),
|
||||
},
|
||||
});
|
||||
|
||||
const svc = new TeamProvisioningService();
|
||||
const result = await svc.getMemberSpawnStatuses(teamName);
|
||||
|
||||
expect(result.statuses.jack).toMatchObject({
|
||||
launchState: 'runtime_pending_bootstrap',
|
||||
runtimeAlive: false,
|
||||
bootstrapConfirmed: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('marks a live teammate bootstrap as confirmed from transcript even when runtime discovery is stale', async () => {
|
||||
allowConsoleLogs();
|
||||
const teamName = 'zz-live-bootstrap-transcript-success-without-runtime';
|
||||
|
|
@ -11144,6 +11864,59 @@ describe('TeamProvisioningService', () => {
|
|||
);
|
||||
});
|
||||
|
||||
it('degrades mixed secondary lanes when lanes.json is active but the lane manifest has no runtime evidence', async () => {
|
||||
const teamName = 'atlas-hq-empty-lane';
|
||||
writeTeamMeta(teamName, {
|
||||
providerId: 'codex',
|
||||
providerBackendId: 'codex-native',
|
||||
model: 'gpt-5.4',
|
||||
});
|
||||
writeMembersMeta(teamName, [
|
||||
{
|
||||
name: 'bob',
|
||||
providerId: 'opencode',
|
||||
model: 'openrouter/moonshotai/kimi-k2.6',
|
||||
},
|
||||
{
|
||||
name: 'jack',
|
||||
providerId: 'codex',
|
||||
model: 'gpt-5.4',
|
||||
},
|
||||
]);
|
||||
writeLaunchConfig(teamName, '/Users/test/proj', 'lead-session', ['jack']);
|
||||
writeBootstrapState(teamName, [{ name: 'jack', status: 'registered' }]);
|
||||
await upsertOpenCodeRuntimeLaneIndexEntry({
|
||||
teamsBasePath: tempTeamsBase,
|
||||
teamName,
|
||||
laneId: 'secondary:opencode:bob',
|
||||
state: 'active',
|
||||
});
|
||||
await setOpenCodeRuntimeActiveRunManifest({
|
||||
teamsBasePath: tempTeamsBase,
|
||||
teamName,
|
||||
laneId: 'secondary:opencode:bob',
|
||||
runId: 'run-empty-bob',
|
||||
clock: () => new Date('2026-04-20T10:00:00.000Z'),
|
||||
});
|
||||
|
||||
const svc = new TeamProvisioningService();
|
||||
const result = await svc.getMemberSpawnStatuses(teamName);
|
||||
|
||||
expect(result.teamLaunchState).toBe('partial_failure');
|
||||
expect(result.statuses.bob).toMatchObject({
|
||||
status: 'error',
|
||||
launchState: 'failed_to_start',
|
||||
error: expect.stringContaining('no committed runtime evidence after launch grace'),
|
||||
});
|
||||
await expect(readOpenCodeRuntimeLaneIndex(tempTeamsBase, teamName)).resolves.toMatchObject({
|
||||
lanes: {
|
||||
'secondary:opencode:bob': {
|
||||
state: 'degraded',
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('recovers stale mixed secondary lanes from live OpenCode runtime reconcile before degrading them', async () => {
|
||||
const teamName = 'relay-works-7';
|
||||
writeTeamMeta(teamName, {
|
||||
|
|
@ -11282,6 +12055,80 @@ describe('TeamProvisioningService', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('does not keep an empty active OpenCode lane pending when runtime reconcile has no runtime handle', async () => {
|
||||
const teamName = 'atlas-hq-empty-lane-nonrecoverable';
|
||||
writeTeamMeta(teamName, {
|
||||
providerId: 'codex',
|
||||
providerBackendId: 'codex-native',
|
||||
model: 'gpt-5.4',
|
||||
});
|
||||
writeMembersMeta(teamName, [
|
||||
{
|
||||
name: 'bob',
|
||||
providerId: 'opencode',
|
||||
model: 'openrouter/moonshotai/kimi-k2.6',
|
||||
},
|
||||
]);
|
||||
writeLaunchConfig(teamName, '/Users/test/proj', 'lead-session', []);
|
||||
await upsertOpenCodeRuntimeLaneIndexEntry({
|
||||
teamsBasePath: tempTeamsBase,
|
||||
teamName,
|
||||
laneId: 'secondary:opencode:bob',
|
||||
state: 'active',
|
||||
});
|
||||
await setOpenCodeRuntimeActiveRunManifest({
|
||||
teamsBasePath: tempTeamsBase,
|
||||
teamName,
|
||||
laneId: 'secondary:opencode:bob',
|
||||
runId: 'run-empty-bob',
|
||||
clock: () => new Date('2026-04-20T10:00:00.000Z'),
|
||||
});
|
||||
|
||||
const adapterReconcile = vi.fn(async () => ({
|
||||
runId: 'reconcile-run',
|
||||
teamName,
|
||||
launchPhase: 'reconciled' as const,
|
||||
teamLaunchState: 'partial_pending' as const,
|
||||
members: {
|
||||
bob: {
|
||||
memberName: 'bob',
|
||||
providerId: 'opencode' as const,
|
||||
launchState: 'runtime_pending_bootstrap' as const,
|
||||
agentToolAccepted: false,
|
||||
runtimeAlive: false,
|
||||
bootstrapConfirmed: false,
|
||||
hardFailure: false,
|
||||
livenessKind: 'registered_only' as const,
|
||||
diagnostics: ['bridge has no runtime session'],
|
||||
},
|
||||
},
|
||||
snapshot: null,
|
||||
warnings: [],
|
||||
diagnostics: [],
|
||||
}));
|
||||
const svc = new TeamProvisioningService();
|
||||
svc.setRuntimeAdapterRegistry(
|
||||
new TeamRuntimeAdapterRegistry([
|
||||
{
|
||||
providerId: 'opencode',
|
||||
prepare: vi.fn(),
|
||||
launch: vi.fn(),
|
||||
reconcile: adapterReconcile,
|
||||
stop: vi.fn(),
|
||||
} as any,
|
||||
])
|
||||
);
|
||||
|
||||
const result = await svc.getMemberSpawnStatuses(teamName);
|
||||
|
||||
expect(adapterReconcile).toHaveBeenCalledTimes(1);
|
||||
expect(result.statuses.bob).toMatchObject({
|
||||
status: 'error',
|
||||
launchState: 'failed_to_start',
|
||||
error: expect.stringContaining('no committed runtime evidence after launch grace'),
|
||||
});
|
||||
});
|
||||
|
||||
it('recovers missing mixed secondary lane index from materialized OpenCode runtime evidence', async () => {
|
||||
const teamName = 'relay-works-missing-lane-recovery';
|
||||
writeTeamMeta(teamName, {
|
||||
|
|
|
|||
Loading…
Reference in a new issue