fix(team): suppress replayed member spawn inbox churn
This commit is contained in:
parent
821e23e633
commit
d8672c32f6
2 changed files with 570 additions and 17 deletions
|
|
@ -700,6 +700,13 @@ interface ProvisioningRun {
|
||||||
>;
|
>;
|
||||||
/** Agent tool_use_id -> teammate name for persistent teammate spawns. */
|
/** Agent tool_use_id -> teammate name for persistent teammate spawns. */
|
||||||
memberSpawnToolUseIds: Map<string, string>;
|
memberSpawnToolUseIds: Map<string, string>;
|
||||||
|
/** Per-member latest processed lead-inbox bootstrap signal cursor for the current live run. */
|
||||||
|
memberSpawnLeadInboxCursorByMember: Map<string, MemberSpawnInboxCursor>;
|
||||||
|
/**
|
||||||
|
* Per-member exact processed lead-inbox messageIds for the current live run.
|
||||||
|
* This owns live-path correctness and protects against out-of-order inserts.
|
||||||
|
*/
|
||||||
|
memberSpawnProcessedLeadInboxMessageIdsByMember: Map<string, Set<string>>;
|
||||||
/** Highest accepted deterministic bootstrap event sequence for this run. */
|
/** Highest accepted deterministic bootstrap event sequence for this run. */
|
||||||
lastDeterministicBootstrapSeq: number;
|
lastDeterministicBootstrapSeq: number;
|
||||||
/** Throttles config/inbox audit work triggered by frequent status polling. */
|
/** Throttles config/inbox audit work triggered by frequent status polling. */
|
||||||
|
|
@ -787,6 +794,88 @@ interface LiveTeamAgentRuntimeMetadata {
|
||||||
model?: string;
|
model?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
interface MemberSpawnInboxCursor {
|
||||||
|
timestamp: string;
|
||||||
|
messageId: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
type LeadInboxMemberSpawnMessage = InboxMessage & { messageId: string };
|
||||||
|
|
||||||
|
function compareMemberSpawnInboxCursor(
|
||||||
|
left: MemberSpawnInboxCursor,
|
||||||
|
right: MemberSpawnInboxCursor
|
||||||
|
): number {
|
||||||
|
const leftMs = Date.parse(left.timestamp);
|
||||||
|
const rightMs = Date.parse(right.timestamp);
|
||||||
|
const leftValid = Number.isFinite(leftMs);
|
||||||
|
const rightValid = Number.isFinite(rightMs);
|
||||||
|
|
||||||
|
if (leftValid && rightValid && leftMs !== rightMs) {
|
||||||
|
return leftMs - rightMs;
|
||||||
|
}
|
||||||
|
if (leftValid !== rightValid) {
|
||||||
|
return leftValid ? -1 : 1;
|
||||||
|
}
|
||||||
|
return left.messageId.localeCompare(right.messageId);
|
||||||
|
}
|
||||||
|
|
||||||
|
function toMemberSpawnInboxCursor(
|
||||||
|
message: Pick<InboxMessage, 'timestamp' | 'messageId'>
|
||||||
|
): MemberSpawnInboxCursor | null {
|
||||||
|
const messageId = typeof message.messageId === 'string' ? message.messageId.trim() : '';
|
||||||
|
if (!messageId) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
timestamp: message.timestamp,
|
||||||
|
messageId,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function maxMemberSpawnInboxCursor(
|
||||||
|
left: MemberSpawnInboxCursor | undefined,
|
||||||
|
right: MemberSpawnInboxCursor
|
||||||
|
): MemberSpawnInboxCursor {
|
||||||
|
if (!left) {
|
||||||
|
return right;
|
||||||
|
}
|
||||||
|
return compareMemberSpawnInboxCursor(left, right) >= 0 ? left : right;
|
||||||
|
}
|
||||||
|
|
||||||
|
function getOrCreateMemberSpawnProcessedMessageIds(
|
||||||
|
run: ProvisioningRun,
|
||||||
|
memberName: string
|
||||||
|
): Set<string> {
|
||||||
|
const existing = run.memberSpawnProcessedLeadInboxMessageIdsByMember.get(memberName);
|
||||||
|
if (existing) {
|
||||||
|
return existing;
|
||||||
|
}
|
||||||
|
const created = new Set<string>();
|
||||||
|
run.memberSpawnProcessedLeadInboxMessageIdsByMember.set(memberName, created);
|
||||||
|
return created;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isMemberSpawnHeartbeatTimestampNewer(
|
||||||
|
previous: string | undefined,
|
||||||
|
incoming: string | undefined
|
||||||
|
): boolean {
|
||||||
|
const normalizedIncoming = incoming?.trim();
|
||||||
|
if (!normalizedIncoming) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
const normalizedPrevious = previous?.trim();
|
||||||
|
if (!normalizedPrevious) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
const previousMs = Date.parse(normalizedPrevious);
|
||||||
|
const incomingMs = Date.parse(normalizedIncoming);
|
||||||
|
if (Number.isFinite(previousMs) && Number.isFinite(incomingMs)) {
|
||||||
|
return incomingMs > previousMs;
|
||||||
|
}
|
||||||
|
return normalizedIncoming > normalizedPrevious;
|
||||||
|
}
|
||||||
|
|
||||||
function stripWrappedCliFlagValue(raw: string | undefined): string | undefined {
|
function stripWrappedCliFlagValue(raw: string | undefined): string | undefined {
|
||||||
const trimmed = raw?.trim();
|
const trimmed = raw?.trim();
|
||||||
if (!trimmed) {
|
if (!trimmed) {
|
||||||
|
|
@ -2841,12 +2930,15 @@ export class TeamProvisioningService {
|
||||||
}
|
}
|
||||||
|
|
||||||
const runStartedAtMs = Date.parse(run.startedAt);
|
const runStartedAtMs = Date.parse(run.startedAt);
|
||||||
const expectedMembers = Array.isArray(run.expectedMembers) ? run.expectedMembers : [];
|
const expectedMembers = new Set(Array.isArray(run.expectedMembers) ? run.expectedMembers : []);
|
||||||
const teammateMessages = leadInboxMessages
|
const teammateMessages = leadInboxMessages
|
||||||
.filter((message) => {
|
.filter((message): message is LeadInboxMemberSpawnMessage => {
|
||||||
const from = typeof message.from === 'string' ? message.from.trim() : '';
|
const from = typeof message.from === 'string' ? message.from.trim() : '';
|
||||||
if (!from || from === leadName || from === 'user' || from === 'system') return false;
|
if (!from || from === leadName || from === 'user' || from === 'system') return false;
|
||||||
if (!expectedMembers.includes(from)) return false;
|
if (!expectedMembers.has(from)) return false;
|
||||||
|
if (typeof message.messageId !== 'string' || message.messageId.trim().length === 0) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
const messageTs = Date.parse(message.timestamp);
|
const messageTs = Date.parse(message.timestamp);
|
||||||
if (
|
if (
|
||||||
Number.isFinite(messageTs) &&
|
Number.isFinite(messageTs) &&
|
||||||
|
|
@ -2857,26 +2949,83 @@ export class TeamProvisioningService {
|
||||||
}
|
}
|
||||||
return typeof message.text === 'string' && message.text.trim().length > 0;
|
return typeof message.text === 'string' && message.text.trim().length > 0;
|
||||||
})
|
})
|
||||||
.sort((a, b) => Date.parse(a.timestamp) - Date.parse(b.timestamp));
|
.sort((left, right) =>
|
||||||
|
compareMemberSpawnInboxCursor(
|
||||||
|
{ timestamp: left.timestamp, messageId: left.messageId },
|
||||||
|
{ timestamp: right.timestamp, messageId: right.messageId }
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
const messagesByMember = new Map<string, LeadInboxMemberSpawnMessage[]>();
|
||||||
for (const message of teammateMessages) {
|
for (const message of teammateMessages) {
|
||||||
const from = message.from.trim();
|
const memberName = message.from.trim();
|
||||||
const reason = extractBootstrapFailureReason(message.text);
|
const bucket = messagesByMember.get(memberName) ?? [];
|
||||||
if (reason) {
|
bucket.push(message);
|
||||||
this.setMemberSpawnStatus(run, from, 'error', reason);
|
messagesByMember.set(memberName, bucket);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const [memberName, messages] of messagesByMember.entries()) {
|
||||||
|
const processedMessageIds = getOrCreateMemberSpawnProcessedMessageIds(run, memberName);
|
||||||
|
const currentCursor = run.memberSpawnLeadInboxCursorByMember.get(memberName);
|
||||||
|
const newlyProcessedMessageIds: string[] = [];
|
||||||
|
let nextCursor = currentCursor;
|
||||||
|
|
||||||
|
for (const message of messages) {
|
||||||
|
if (processedMessageIds.has(message.messageId)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const messageCursor = toMemberSpawnInboxCursor(message);
|
||||||
|
const shouldApplySignal =
|
||||||
|
messageCursor == null ||
|
||||||
|
currentCursor == null ||
|
||||||
|
compareMemberSpawnInboxCursor(messageCursor, currentCursor) > 0;
|
||||||
|
|
||||||
|
if (shouldApplySignal) {
|
||||||
|
this.applyLeadInboxSpawnSignal(run, memberName, message);
|
||||||
|
if (messageCursor) {
|
||||||
|
nextCursor = maxMemberSpawnInboxCursor(nextCursor, messageCursor);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Mark late out-of-order signals as seen so they cannot replay forever, but only
|
||||||
|
// let strictly newer cursors mutate the already-advanced live member state.
|
||||||
|
newlyProcessedMessageIds.push(message.messageId);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (newlyProcessedMessageIds.length === 0) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
this.setMemberSpawnStatus(
|
|
||||||
run,
|
for (const messageId of newlyProcessedMessageIds) {
|
||||||
from,
|
processedMessageIds.add(messageId);
|
||||||
'online',
|
}
|
||||||
undefined,
|
if (nextCursor) {
|
||||||
'heartbeat',
|
run.memberSpawnLeadInboxCursorByMember.set(memberName, nextCursor);
|
||||||
extractHeartbeatTimestamp(message.text, message.timestamp)
|
}
|
||||||
);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private applyLeadInboxSpawnSignal(
|
||||||
|
run: ProvisioningRun,
|
||||||
|
memberName: string,
|
||||||
|
message: LeadInboxMemberSpawnMessage
|
||||||
|
): void {
|
||||||
|
const reason = extractBootstrapFailureReason(message.text);
|
||||||
|
if (reason) {
|
||||||
|
this.setMemberSpawnStatus(run, memberName, 'error', reason);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
this.setMemberSpawnStatus(
|
||||||
|
run,
|
||||||
|
memberName,
|
||||||
|
'online',
|
||||||
|
undefined,
|
||||||
|
'heartbeat',
|
||||||
|
extractHeartbeatTimestamp(message.text, message.timestamp)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
private persistSentMessage(teamName: string, message: InboxMessage): void {
|
private persistSentMessage(teamName: string, message: InboxMessage): void {
|
||||||
try {
|
try {
|
||||||
createController({
|
createController({
|
||||||
|
|
@ -3341,8 +3490,14 @@ export class TeamProvisioningService {
|
||||||
next.livenessSource = livenessSource;
|
next.livenessSource = livenessSource;
|
||||||
next.firstSpawnAcceptedAt = prev.firstSpawnAcceptedAt ?? updatedAt;
|
next.firstSpawnAcceptedAt = prev.firstSpawnAcceptedAt ?? updatedAt;
|
||||||
if (livenessSource === 'heartbeat') {
|
if (livenessSource === 'heartbeat') {
|
||||||
|
const incomingHeartbeatAt = heartbeatAt?.trim() || updatedAt;
|
||||||
next.bootstrapConfirmed = true;
|
next.bootstrapConfirmed = true;
|
||||||
next.lastHeartbeatAt = heartbeatAt?.trim() || prev.lastHeartbeatAt || updatedAt;
|
next.lastHeartbeatAt = isMemberSpawnHeartbeatTimestampNewer(
|
||||||
|
prev.lastHeartbeatAt,
|
||||||
|
incomingHeartbeatAt
|
||||||
|
)
|
||||||
|
? incomingHeartbeatAt
|
||||||
|
: prev.lastHeartbeatAt;
|
||||||
}
|
}
|
||||||
next.hardFailure = false;
|
next.hardFailure = false;
|
||||||
next.error = undefined;
|
next.error = undefined;
|
||||||
|
|
@ -4971,6 +5126,8 @@ export class TeamProvisioningService {
|
||||||
request.members.map((m) => [m.name, createInitialMemberSpawnStatusEntry()])
|
request.members.map((m) => [m.name, createInitialMemberSpawnStatusEntry()])
|
||||||
),
|
),
|
||||||
memberSpawnToolUseIds: new Map(),
|
memberSpawnToolUseIds: new Map(),
|
||||||
|
memberSpawnLeadInboxCursorByMember: new Map(),
|
||||||
|
memberSpawnProcessedLeadInboxMessageIdsByMember: new Map(),
|
||||||
lastDeterministicBootstrapSeq: 0,
|
lastDeterministicBootstrapSeq: 0,
|
||||||
lastMemberSpawnAuditAt: 0,
|
lastMemberSpawnAuditAt: 0,
|
||||||
lastMemberSpawnAuditConfigReadWarningAt: 0,
|
lastMemberSpawnAuditConfigReadWarningAt: 0,
|
||||||
|
|
@ -5550,6 +5707,8 @@ export class TeamProvisioningService {
|
||||||
expectedMembers.map((name) => [name, createInitialMemberSpawnStatusEntry()])
|
expectedMembers.map((name) => [name, createInitialMemberSpawnStatusEntry()])
|
||||||
),
|
),
|
||||||
memberSpawnToolUseIds: new Map(),
|
memberSpawnToolUseIds: new Map(),
|
||||||
|
memberSpawnLeadInboxCursorByMember: new Map(),
|
||||||
|
memberSpawnProcessedLeadInboxMessageIdsByMember: new Map(),
|
||||||
lastDeterministicBootstrapSeq: 0,
|
lastDeterministicBootstrapSeq: 0,
|
||||||
lastMemberSpawnAuditAt: 0,
|
lastMemberSpawnAuditAt: 0,
|
||||||
lastMemberSpawnAuditConfigReadWarningAt: 0,
|
lastMemberSpawnAuditConfigReadWarningAt: 0,
|
||||||
|
|
|
||||||
|
|
@ -143,6 +143,68 @@ function writeLaunchState(
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function createMemberSpawnStatusEntry(
|
||||||
|
overrides: Record<string, unknown> = {}
|
||||||
|
): Record<string, unknown> {
|
||||||
|
return {
|
||||||
|
status: 'waiting',
|
||||||
|
launchState: 'runtime_pending_bootstrap',
|
||||||
|
error: undefined,
|
||||||
|
updatedAt: new Date().toISOString(),
|
||||||
|
runtimeAlive: false,
|
||||||
|
livenessSource: undefined,
|
||||||
|
bootstrapConfirmed: false,
|
||||||
|
hardFailure: false,
|
||||||
|
agentToolAccepted: true,
|
||||||
|
firstSpawnAcceptedAt: new Date().toISOString(),
|
||||||
|
lastHeartbeatAt: undefined,
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function createMemberSpawnRun(params?: {
|
||||||
|
runId?: string;
|
||||||
|
teamName?: string;
|
||||||
|
startedAt?: string;
|
||||||
|
expectedMembers?: string[];
|
||||||
|
memberSpawnStatuses?: Map<string, Record<string, unknown>>;
|
||||||
|
memberSpawnLeadInboxCursorByMember?: Map<string, { timestamp: string; messageId: string }>;
|
||||||
|
memberSpawnProcessedLeadInboxMessageIdsByMember?: Map<string, Set<string>>;
|
||||||
|
}) {
|
||||||
|
const teamName = params?.teamName ?? 'member-spawn-team';
|
||||||
|
const expectedMembers = params?.expectedMembers ?? ['alice'];
|
||||||
|
const memberSpawnStatuses =
|
||||||
|
params?.memberSpawnStatuses ??
|
||||||
|
new Map([
|
||||||
|
[
|
||||||
|
expectedMembers[0]!,
|
||||||
|
createMemberSpawnStatusEntry({
|
||||||
|
firstSpawnAcceptedAt: new Date(Date.now() - 5_000).toISOString(),
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
]);
|
||||||
|
|
||||||
|
return {
|
||||||
|
runId: params?.runId ?? 'run-member-spawn-1',
|
||||||
|
teamName,
|
||||||
|
startedAt: params?.startedAt ?? new Date(Date.now() - 60_000).toISOString(),
|
||||||
|
request: {
|
||||||
|
members: [],
|
||||||
|
},
|
||||||
|
expectedMembers,
|
||||||
|
memberSpawnStatuses,
|
||||||
|
memberSpawnToolUseIds: new Map(),
|
||||||
|
memberSpawnLeadInboxCursorByMember:
|
||||||
|
params?.memberSpawnLeadInboxCursorByMember ?? new Map(),
|
||||||
|
memberSpawnProcessedLeadInboxMessageIdsByMember:
|
||||||
|
params?.memberSpawnProcessedLeadInboxMessageIdsByMember ?? new Map(),
|
||||||
|
provisioningOutputParts: [],
|
||||||
|
activeToolCalls: new Map(),
|
||||||
|
isLaunch: false,
|
||||||
|
provisioningComplete: false,
|
||||||
|
} as any;
|
||||||
|
}
|
||||||
|
|
||||||
describe('TeamProvisioningService', () => {
|
describe('TeamProvisioningService', () => {
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
vi.clearAllMocks();
|
vi.clearAllMocks();
|
||||||
|
|
@ -919,4 +981,336 @@ describe('TeamProvisioningService', () => {
|
||||||
expect(result.statuses.jack?.hardFailureReason).toContain('requested model is not available');
|
expect(result.statuses.jack?.hardFailureReason).toContain('requested model is not available');
|
||||||
expect(result.teamLaunchState).toBe('partial_failure');
|
expect(result.teamLaunchState).toBe('partial_failure');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('does not reprocess already-seen teammate lead inbox messages', async () => {
|
||||||
|
const svc = new TeamProvisioningService();
|
||||||
|
const run = createMemberSpawnRun({
|
||||||
|
startedAt: '2026-04-16T09:00:00.000Z',
|
||||||
|
memberSpawnProcessedLeadInboxMessageIdsByMember: new Map([
|
||||||
|
['alice', new Set(['msg-1', 'msg-2'])],
|
||||||
|
]),
|
||||||
|
memberSpawnLeadInboxCursorByMember: new Map([
|
||||||
|
[
|
||||||
|
'alice',
|
||||||
|
{
|
||||||
|
timestamp: '2026-04-16T10:00:00.000Z',
|
||||||
|
messageId: 'msg-2',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
]),
|
||||||
|
});
|
||||||
|
|
||||||
|
vi.spyOn((svc as any).inboxReader, 'getMessagesFor').mockResolvedValue([
|
||||||
|
{
|
||||||
|
from: 'alice',
|
||||||
|
text: 'heartbeat',
|
||||||
|
timestamp: '2026-04-16T10:00:00.000Z',
|
||||||
|
messageId: 'msg-1',
|
||||||
|
read: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
from: 'alice',
|
||||||
|
text: 'heartbeat',
|
||||||
|
timestamp: '2026-04-16T10:00:00.000Z',
|
||||||
|
messageId: 'msg-2',
|
||||||
|
read: false,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
const applySignalSpy = vi.spyOn(svc as any, 'applyLeadInboxSpawnSignal');
|
||||||
|
|
||||||
|
await (svc as any).refreshMemberSpawnStatusesFromLeadInbox(run);
|
||||||
|
|
||||||
|
expect(applySignalSpy).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('processes an unseen teammate heartbeat on the first refresh', async () => {
|
||||||
|
const svc = new TeamProvisioningService();
|
||||||
|
const run = createMemberSpawnRun({
|
||||||
|
startedAt: '2026-04-16T09:00:00.000Z',
|
||||||
|
});
|
||||||
|
|
||||||
|
vi.spyOn((svc as any).inboxReader, 'getMessagesFor').mockResolvedValue([
|
||||||
|
{
|
||||||
|
from: 'alice',
|
||||||
|
text: '{"type":"heartbeat","timestamp":"2026-04-16T10:00:00.000Z"}',
|
||||||
|
timestamp: '2026-04-16T10:00:00.000Z',
|
||||||
|
messageId: 'msg-1',
|
||||||
|
read: false,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
await (svc as any).refreshMemberSpawnStatusesFromLeadInbox(run);
|
||||||
|
|
||||||
|
expect(run.memberSpawnStatuses.get('alice')).toMatchObject({
|
||||||
|
status: 'online',
|
||||||
|
launchState: 'confirmed_alive',
|
||||||
|
bootstrapConfirmed: true,
|
||||||
|
hardFailure: false,
|
||||||
|
lastHeartbeatAt: '2026-04-16T10:00:00.000Z',
|
||||||
|
});
|
||||||
|
expect(run.memberSpawnLeadInboxCursorByMember.get('alice')).toEqual({
|
||||||
|
timestamp: '2026-04-16T10:00:00.000Z',
|
||||||
|
messageId: 'msg-1',
|
||||||
|
});
|
||||||
|
expect(run.memberSpawnProcessedLeadInboxMessageIdsByMember.get('alice')).toEqual(
|
||||||
|
new Set(['msg-1'])
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('ignores teammate lead inbox signals that predate the current run', async () => {
|
||||||
|
const svc = new TeamProvisioningService();
|
||||||
|
const run = createMemberSpawnRun({
|
||||||
|
startedAt: '2026-04-16T10:00:00.000Z',
|
||||||
|
});
|
||||||
|
|
||||||
|
vi.spyOn((svc as any).inboxReader, 'getMessagesFor').mockResolvedValue([
|
||||||
|
{
|
||||||
|
from: 'alice',
|
||||||
|
text: '{"type":"heartbeat","timestamp":"2026-04-16T09:59:59.000Z"}',
|
||||||
|
timestamp: '2026-04-16T09:59:59.000Z',
|
||||||
|
messageId: 'msg-early',
|
||||||
|
read: false,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
const applySignalSpy = vi.spyOn(svc as any, 'applyLeadInboxSpawnSignal');
|
||||||
|
|
||||||
|
await (svc as any).refreshMemberSpawnStatusesFromLeadInbox(run);
|
||||||
|
|
||||||
|
expect(applySignalSpy).not.toHaveBeenCalled();
|
||||||
|
expect(run.memberSpawnLeadInboxCursorByMember.size).toBe(0);
|
||||||
|
expect(run.memberSpawnProcessedLeadInboxMessageIdsByMember.size).toBe(0);
|
||||||
|
expect(run.memberSpawnStatuses.get('alice')).toMatchObject({
|
||||||
|
status: 'waiting',
|
||||||
|
launchState: 'runtime_pending_bootstrap',
|
||||||
|
bootstrapConfirmed: false,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('marks an unseen older lead inbox signal as processed without replaying older state', async () => {
|
||||||
|
const latestHeartbeatAt = '2026-04-16T10:05:00.000Z';
|
||||||
|
const existingEntry = createMemberSpawnStatusEntry({
|
||||||
|
status: 'online',
|
||||||
|
launchState: 'confirmed_alive',
|
||||||
|
runtimeAlive: true,
|
||||||
|
livenessSource: 'heartbeat',
|
||||||
|
bootstrapConfirmed: true,
|
||||||
|
lastHeartbeatAt: latestHeartbeatAt,
|
||||||
|
});
|
||||||
|
const run = createMemberSpawnRun({
|
||||||
|
startedAt: '2026-04-16T09:00:00.000Z',
|
||||||
|
memberSpawnStatuses: new Map([['alice', existingEntry]]),
|
||||||
|
memberSpawnProcessedLeadInboxMessageIdsByMember: new Map([
|
||||||
|
['alice', new Set(['msg-3'])],
|
||||||
|
]),
|
||||||
|
memberSpawnLeadInboxCursorByMember: new Map([
|
||||||
|
[
|
||||||
|
'alice',
|
||||||
|
{
|
||||||
|
timestamp: latestHeartbeatAt,
|
||||||
|
messageId: 'msg-3',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
]),
|
||||||
|
});
|
||||||
|
const svc = new TeamProvisioningService();
|
||||||
|
|
||||||
|
vi.spyOn((svc as any).inboxReader, 'getMessagesFor').mockResolvedValue([
|
||||||
|
{
|
||||||
|
from: 'alice',
|
||||||
|
text: 'Bootstrap failed: unsupported model',
|
||||||
|
timestamp: '2026-04-16T10:04:00.000Z',
|
||||||
|
messageId: 'msg-2b',
|
||||||
|
read: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
from: 'alice',
|
||||||
|
text: 'heartbeat',
|
||||||
|
timestamp: latestHeartbeatAt,
|
||||||
|
messageId: 'msg-3',
|
||||||
|
read: false,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
const applySignalSpy = vi.spyOn(svc as any, 'applyLeadInboxSpawnSignal');
|
||||||
|
|
||||||
|
await (svc as any).refreshMemberSpawnStatusesFromLeadInbox(run);
|
||||||
|
|
||||||
|
expect(applySignalSpy).not.toHaveBeenCalled();
|
||||||
|
expect(run.memberSpawnStatuses.get('alice')).toBe(existingEntry);
|
||||||
|
expect(
|
||||||
|
run.memberSpawnProcessedLeadInboxMessageIdsByMember.get('alice')?.has('msg-2b')
|
||||||
|
).toBe(true);
|
||||||
|
expect(run.memberSpawnLeadInboxCursorByMember.get('alice')).toEqual({
|
||||||
|
timestamp: latestHeartbeatAt,
|
||||||
|
messageId: 'msg-3',
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('applies an unseen newer failure signal and transitions the member to failed_to_start', async () => {
|
||||||
|
const latestHeartbeatAt = '2026-04-16T10:00:00.000Z';
|
||||||
|
const run = createMemberSpawnRun({
|
||||||
|
startedAt: '2026-04-16T09:00:00.000Z',
|
||||||
|
memberSpawnStatuses: new Map([
|
||||||
|
[
|
||||||
|
'alice',
|
||||||
|
createMemberSpawnStatusEntry({
|
||||||
|
status: 'online',
|
||||||
|
launchState: 'confirmed_alive',
|
||||||
|
runtimeAlive: true,
|
||||||
|
livenessSource: 'heartbeat',
|
||||||
|
bootstrapConfirmed: true,
|
||||||
|
lastHeartbeatAt: latestHeartbeatAt,
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
]),
|
||||||
|
memberSpawnProcessedLeadInboxMessageIdsByMember: new Map([
|
||||||
|
['alice', new Set(['msg-1'])],
|
||||||
|
]),
|
||||||
|
memberSpawnLeadInboxCursorByMember: new Map([
|
||||||
|
[
|
||||||
|
'alice',
|
||||||
|
{
|
||||||
|
timestamp: latestHeartbeatAt,
|
||||||
|
messageId: 'msg-1',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
]),
|
||||||
|
});
|
||||||
|
const svc = new TeamProvisioningService();
|
||||||
|
|
||||||
|
vi.spyOn((svc as any).inboxReader, 'getMessagesFor').mockResolvedValue([
|
||||||
|
{
|
||||||
|
from: 'alice',
|
||||||
|
text: 'Bootstrap failed: unsupported model',
|
||||||
|
timestamp: '2026-04-16T10:01:00.000Z',
|
||||||
|
messageId: 'msg-2',
|
||||||
|
read: false,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
await (svc as any).refreshMemberSpawnStatusesFromLeadInbox(run);
|
||||||
|
|
||||||
|
expect(run.memberSpawnStatuses.get('alice')).toMatchObject({
|
||||||
|
status: 'error',
|
||||||
|
launchState: 'failed_to_start',
|
||||||
|
hardFailure: true,
|
||||||
|
hardFailureReason: 'Bootstrap failed: unsupported model',
|
||||||
|
});
|
||||||
|
expect(run.memberSpawnLeadInboxCursorByMember.get('alice')).toEqual({
|
||||||
|
timestamp: '2026-04-16T10:01:00.000Z',
|
||||||
|
messageId: 'msg-2',
|
||||||
|
});
|
||||||
|
expect(run.memberSpawnProcessedLeadInboxMessageIdsByMember.get('alice')).toEqual(
|
||||||
|
new Set(['msg-1', 'msg-2'])
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('applies an unseen same-timestamp signal with a greater messageId and advances the cursor', async () => {
|
||||||
|
const run = createMemberSpawnRun({
|
||||||
|
startedAt: '2026-04-16T09:00:00.000Z',
|
||||||
|
memberSpawnProcessedLeadInboxMessageIdsByMember: new Map([
|
||||||
|
['alice', new Set(['msg-2'])],
|
||||||
|
]),
|
||||||
|
memberSpawnLeadInboxCursorByMember: new Map([
|
||||||
|
[
|
||||||
|
'alice',
|
||||||
|
{
|
||||||
|
timestamp: '2026-04-16T10:00:00.000Z',
|
||||||
|
messageId: 'msg-2',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
]),
|
||||||
|
});
|
||||||
|
const svc = new TeamProvisioningService();
|
||||||
|
|
||||||
|
vi.spyOn((svc as any).inboxReader, 'getMessagesFor').mockResolvedValue([
|
||||||
|
{
|
||||||
|
from: 'alice',
|
||||||
|
text: 'heartbeat',
|
||||||
|
timestamp: '2026-04-16T10:00:00.000Z',
|
||||||
|
messageId: 'msg-2',
|
||||||
|
read: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
from: 'alice',
|
||||||
|
text: 'heartbeat',
|
||||||
|
timestamp: '2026-04-16T10:00:00.000Z',
|
||||||
|
messageId: 'msg-3',
|
||||||
|
read: false,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
const applySignalSpy = vi.spyOn(svc as any, 'applyLeadInboxSpawnSignal');
|
||||||
|
|
||||||
|
await (svc as any).refreshMemberSpawnStatusesFromLeadInbox(run);
|
||||||
|
|
||||||
|
expect(applySignalSpy).toHaveBeenCalledTimes(1);
|
||||||
|
expect(applySignalSpy).toHaveBeenCalledWith(
|
||||||
|
run,
|
||||||
|
'alice',
|
||||||
|
expect.objectContaining({ messageId: 'msg-3' })
|
||||||
|
);
|
||||||
|
expect(run.memberSpawnLeadInboxCursorByMember.get('alice')).toEqual({
|
||||||
|
timestamp: '2026-04-16T10:00:00.000Z',
|
||||||
|
messageId: 'msg-3',
|
||||||
|
});
|
||||||
|
expect(
|
||||||
|
run.memberSpawnProcessedLeadInboxMessageIdsByMember.get('alice')
|
||||||
|
).toEqual(new Set(['msg-2', 'msg-3']));
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does not bump lastHeartbeatAt for an equal heartbeat timestamp', () => {
|
||||||
|
const existingEntry = createMemberSpawnStatusEntry({
|
||||||
|
status: 'online',
|
||||||
|
launchState: 'confirmed_alive',
|
||||||
|
runtimeAlive: true,
|
||||||
|
livenessSource: 'heartbeat',
|
||||||
|
bootstrapConfirmed: true,
|
||||||
|
lastHeartbeatAt: '2026-04-16T10:00:00.000Z',
|
||||||
|
});
|
||||||
|
const run = createMemberSpawnRun({
|
||||||
|
memberSpawnStatuses: new Map([['alice', existingEntry]]),
|
||||||
|
});
|
||||||
|
const svc = new TeamProvisioningService();
|
||||||
|
|
||||||
|
(svc as any).setMemberSpawnStatus(
|
||||||
|
run,
|
||||||
|
'alice',
|
||||||
|
'online',
|
||||||
|
undefined,
|
||||||
|
'heartbeat',
|
||||||
|
'2026-04-16T10:00:00.000Z'
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(run.memberSpawnStatuses.get('alice')).toBe(existingEntry);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does not bump lastHeartbeatAt for an older heartbeat timestamp', () => {
|
||||||
|
const existingEntry = createMemberSpawnStatusEntry({
|
||||||
|
status: 'online',
|
||||||
|
launchState: 'confirmed_alive',
|
||||||
|
runtimeAlive: true,
|
||||||
|
livenessSource: 'heartbeat',
|
||||||
|
bootstrapConfirmed: true,
|
||||||
|
lastHeartbeatAt: '2026-04-16T10:00:00.000Z',
|
||||||
|
});
|
||||||
|
const run = createMemberSpawnRun({
|
||||||
|
memberSpawnStatuses: new Map([['alice', existingEntry]]),
|
||||||
|
});
|
||||||
|
const svc = new TeamProvisioningService();
|
||||||
|
|
||||||
|
(svc as any).setMemberSpawnStatus(
|
||||||
|
run,
|
||||||
|
'alice',
|
||||||
|
'online',
|
||||||
|
undefined,
|
||||||
|
'heartbeat',
|
||||||
|
'2026-04-16T09:59:59.000Z'
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(run.memberSpawnStatuses.get('alice')).toBe(existingEntry);
|
||||||
|
});
|
||||||
|
|
||||||
});
|
});
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue