fix(team-runtime): ignore stale launch advisories

This commit is contained in:
777genius 2026-05-31 22:21:01 +03:00
parent 61418cf2f2
commit 91ccd75ddf
3 changed files with 212 additions and 44 deletions

View file

@ -70,6 +70,7 @@ import type {
KanbanColumnId, KanbanColumnId,
KanbanState, KanbanState,
MessagesPage, MessagesPage,
PersistedTeamLaunchSnapshot,
ReplaceMembersRequest, ReplaceMembersRequest,
SendMessageRequest, SendMessageRequest,
SendMessageResult, SendMessageResult,
@ -598,9 +599,12 @@ export class TeamDataService {
private async getMemberRuntimeAdvisoriesForSnapshot( private async getMemberRuntimeAdvisoriesForSnapshot(
teamName: string, teamName: string,
members: readonly Pick<TeamMemberSnapshot, 'name' | 'removedAt'>[] members: readonly Pick<TeamMemberSnapshot, 'name' | 'removedAt'>[],
observedAfterMs: number | null = null
): Promise<Map<string, NonNullable<TeamMemberSnapshot['runtimeAdvisory']>>> { ): Promise<Map<string, NonNullable<TeamMemberSnapshot['runtimeAdvisory']>>> {
const request = this.memberRuntimeAdvisoryService.getMemberAdvisories(teamName, members); const request = this.memberRuntimeAdvisoryService.getMemberAdvisories(teamName, members, {
observedAfterMs,
});
const timeoutToken = Symbol('member-runtime-advisory-timeout'); const timeoutToken = Symbol('member-runtime-advisory-timeout');
let timeoutHandle: ReturnType<typeof setTimeout> | null = null; let timeoutHandle: ReturnType<typeof setTimeout> | null = null;
const timeout = new Promise<typeof timeoutToken>((resolve) => { const timeout = new Promise<typeof timeoutToken>((resolve) => {
@ -628,6 +632,27 @@ export class TeamDataService {
return result; return result;
} }
private getRuntimeAdvisoryObservedAfterMs(
launchSnapshot: PersistedTeamLaunchSnapshot | null
): number | null {
if (!launchSnapshot) {
return null;
}
const candidates = [
launchSnapshot.updatedAt,
...Object.values(launchSnapshot.members).flatMap((member) => [
member.lastEvaluatedAt,
member.firstSpawnAcceptedAt,
member.lastHeartbeatAt,
]),
];
const validTimes = candidates
.map((value) => (typeof value === 'string' ? Date.parse(value) : Number.NaN))
.filter((value) => Number.isFinite(value) && value > 0);
return validTimes.length > 0 ? Math.min(...validTimes) : null;
}
private async synthesizeLeadMemberIfMissing( private async synthesizeLeadMemberIfMissing(
teamName: string, teamName: string,
config: TeamConfig, config: TeamConfig,
@ -1494,7 +1519,11 @@ export class TeamDataService {
mark('resolveMembers'); mark('resolveMembers');
try { try {
const runtimeAdvisories = await this.getMemberRuntimeAdvisoriesForSnapshot(teamName, members); const runtimeAdvisories = await this.getMemberRuntimeAdvisoriesForSnapshot(
teamName,
members,
this.getRuntimeAdvisoryObservedAfterMs(launchSnapshot)
);
for (const member of members) { for (const member of members) {
const advisory = runtimeAdvisories.get(member.name); const advisory = runtimeAdvisories.get(member.name);
if (advisory) { if (advisory) {

View file

@ -44,6 +44,10 @@ interface RuntimeAdvisoryLogsFinder {
): Promise<RuntimeAdvisoryLogFileRef[]>; ): Promise<RuntimeAdvisoryLogFileRef[]>;
} }
interface RuntimeAdvisoryLookupOptions {
observedAfterMs?: number | null;
}
const LOOKBACK_MS = 10 * 60 * 1000; const LOOKBACK_MS = 10 * 60 * 1000;
const CACHE_TTL_MS = 30_000; const CACHE_TTL_MS = 30_000;
const TAIL_BYTES = 64 * 1024; const TAIL_BYTES = 64 * 1024;
@ -59,6 +63,7 @@ interface CachedRuntimeAdvisory {
interface CachedTeamBatchAdvisories { interface CachedTeamBatchAdvisories {
membersSignature: string; membersSignature: string;
observedAfterScopeKey: string;
value: Map<string, MemberRuntimeAdvisory>; value: Map<string, MemberRuntimeAdvisory>;
expiresAt: number; expiresAt: number;
} }
@ -138,7 +143,8 @@ export class TeamMemberRuntimeAdvisoryService {
async getMemberAdvisories( async getMemberAdvisories(
teamName: string, teamName: string,
members: readonly Pick<ResolvedTeamMember, 'name' | 'removedAt'>[] members: readonly Pick<ResolvedTeamMember, 'name' | 'removedAt'>[],
options?: RuntimeAdvisoryLookupOptions
): Promise<Map<string, MemberRuntimeAdvisory>> { ): Promise<Map<string, MemberRuntimeAdvisory>> {
const activeMembers = members.filter((member) => !member.removedAt); const activeMembers = members.filter((member) => !member.removedAt);
if (activeMembers.length === 0) { if (activeMembers.length === 0) {
@ -147,19 +153,32 @@ export class TeamMemberRuntimeAdvisoryService {
const teamKey = this.normalizeToken(teamName); const teamKey = this.normalizeToken(teamName);
const membersSignature = this.buildMembersSignature(activeMembers); const membersSignature = this.buildMembersSignature(activeMembers);
const observedAfterMs = this.normalizeObservedAfterMs(options?.observedAfterMs);
const scopeKey = this.buildObservedAfterScopeKey(observedAfterMs);
const now = Date.now(); const now = Date.now();
const cachedBatch = this.teamBatchCacheByTeam.get(teamKey); const cachedBatch = this.teamBatchCacheByTeam.get(teamKey);
if (cachedBatch?.membersSignature === membersSignature && cachedBatch.expiresAt > now) { if (
cachedBatch?.membersSignature === membersSignature &&
cachedBatch.observedAfterScopeKey === scopeKey &&
cachedBatch.expiresAt > now
) {
return this.materializeBatchAdvisories(activeMembers, cachedBatch.value); return this.materializeBatchAdvisories(activeMembers, cachedBatch.value);
} }
const inFlightKey = `${teamKey}::${membersSignature}`; const inFlightKey = `${teamKey}::${membersSignature}::${scopeKey}`;
const existingRequest = this.inFlightBatchRequests.get(inFlightKey); const existingRequest = this.inFlightBatchRequests.get(inFlightKey);
if (existingRequest) { if (existingRequest) {
return this.materializeBatchAdvisories(activeMembers, await existingRequest); return this.materializeBatchAdvisories(activeMembers, await existingRequest);
} }
const request = this.loadBatchAdvisories(teamName, teamKey, activeMembers, membersSignature); const request = this.loadBatchAdvisories(
teamName,
teamKey,
activeMembers,
membersSignature,
observedAfterMs,
scopeKey
);
this.inFlightBatchRequests.set(inFlightKey, request); this.inFlightBatchRequests.set(inFlightKey, request);
try { try {
@ -173,17 +192,19 @@ export class TeamMemberRuntimeAdvisoryService {
async getMemberAdvisory( async getMemberAdvisory(
teamName: string, teamName: string,
memberName: string memberName: string,
options?: RuntimeAdvisoryLookupOptions
): Promise<MemberRuntimeAdvisory | null> { ): Promise<MemberRuntimeAdvisory | null> {
const teamKey = this.normalizeToken(teamName); const teamKey = this.normalizeToken(teamName);
const cacheKey = this.getMemberCacheKey(teamName, memberName); const observedAfterMs = this.normalizeObservedAfterMs(options?.observedAfterMs);
const cacheKey = this.getMemberCacheKey(teamName, memberName, observedAfterMs);
const cached = this.memberCache.get(cacheKey); const cached = this.memberCache.get(cacheKey);
if (cached && cached.expiresAt > Date.now()) { if (cached && cached.expiresAt > Date.now()) {
return cached.value ? this.cloneAdvisory(cached.value) : null; return cached.value ? this.cloneAdvisory(cached.value) : null;
} }
const generationAtStart = this.cacheGenerationByTeam.get(teamKey) ?? 0; const generationAtStart = this.cacheGenerationByTeam.get(teamKey) ?? 0;
const advisory = await this.findRecentMemberAdvisory(teamName, memberName); const advisory = await this.findRecentMemberAdvisory(teamName, memberName, observedAfterMs);
if ((this.cacheGenerationByTeam.get(teamKey) ?? 0) === generationAtStart) { if ((this.cacheGenerationByTeam.get(teamKey) ?? 0) === generationAtStart) {
this.memberCache.set(cacheKey, { this.memberCache.set(cacheKey, {
value: advisory, value: advisory,
@ -197,7 +218,9 @@ export class TeamMemberRuntimeAdvisoryService {
teamName: string, teamName: string,
teamKey: string, teamKey: string,
activeMembers: readonly Pick<ResolvedTeamMember, 'name'>[], activeMembers: readonly Pick<ResolvedTeamMember, 'name'>[],
membersSignature: string membersSignature: string,
observedAfterMs: number | null,
observedAfterScopeKey: string
): Promise<Map<string, MemberRuntimeAdvisory>> { ): Promise<Map<string, MemberRuntimeAdvisory>> {
const startedAt = performance.now(); const startedAt = performance.now();
const now = Date.now(); const now = Date.now();
@ -209,7 +232,7 @@ export class TeamMemberRuntimeAdvisoryService {
for (const member of activeMembers) { for (const member of activeMembers) {
const normalizedMemberName = this.normalizeToken(member.name); const normalizedMemberName = this.normalizeToken(member.name);
const cacheKey = `${teamKey}::${normalizedMemberName}`; const cacheKey = this.getMemberCacheKey(teamName, member.name, observedAfterMs);
const cached = this.memberCache.get(cacheKey); const cached = this.memberCache.get(cacheKey);
if (cached && cached.expiresAt > now) { if (cached && cached.expiresAt > now) {
memberCacheHits += 1; memberCacheHits += 1;
@ -224,14 +247,18 @@ export class TeamMemberRuntimeAdvisoryService {
} }
if (membersToFetch.length > 0) { if (membersToFetch.length > 0) {
const fetched = await this.findRecentMemberAdvisories(teamName, membersToFetch); const fetched = await this.findRecentMemberAdvisories(
teamName,
membersToFetch,
observedAfterMs
);
const fetchedAt = Date.now(); const fetchedAt = Date.now();
const cacheStillCurrent = const cacheStillCurrent =
(this.cacheGenerationByTeam.get(teamKey) ?? 0) === generationAtStart; (this.cacheGenerationByTeam.get(teamKey) ?? 0) === generationAtStart;
for (const [memberName, advisory] of fetched) { for (const [memberName, advisory] of fetched) {
const normalizedMemberName = this.normalizeToken(memberName); const normalizedMemberName = this.normalizeToken(memberName);
if (cacheStillCurrent) { if (cacheStillCurrent) {
this.memberCache.set(`${teamKey}::${normalizedMemberName}`, { this.memberCache.set(this.getMemberCacheKey(teamName, memberName, observedAfterMs), {
value: advisory, value: advisory,
expiresAt: fetchedAt + CACHE_TTL_MS, expiresAt: fetchedAt + CACHE_TTL_MS,
}); });
@ -245,6 +272,7 @@ export class TeamMemberRuntimeAdvisoryService {
if ((this.cacheGenerationByTeam.get(teamKey) ?? 0) === generationAtStart) { if ((this.cacheGenerationByTeam.get(teamKey) ?? 0) === generationAtStart) {
this.teamBatchCacheByTeam.set(teamKey, { this.teamBatchCacheByTeam.set(teamKey, {
membersSignature, membersSignature,
observedAfterScopeKey,
value: this.cloneNormalizedAdvisories(result), value: this.cloneNormalizedAdvisories(result),
expiresAt: Date.now() + CACHE_TTL_MS, expiresAt: Date.now() + CACHE_TTL_MS,
}); });
@ -260,8 +288,24 @@ export class TeamMemberRuntimeAdvisoryService {
return result; return result;
} }
private getMemberCacheKey(teamName: string, memberName: string): string { private getMemberCacheKey(
return `${this.normalizeToken(teamName)}::${this.normalizeToken(memberName)}`; teamName: string,
memberName: string,
observedAfterMs?: number | null
): string {
return `${this.normalizeToken(teamName)}::${this.normalizeToken(memberName)}::${this.buildObservedAfterScopeKey(
observedAfterMs
)}`;
}
private normalizeObservedAfterMs(value: number | null | undefined): number | null {
return typeof value === 'number' && Number.isFinite(value) && value > 0
? Math.floor(value)
: null;
}
private buildObservedAfterScopeKey(observedAfterMs: number | null | undefined): string {
return observedAfterMs == null ? 'recent' : `after:${observedAfterMs}`;
} }
private buildMembersSignature(members: readonly Pick<ResolvedTeamMember, 'name'>[]): string { private buildMembersSignature(members: readonly Pick<ResolvedTeamMember, 'name'>[]): string {
@ -302,9 +346,14 @@ export class TeamMemberRuntimeAdvisoryService {
private async findRecentMemberAdvisory( private async findRecentMemberAdvisory(
teamName: string, teamName: string,
memberName: string memberName: string,
observedAfterMs: number | null
): Promise<MemberRuntimeAdvisory | null> { ): Promise<MemberRuntimeAdvisory | null> {
const openCodeAdvisory = await this.findRecentOpenCodeDeliveryAdvisory(teamName, memberName); const openCodeAdvisory = await this.findRecentOpenCodeDeliveryAdvisory(
teamName,
memberName,
observedAfterMs
);
if (openCodeAdvisory) { if (openCodeAdvisory) {
return openCodeAdvisory; return openCodeAdvisory;
} }
@ -312,20 +361,23 @@ export class TeamMemberRuntimeAdvisoryService {
const summaries = await this.logsFinder.findMemberLogs( const summaries = await this.logsFinder.findMemberLogs(
teamName, teamName,
memberName, memberName,
Date.now() - LOOKBACK_MS Math.max(Date.now() - LOOKBACK_MS, observedAfterMs ?? 0)
); );
return this.findRecentMemberAdvisoryInFiles( return this.findRecentMemberAdvisoryInFiles(
summaries.flatMap((summary) => summary.filePath ?? []) summaries.flatMap((summary) => summary.filePath ?? []),
observedAfterMs
); );
} }
private async findRecentMemberAdvisories( private async findRecentMemberAdvisories(
teamName: string, teamName: string,
memberNames: readonly string[] memberNames: readonly string[],
observedAfterMs: number | null
): Promise<readonly (readonly [string, MemberRuntimeAdvisory | null])[]> { ): Promise<readonly (readonly [string, MemberRuntimeAdvisory | null])[]> {
const openCodeAdvisories = await this.findRecentOpenCodeDeliveryAdvisories( const openCodeAdvisories = await this.findRecentOpenCodeDeliveryAdvisories(
teamName, teamName,
memberNames memberNames,
observedAfterMs
); );
const remainingMemberNames = memberNames.filter( const remainingMemberNames = memberNames.filter(
(memberName) => !openCodeAdvisories.has(memberName) (memberName) => !openCodeAdvisories.has(memberName)
@ -340,7 +392,8 @@ export class TeamMemberRuntimeAdvisoryService {
try { try {
const logAdvisories = await this.findRecentMemberAdvisoriesFromBatchRefs( const logAdvisories = await this.findRecentMemberAdvisoriesFromBatchRefs(
teamName, teamName,
remainingMemberNames remainingMemberNames,
observedAfterMs
); );
const logMap = new Map(logAdvisories); const logMap = new Map(logAdvisories);
return memberNames.map( return memberNames.map(
@ -365,12 +418,13 @@ export class TeamMemberRuntimeAdvisoryService {
const summaries = await this.logsFinder.findMemberLogs( const summaries = await this.logsFinder.findMemberLogs(
teamName, teamName,
memberName, memberName,
Date.now() - LOOKBACK_MS Math.max(Date.now() - LOOKBACK_MS, observedAfterMs ?? 0)
); );
return [ return [
memberName, memberName,
await this.findRecentMemberAdvisoryInFiles( await this.findRecentMemberAdvisoryInFiles(
summaries.flatMap((summary) => summary.filePath ?? []) summaries.flatMap((summary) => summary.filePath ?? []),
observedAfterMs
), ),
] as const; ] as const;
} }
@ -384,15 +438,21 @@ export class TeamMemberRuntimeAdvisoryService {
private async findRecentOpenCodeDeliveryAdvisory( private async findRecentOpenCodeDeliveryAdvisory(
teamName: string, teamName: string,
memberName: string memberName: string,
observedAfterMs: number | null
): Promise<MemberRuntimeAdvisory | null> { ): Promise<MemberRuntimeAdvisory | null> {
const advisories = await this.findRecentOpenCodeDeliveryAdvisories(teamName, [memberName]); const advisories = await this.findRecentOpenCodeDeliveryAdvisories(
teamName,
[memberName],
observedAfterMs
);
return advisories.get(memberName) ?? null; return advisories.get(memberName) ?? null;
} }
private async findRecentOpenCodeDeliveryAdvisories( private async findRecentOpenCodeDeliveryAdvisories(
teamName: string, teamName: string,
memberNames: readonly string[] memberNames: readonly string[],
observedAfterMs: number | null
): Promise<Map<string, MemberRuntimeAdvisory>> { ): Promise<Map<string, MemberRuntimeAdvisory>> {
const activeMembersByKey = new Map<string, string>(); const activeMembersByKey = new Map<string, string>();
for (const memberName of memberNames) { for (const memberName of memberNames) {
@ -438,7 +498,11 @@ export class TeamMemberRuntimeAdvisoryService {
const memberKeysWithRecentErrors = new Set<string>(); const memberKeysWithRecentErrors = new Set<string>();
for (const [memberKey, records] of recordsByMember) { for (const [memberKey, records] of recordsByMember) {
if (records.some((record) => this.isOpenCodeDeliveryAdvisoryCandidate(record, now))) { if (
records.some((record) =>
this.isOpenCodeDeliveryAdvisoryCandidate(record, now, observedAfterMs)
)
) {
memberKeysWithRecentErrors.add(memberKey); memberKeysWithRecentErrors.add(memberKey);
} }
} }
@ -468,7 +532,13 @@ export class TeamMemberRuntimeAdvisoryService {
} }
const originalName = activeMembersByKey.get(memberKey); const originalName = activeMembersByKey.get(memberKey);
const advisory = originalName const advisory = originalName
? this.buildOpenCodeDeliveryAdvisoryFromRecords(originalName, records, now, proofIndex) ? this.buildOpenCodeDeliveryAdvisoryFromRecords(
originalName,
records,
now,
proofIndex,
observedAfterMs
)
: null; : null;
if (advisory && originalName) { if (advisory && originalName) {
result.set(originalName, advisory); result.set(originalName, advisory);
@ -489,7 +559,8 @@ export class TeamMemberRuntimeAdvisoryService {
memberName: string, memberName: string,
records: readonly OpenCodePromptDeliveryLedgerRecord[], records: readonly OpenCodePromptDeliveryLedgerRecord[],
now: number, now: number,
proofIndex: OpenCodeRuntimeDeliveryProofIndex proofIndex: OpenCodeRuntimeDeliveryProofIndex,
observedAfterMs: number | null
): MemberRuntimeAdvisory | null { ): MemberRuntimeAdvisory | null {
const ordered = records const ordered = records
.slice() .slice()
@ -499,7 +570,7 @@ export class TeamMemberRuntimeAdvisoryService {
getOpenCodeRuntimeDeliveryRecordTimeMs(left) getOpenCodeRuntimeDeliveryRecordTimeMs(left)
); );
const latestError = ordered.find((record) => { const latestError = ordered.find((record) => {
return this.isOpenCodeDeliveryAdvisoryCandidate(record, now); return this.isOpenCodeDeliveryAdvisoryCandidate(record, now, observedAfterMs);
}); });
if (!latestError) { if (!latestError) {
return null; return null;
@ -584,8 +655,13 @@ export class TeamMemberRuntimeAdvisoryService {
private isOpenCodeDeliveryAdvisoryCandidate( private isOpenCodeDeliveryAdvisoryCandidate(
record: OpenCodePromptDeliveryLedgerRecord, record: OpenCodePromptDeliveryLedgerRecord,
now: number now: number,
observedAfterMs: number | null
): boolean { ): boolean {
const observedAt = getOpenCodeRuntimeDeliveryRecordTimeMs(record);
if (observedAfterMs != null && Number.isFinite(observedAt) && observedAt < observedAfterMs) {
return false;
}
if (!isPotentialOpenCodeRuntimeDeliveryError(record)) { if (!isPotentialOpenCodeRuntimeDeliveryError(record)) {
return false; return false;
} }
@ -595,13 +671,13 @@ export class TeamMemberRuntimeAdvisoryService {
) { ) {
return true; return true;
} }
const observedAt = getOpenCodeRuntimeDeliveryRecordTimeMs(record);
return Number.isFinite(observedAt) && now - observedAt <= OPENCODE_DELIVERY_ERROR_LOOKBACK_MS; return Number.isFinite(observedAt) && now - observedAt <= OPENCODE_DELIVERY_ERROR_LOOKBACK_MS;
} }
private async findRecentMemberAdvisoriesFromBatchRefs( private async findRecentMemberAdvisoriesFromBatchRefs(
teamName: string, teamName: string,
memberNames: readonly string[] memberNames: readonly string[],
observedAfterMs: number | null
): Promise<readonly (readonly [string, MemberRuntimeAdvisory | null])[]> { ): Promise<readonly (readonly [string, MemberRuntimeAdvisory | null])[]> {
const memberNamesByKey = new Map<string, string>(); const memberNamesByKey = new Map<string, string>();
for (const memberName of memberNames) { for (const memberName of memberNames) {
@ -614,7 +690,7 @@ export class TeamMemberRuntimeAdvisoryService {
const refs = await this.logsFinder.findRecentMemberLogFileRefsByMember!( const refs = await this.logsFinder.findRecentMemberLogFileRefsByMember!(
teamName, teamName,
memberNames, memberNames,
Date.now() - LOOKBACK_MS Math.max(Date.now() - LOOKBACK_MS, observedAfterMs ?? 0)
); );
const refsByMember = new Map<string, RuntimeAdvisoryLogFileRef[]>(); const refsByMember = new Map<string, RuntimeAdvisoryLogFileRef[]>();
for (const ref of refs) { for (const ref of refs) {
@ -641,15 +717,19 @@ export class TeamMemberRuntimeAdvisoryService {
seenFilePaths.add(ref.filePath); seenFilePaths.add(ref.filePath);
return [ref.filePath]; return [ref.filePath];
}); });
return [memberName, await this.findRecentMemberAdvisoryInFiles(filePaths)] as const; return [
memberName,
await this.findRecentMemberAdvisoryInFiles(filePaths, observedAfterMs),
] as const;
}); });
} }
private async findRecentMemberAdvisoryInFiles( private async findRecentMemberAdvisoryInFiles(
filePaths: readonly string[] filePaths: readonly string[],
observedAfterMs: number | null
): Promise<MemberRuntimeAdvisory | null> { ): Promise<MemberRuntimeAdvisory | null> {
for (const filePath of filePaths) { for (const filePath of filePaths) {
const advisory = await this.readRecentApiRetryAdvisory(filePath); const advisory = await this.readRecentApiRetryAdvisory(filePath, observedAfterMs);
if (advisory) { if (advisory) {
return advisory; return advisory;
} }
@ -658,7 +738,8 @@ export class TeamMemberRuntimeAdvisoryService {
} }
private async readRecentApiRetryAdvisory( private async readRecentApiRetryAdvisory(
filePath: string filePath: string,
observedAfterMs: number | null = null
): Promise<MemberRuntimeAdvisory | null> { ): Promise<MemberRuntimeAdvisory | null> {
let handle: fs.FileHandle | null = null; let handle: fs.FileHandle | null = null;
try { try {
@ -682,7 +763,8 @@ export class TeamMemberRuntimeAdvisoryService {
for (let index = lines.length - 1; index >= 0; index -= 1) { for (let index = lines.length - 1; index >= 0; index -= 1) {
const line = lines[index]?.trim() ?? ''; const line = lines[index]?.trim() ?? '';
const advisory = const advisory =
this.extractApiRetryAdvisory(line, now) ?? this.extractApiErrorAdvisory(line, now); this.extractApiRetryAdvisory(line, now, observedAfterMs) ??
this.extractApiErrorAdvisory(line, now, observedAfterMs);
if (advisory) { if (advisory) {
return advisory; return advisory;
} }
@ -695,7 +777,11 @@ export class TeamMemberRuntimeAdvisoryService {
} }
} }
private extractApiRetryAdvisory(line: string, now = Date.now()): MemberRuntimeAdvisory | null { private extractApiRetryAdvisory(
line: string,
now = Date.now(),
observedAfterMs: number | null = null
): MemberRuntimeAdvisory | null {
if ( if (
!line || !line ||
(!line.includes('"subtype":"api_error"') && !line.includes('"subtype": "api_error"')) (!line.includes('"subtype":"api_error"') && !line.includes('"subtype": "api_error"'))
@ -735,6 +821,9 @@ export class TeamMemberRuntimeAdvisoryService {
if (!retryInMs || !Number.isFinite(observedAt)) { if (!retryInMs || !Number.isFinite(observedAt)) {
return null; return null;
} }
if (observedAfterMs != null && observedAt < observedAfterMs) {
return null;
}
const retryUntil = observedAt + retryInMs; const retryUntil = observedAt + retryInMs;
if (retryUntil <= now) { if (retryUntil <= now) {
@ -760,7 +849,11 @@ export class TeamMemberRuntimeAdvisoryService {
} }
} }
private extractApiErrorAdvisory(line: string, now = Date.now()): MemberRuntimeAdvisory | null { private extractApiErrorAdvisory(
line: string,
now = Date.now(),
observedAfterMs: number | null = null
): MemberRuntimeAdvisory | null {
if ( if (
!line || !line ||
(!line.includes('"isApiErrorMessage":true') && (!line.includes('"isApiErrorMessage":true') &&
@ -791,6 +884,9 @@ export class TeamMemberRuntimeAdvisoryService {
if (!Number.isFinite(observedAt) || observedAt < now - LOOKBACK_MS) { if (!Number.isFinite(observedAt) || observedAt < now - LOOKBACK_MS) {
return null; return null;
} }
if (observedAfterMs != null && observedAt < observedAfterMs) {
return null;
}
const message = this.extractAssistantText(parsed.message?.content); const message = this.extractAssistantText(parsed.message?.content);
if (!parsed.isApiErrorMessage && parsed.error !== 'authentication_failed') { if (!parsed.isApiErrorMessage && parsed.error !== 'authentication_failed') {

View file

@ -183,7 +183,11 @@ async function writeOpenCodeDeliveryFixture(input: {
const inboxDir = path.join(teamDir, 'inboxes'); const inboxDir = path.join(teamDir, 'inboxes');
await fs.mkdir(inboxDir, { recursive: true }); await fs.mkdir(inboxDir, { recursive: true });
for (const [inboxName, messages] of Object.entries(input.inboxes)) { for (const [inboxName, messages] of Object.entries(input.inboxes)) {
await fs.writeFile(path.join(inboxDir, `${inboxName}.json`), JSON.stringify(messages), 'utf8'); await fs.writeFile(
path.join(inboxDir, `${inboxName}.json`),
JSON.stringify(messages),
'utf8'
);
} }
} }
@ -367,6 +371,45 @@ describe('TeamMemberRuntimeAdvisoryService', () => {
expect(advisory?.message).toContain('auth_unavailable'); expect(advisory?.message).toContain('auth_unavailable');
}); });
it('does not reuse API errors observed before the current launch floor', async () => {
vi.useFakeTimers();
vi.setSystemTime(new Date('2026-05-31T12:10:00.000Z'));
tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'claude-team-advisory-'));
const logPath = path.join(tmpDir, 'bob.jsonl');
await fs.writeFile(
logPath,
`${JSON.stringify({
type: 'assistant',
timestamp: '2026-05-31T12:01:00.000Z',
isApiErrorMessage: true,
error: 'unknown',
message: {
content: [
{
type: 'text',
text: 'API Error: 500 {"error":{"message":"old Codex API error","type":"server_error"}}',
},
],
},
})}\n`,
'utf8'
);
const service = new TeamMemberRuntimeAdvisoryService({
findMemberLogs: vi.fn(async () => [{ filePath: logPath }]),
});
await expect(service.getMemberAdvisory('signal-ops', 'bob')).resolves.toMatchObject({
kind: 'api_error',
message: expect.stringContaining('old Codex API error'),
});
await expect(
service.getMemberAdvisory('signal-ops', 'bob', {
observedAfterMs: Date.parse('2026-05-31T12:05:00.000Z'),
})
).resolves.toBeNull();
});
it('treats Claude Code account access failures as auth errors', () => { it('treats Claude Code account access failures as auth errors', () => {
const service = new TeamMemberRuntimeAdvisoryService({} as never); const service = new TeamMemberRuntimeAdvisoryService({} as never);
const observedAt = '2099-04-09T10:00:00.000Z'; const observedAt = '2099-04-09T10:00:00.000Z';