fix(team-runtime): ignore stale launch advisories
This commit is contained in:
parent
61418cf2f2
commit
91ccd75ddf
3 changed files with 212 additions and 44 deletions
|
|
@ -70,6 +70,7 @@ import type {
|
||||||
KanbanColumnId,
|
KanbanColumnId,
|
||||||
KanbanState,
|
KanbanState,
|
||||||
MessagesPage,
|
MessagesPage,
|
||||||
|
PersistedTeamLaunchSnapshot,
|
||||||
ReplaceMembersRequest,
|
ReplaceMembersRequest,
|
||||||
SendMessageRequest,
|
SendMessageRequest,
|
||||||
SendMessageResult,
|
SendMessageResult,
|
||||||
|
|
@ -598,9 +599,12 @@ export class TeamDataService {
|
||||||
|
|
||||||
private async getMemberRuntimeAdvisoriesForSnapshot(
|
private async getMemberRuntimeAdvisoriesForSnapshot(
|
||||||
teamName: string,
|
teamName: string,
|
||||||
members: readonly Pick<TeamMemberSnapshot, 'name' | 'removedAt'>[]
|
members: readonly Pick<TeamMemberSnapshot, 'name' | 'removedAt'>[],
|
||||||
|
observedAfterMs: number | null = null
|
||||||
): Promise<Map<string, NonNullable<TeamMemberSnapshot['runtimeAdvisory']>>> {
|
): Promise<Map<string, NonNullable<TeamMemberSnapshot['runtimeAdvisory']>>> {
|
||||||
const request = this.memberRuntimeAdvisoryService.getMemberAdvisories(teamName, members);
|
const request = this.memberRuntimeAdvisoryService.getMemberAdvisories(teamName, members, {
|
||||||
|
observedAfterMs,
|
||||||
|
});
|
||||||
const timeoutToken = Symbol('member-runtime-advisory-timeout');
|
const timeoutToken = Symbol('member-runtime-advisory-timeout');
|
||||||
let timeoutHandle: ReturnType<typeof setTimeout> | null = null;
|
let timeoutHandle: ReturnType<typeof setTimeout> | null = null;
|
||||||
const timeout = new Promise<typeof timeoutToken>((resolve) => {
|
const timeout = new Promise<typeof timeoutToken>((resolve) => {
|
||||||
|
|
@ -628,6 +632,27 @@ export class TeamDataService {
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private getRuntimeAdvisoryObservedAfterMs(
|
||||||
|
launchSnapshot: PersistedTeamLaunchSnapshot | null
|
||||||
|
): number | null {
|
||||||
|
if (!launchSnapshot) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const candidates = [
|
||||||
|
launchSnapshot.updatedAt,
|
||||||
|
...Object.values(launchSnapshot.members).flatMap((member) => [
|
||||||
|
member.lastEvaluatedAt,
|
||||||
|
member.firstSpawnAcceptedAt,
|
||||||
|
member.lastHeartbeatAt,
|
||||||
|
]),
|
||||||
|
];
|
||||||
|
const validTimes = candidates
|
||||||
|
.map((value) => (typeof value === 'string' ? Date.parse(value) : Number.NaN))
|
||||||
|
.filter((value) => Number.isFinite(value) && value > 0);
|
||||||
|
return validTimes.length > 0 ? Math.min(...validTimes) : null;
|
||||||
|
}
|
||||||
|
|
||||||
private async synthesizeLeadMemberIfMissing(
|
private async synthesizeLeadMemberIfMissing(
|
||||||
teamName: string,
|
teamName: string,
|
||||||
config: TeamConfig,
|
config: TeamConfig,
|
||||||
|
|
@ -1494,7 +1519,11 @@ export class TeamDataService {
|
||||||
mark('resolveMembers');
|
mark('resolveMembers');
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const runtimeAdvisories = await this.getMemberRuntimeAdvisoriesForSnapshot(teamName, members);
|
const runtimeAdvisories = await this.getMemberRuntimeAdvisoriesForSnapshot(
|
||||||
|
teamName,
|
||||||
|
members,
|
||||||
|
this.getRuntimeAdvisoryObservedAfterMs(launchSnapshot)
|
||||||
|
);
|
||||||
for (const member of members) {
|
for (const member of members) {
|
||||||
const advisory = runtimeAdvisories.get(member.name);
|
const advisory = runtimeAdvisories.get(member.name);
|
||||||
if (advisory) {
|
if (advisory) {
|
||||||
|
|
|
||||||
|
|
@ -44,6 +44,10 @@ interface RuntimeAdvisoryLogsFinder {
|
||||||
): Promise<RuntimeAdvisoryLogFileRef[]>;
|
): Promise<RuntimeAdvisoryLogFileRef[]>;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
interface RuntimeAdvisoryLookupOptions {
|
||||||
|
observedAfterMs?: number | null;
|
||||||
|
}
|
||||||
|
|
||||||
const LOOKBACK_MS = 10 * 60 * 1000;
|
const LOOKBACK_MS = 10 * 60 * 1000;
|
||||||
const CACHE_TTL_MS = 30_000;
|
const CACHE_TTL_MS = 30_000;
|
||||||
const TAIL_BYTES = 64 * 1024;
|
const TAIL_BYTES = 64 * 1024;
|
||||||
|
|
@ -59,6 +63,7 @@ interface CachedRuntimeAdvisory {
|
||||||
|
|
||||||
interface CachedTeamBatchAdvisories {
|
interface CachedTeamBatchAdvisories {
|
||||||
membersSignature: string;
|
membersSignature: string;
|
||||||
|
observedAfterScopeKey: string;
|
||||||
value: Map<string, MemberRuntimeAdvisory>;
|
value: Map<string, MemberRuntimeAdvisory>;
|
||||||
expiresAt: number;
|
expiresAt: number;
|
||||||
}
|
}
|
||||||
|
|
@ -138,7 +143,8 @@ export class TeamMemberRuntimeAdvisoryService {
|
||||||
|
|
||||||
async getMemberAdvisories(
|
async getMemberAdvisories(
|
||||||
teamName: string,
|
teamName: string,
|
||||||
members: readonly Pick<ResolvedTeamMember, 'name' | 'removedAt'>[]
|
members: readonly Pick<ResolvedTeamMember, 'name' | 'removedAt'>[],
|
||||||
|
options?: RuntimeAdvisoryLookupOptions
|
||||||
): Promise<Map<string, MemberRuntimeAdvisory>> {
|
): Promise<Map<string, MemberRuntimeAdvisory>> {
|
||||||
const activeMembers = members.filter((member) => !member.removedAt);
|
const activeMembers = members.filter((member) => !member.removedAt);
|
||||||
if (activeMembers.length === 0) {
|
if (activeMembers.length === 0) {
|
||||||
|
|
@ -147,19 +153,32 @@ export class TeamMemberRuntimeAdvisoryService {
|
||||||
|
|
||||||
const teamKey = this.normalizeToken(teamName);
|
const teamKey = this.normalizeToken(teamName);
|
||||||
const membersSignature = this.buildMembersSignature(activeMembers);
|
const membersSignature = this.buildMembersSignature(activeMembers);
|
||||||
|
const observedAfterMs = this.normalizeObservedAfterMs(options?.observedAfterMs);
|
||||||
|
const scopeKey = this.buildObservedAfterScopeKey(observedAfterMs);
|
||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
const cachedBatch = this.teamBatchCacheByTeam.get(teamKey);
|
const cachedBatch = this.teamBatchCacheByTeam.get(teamKey);
|
||||||
if (cachedBatch?.membersSignature === membersSignature && cachedBatch.expiresAt > now) {
|
if (
|
||||||
|
cachedBatch?.membersSignature === membersSignature &&
|
||||||
|
cachedBatch.observedAfterScopeKey === scopeKey &&
|
||||||
|
cachedBatch.expiresAt > now
|
||||||
|
) {
|
||||||
return this.materializeBatchAdvisories(activeMembers, cachedBatch.value);
|
return this.materializeBatchAdvisories(activeMembers, cachedBatch.value);
|
||||||
}
|
}
|
||||||
|
|
||||||
const inFlightKey = `${teamKey}::${membersSignature}`;
|
const inFlightKey = `${teamKey}::${membersSignature}::${scopeKey}`;
|
||||||
const existingRequest = this.inFlightBatchRequests.get(inFlightKey);
|
const existingRequest = this.inFlightBatchRequests.get(inFlightKey);
|
||||||
if (existingRequest) {
|
if (existingRequest) {
|
||||||
return this.materializeBatchAdvisories(activeMembers, await existingRequest);
|
return this.materializeBatchAdvisories(activeMembers, await existingRequest);
|
||||||
}
|
}
|
||||||
|
|
||||||
const request = this.loadBatchAdvisories(teamName, teamKey, activeMembers, membersSignature);
|
const request = this.loadBatchAdvisories(
|
||||||
|
teamName,
|
||||||
|
teamKey,
|
||||||
|
activeMembers,
|
||||||
|
membersSignature,
|
||||||
|
observedAfterMs,
|
||||||
|
scopeKey
|
||||||
|
);
|
||||||
this.inFlightBatchRequests.set(inFlightKey, request);
|
this.inFlightBatchRequests.set(inFlightKey, request);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
|
@ -173,17 +192,19 @@ export class TeamMemberRuntimeAdvisoryService {
|
||||||
|
|
||||||
async getMemberAdvisory(
|
async getMemberAdvisory(
|
||||||
teamName: string,
|
teamName: string,
|
||||||
memberName: string
|
memberName: string,
|
||||||
|
options?: RuntimeAdvisoryLookupOptions
|
||||||
): Promise<MemberRuntimeAdvisory | null> {
|
): Promise<MemberRuntimeAdvisory | null> {
|
||||||
const teamKey = this.normalizeToken(teamName);
|
const teamKey = this.normalizeToken(teamName);
|
||||||
const cacheKey = this.getMemberCacheKey(teamName, memberName);
|
const observedAfterMs = this.normalizeObservedAfterMs(options?.observedAfterMs);
|
||||||
|
const cacheKey = this.getMemberCacheKey(teamName, memberName, observedAfterMs);
|
||||||
const cached = this.memberCache.get(cacheKey);
|
const cached = this.memberCache.get(cacheKey);
|
||||||
if (cached && cached.expiresAt > Date.now()) {
|
if (cached && cached.expiresAt > Date.now()) {
|
||||||
return cached.value ? this.cloneAdvisory(cached.value) : null;
|
return cached.value ? this.cloneAdvisory(cached.value) : null;
|
||||||
}
|
}
|
||||||
|
|
||||||
const generationAtStart = this.cacheGenerationByTeam.get(teamKey) ?? 0;
|
const generationAtStart = this.cacheGenerationByTeam.get(teamKey) ?? 0;
|
||||||
const advisory = await this.findRecentMemberAdvisory(teamName, memberName);
|
const advisory = await this.findRecentMemberAdvisory(teamName, memberName, observedAfterMs);
|
||||||
if ((this.cacheGenerationByTeam.get(teamKey) ?? 0) === generationAtStart) {
|
if ((this.cacheGenerationByTeam.get(teamKey) ?? 0) === generationAtStart) {
|
||||||
this.memberCache.set(cacheKey, {
|
this.memberCache.set(cacheKey, {
|
||||||
value: advisory,
|
value: advisory,
|
||||||
|
|
@ -197,7 +218,9 @@ export class TeamMemberRuntimeAdvisoryService {
|
||||||
teamName: string,
|
teamName: string,
|
||||||
teamKey: string,
|
teamKey: string,
|
||||||
activeMembers: readonly Pick<ResolvedTeamMember, 'name'>[],
|
activeMembers: readonly Pick<ResolvedTeamMember, 'name'>[],
|
||||||
membersSignature: string
|
membersSignature: string,
|
||||||
|
observedAfterMs: number | null,
|
||||||
|
observedAfterScopeKey: string
|
||||||
): Promise<Map<string, MemberRuntimeAdvisory>> {
|
): Promise<Map<string, MemberRuntimeAdvisory>> {
|
||||||
const startedAt = performance.now();
|
const startedAt = performance.now();
|
||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
|
|
@ -209,7 +232,7 @@ export class TeamMemberRuntimeAdvisoryService {
|
||||||
|
|
||||||
for (const member of activeMembers) {
|
for (const member of activeMembers) {
|
||||||
const normalizedMemberName = this.normalizeToken(member.name);
|
const normalizedMemberName = this.normalizeToken(member.name);
|
||||||
const cacheKey = `${teamKey}::${normalizedMemberName}`;
|
const cacheKey = this.getMemberCacheKey(teamName, member.name, observedAfterMs);
|
||||||
const cached = this.memberCache.get(cacheKey);
|
const cached = this.memberCache.get(cacheKey);
|
||||||
if (cached && cached.expiresAt > now) {
|
if (cached && cached.expiresAt > now) {
|
||||||
memberCacheHits += 1;
|
memberCacheHits += 1;
|
||||||
|
|
@ -224,14 +247,18 @@ export class TeamMemberRuntimeAdvisoryService {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (membersToFetch.length > 0) {
|
if (membersToFetch.length > 0) {
|
||||||
const fetched = await this.findRecentMemberAdvisories(teamName, membersToFetch);
|
const fetched = await this.findRecentMemberAdvisories(
|
||||||
|
teamName,
|
||||||
|
membersToFetch,
|
||||||
|
observedAfterMs
|
||||||
|
);
|
||||||
const fetchedAt = Date.now();
|
const fetchedAt = Date.now();
|
||||||
const cacheStillCurrent =
|
const cacheStillCurrent =
|
||||||
(this.cacheGenerationByTeam.get(teamKey) ?? 0) === generationAtStart;
|
(this.cacheGenerationByTeam.get(teamKey) ?? 0) === generationAtStart;
|
||||||
for (const [memberName, advisory] of fetched) {
|
for (const [memberName, advisory] of fetched) {
|
||||||
const normalizedMemberName = this.normalizeToken(memberName);
|
const normalizedMemberName = this.normalizeToken(memberName);
|
||||||
if (cacheStillCurrent) {
|
if (cacheStillCurrent) {
|
||||||
this.memberCache.set(`${teamKey}::${normalizedMemberName}`, {
|
this.memberCache.set(this.getMemberCacheKey(teamName, memberName, observedAfterMs), {
|
||||||
value: advisory,
|
value: advisory,
|
||||||
expiresAt: fetchedAt + CACHE_TTL_MS,
|
expiresAt: fetchedAt + CACHE_TTL_MS,
|
||||||
});
|
});
|
||||||
|
|
@ -245,6 +272,7 @@ export class TeamMemberRuntimeAdvisoryService {
|
||||||
if ((this.cacheGenerationByTeam.get(teamKey) ?? 0) === generationAtStart) {
|
if ((this.cacheGenerationByTeam.get(teamKey) ?? 0) === generationAtStart) {
|
||||||
this.teamBatchCacheByTeam.set(teamKey, {
|
this.teamBatchCacheByTeam.set(teamKey, {
|
||||||
membersSignature,
|
membersSignature,
|
||||||
|
observedAfterScopeKey,
|
||||||
value: this.cloneNormalizedAdvisories(result),
|
value: this.cloneNormalizedAdvisories(result),
|
||||||
expiresAt: Date.now() + CACHE_TTL_MS,
|
expiresAt: Date.now() + CACHE_TTL_MS,
|
||||||
});
|
});
|
||||||
|
|
@ -260,8 +288,24 @@ export class TeamMemberRuntimeAdvisoryService {
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
private getMemberCacheKey(teamName: string, memberName: string): string {
|
private getMemberCacheKey(
|
||||||
return `${this.normalizeToken(teamName)}::${this.normalizeToken(memberName)}`;
|
teamName: string,
|
||||||
|
memberName: string,
|
||||||
|
observedAfterMs?: number | null
|
||||||
|
): string {
|
||||||
|
return `${this.normalizeToken(teamName)}::${this.normalizeToken(memberName)}::${this.buildObservedAfterScopeKey(
|
||||||
|
observedAfterMs
|
||||||
|
)}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
private normalizeObservedAfterMs(value: number | null | undefined): number | null {
|
||||||
|
return typeof value === 'number' && Number.isFinite(value) && value > 0
|
||||||
|
? Math.floor(value)
|
||||||
|
: null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private buildObservedAfterScopeKey(observedAfterMs: number | null | undefined): string {
|
||||||
|
return observedAfterMs == null ? 'recent' : `after:${observedAfterMs}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
private buildMembersSignature(members: readonly Pick<ResolvedTeamMember, 'name'>[]): string {
|
private buildMembersSignature(members: readonly Pick<ResolvedTeamMember, 'name'>[]): string {
|
||||||
|
|
@ -302,9 +346,14 @@ export class TeamMemberRuntimeAdvisoryService {
|
||||||
|
|
||||||
private async findRecentMemberAdvisory(
|
private async findRecentMemberAdvisory(
|
||||||
teamName: string,
|
teamName: string,
|
||||||
memberName: string
|
memberName: string,
|
||||||
|
observedAfterMs: number | null
|
||||||
): Promise<MemberRuntimeAdvisory | null> {
|
): Promise<MemberRuntimeAdvisory | null> {
|
||||||
const openCodeAdvisory = await this.findRecentOpenCodeDeliveryAdvisory(teamName, memberName);
|
const openCodeAdvisory = await this.findRecentOpenCodeDeliveryAdvisory(
|
||||||
|
teamName,
|
||||||
|
memberName,
|
||||||
|
observedAfterMs
|
||||||
|
);
|
||||||
if (openCodeAdvisory) {
|
if (openCodeAdvisory) {
|
||||||
return openCodeAdvisory;
|
return openCodeAdvisory;
|
||||||
}
|
}
|
||||||
|
|
@ -312,20 +361,23 @@ export class TeamMemberRuntimeAdvisoryService {
|
||||||
const summaries = await this.logsFinder.findMemberLogs(
|
const summaries = await this.logsFinder.findMemberLogs(
|
||||||
teamName,
|
teamName,
|
||||||
memberName,
|
memberName,
|
||||||
Date.now() - LOOKBACK_MS
|
Math.max(Date.now() - LOOKBACK_MS, observedAfterMs ?? 0)
|
||||||
);
|
);
|
||||||
return this.findRecentMemberAdvisoryInFiles(
|
return this.findRecentMemberAdvisoryInFiles(
|
||||||
summaries.flatMap((summary) => summary.filePath ?? [])
|
summaries.flatMap((summary) => summary.filePath ?? []),
|
||||||
|
observedAfterMs
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
private async findRecentMemberAdvisories(
|
private async findRecentMemberAdvisories(
|
||||||
teamName: string,
|
teamName: string,
|
||||||
memberNames: readonly string[]
|
memberNames: readonly string[],
|
||||||
|
observedAfterMs: number | null
|
||||||
): Promise<readonly (readonly [string, MemberRuntimeAdvisory | null])[]> {
|
): Promise<readonly (readonly [string, MemberRuntimeAdvisory | null])[]> {
|
||||||
const openCodeAdvisories = await this.findRecentOpenCodeDeliveryAdvisories(
|
const openCodeAdvisories = await this.findRecentOpenCodeDeliveryAdvisories(
|
||||||
teamName,
|
teamName,
|
||||||
memberNames
|
memberNames,
|
||||||
|
observedAfterMs
|
||||||
);
|
);
|
||||||
const remainingMemberNames = memberNames.filter(
|
const remainingMemberNames = memberNames.filter(
|
||||||
(memberName) => !openCodeAdvisories.has(memberName)
|
(memberName) => !openCodeAdvisories.has(memberName)
|
||||||
|
|
@ -340,7 +392,8 @@ export class TeamMemberRuntimeAdvisoryService {
|
||||||
try {
|
try {
|
||||||
const logAdvisories = await this.findRecentMemberAdvisoriesFromBatchRefs(
|
const logAdvisories = await this.findRecentMemberAdvisoriesFromBatchRefs(
|
||||||
teamName,
|
teamName,
|
||||||
remainingMemberNames
|
remainingMemberNames,
|
||||||
|
observedAfterMs
|
||||||
);
|
);
|
||||||
const logMap = new Map(logAdvisories);
|
const logMap = new Map(logAdvisories);
|
||||||
return memberNames.map(
|
return memberNames.map(
|
||||||
|
|
@ -365,12 +418,13 @@ export class TeamMemberRuntimeAdvisoryService {
|
||||||
const summaries = await this.logsFinder.findMemberLogs(
|
const summaries = await this.logsFinder.findMemberLogs(
|
||||||
teamName,
|
teamName,
|
||||||
memberName,
|
memberName,
|
||||||
Date.now() - LOOKBACK_MS
|
Math.max(Date.now() - LOOKBACK_MS, observedAfterMs ?? 0)
|
||||||
);
|
);
|
||||||
return [
|
return [
|
||||||
memberName,
|
memberName,
|
||||||
await this.findRecentMemberAdvisoryInFiles(
|
await this.findRecentMemberAdvisoryInFiles(
|
||||||
summaries.flatMap((summary) => summary.filePath ?? [])
|
summaries.flatMap((summary) => summary.filePath ?? []),
|
||||||
|
observedAfterMs
|
||||||
),
|
),
|
||||||
] as const;
|
] as const;
|
||||||
}
|
}
|
||||||
|
|
@ -384,15 +438,21 @@ export class TeamMemberRuntimeAdvisoryService {
|
||||||
|
|
||||||
private async findRecentOpenCodeDeliveryAdvisory(
|
private async findRecentOpenCodeDeliveryAdvisory(
|
||||||
teamName: string,
|
teamName: string,
|
||||||
memberName: string
|
memberName: string,
|
||||||
|
observedAfterMs: number | null
|
||||||
): Promise<MemberRuntimeAdvisory | null> {
|
): Promise<MemberRuntimeAdvisory | null> {
|
||||||
const advisories = await this.findRecentOpenCodeDeliveryAdvisories(teamName, [memberName]);
|
const advisories = await this.findRecentOpenCodeDeliveryAdvisories(
|
||||||
|
teamName,
|
||||||
|
[memberName],
|
||||||
|
observedAfterMs
|
||||||
|
);
|
||||||
return advisories.get(memberName) ?? null;
|
return advisories.get(memberName) ?? null;
|
||||||
}
|
}
|
||||||
|
|
||||||
private async findRecentOpenCodeDeliveryAdvisories(
|
private async findRecentOpenCodeDeliveryAdvisories(
|
||||||
teamName: string,
|
teamName: string,
|
||||||
memberNames: readonly string[]
|
memberNames: readonly string[],
|
||||||
|
observedAfterMs: number | null
|
||||||
): Promise<Map<string, MemberRuntimeAdvisory>> {
|
): Promise<Map<string, MemberRuntimeAdvisory>> {
|
||||||
const activeMembersByKey = new Map<string, string>();
|
const activeMembersByKey = new Map<string, string>();
|
||||||
for (const memberName of memberNames) {
|
for (const memberName of memberNames) {
|
||||||
|
|
@ -438,7 +498,11 @@ export class TeamMemberRuntimeAdvisoryService {
|
||||||
|
|
||||||
const memberKeysWithRecentErrors = new Set<string>();
|
const memberKeysWithRecentErrors = new Set<string>();
|
||||||
for (const [memberKey, records] of recordsByMember) {
|
for (const [memberKey, records] of recordsByMember) {
|
||||||
if (records.some((record) => this.isOpenCodeDeliveryAdvisoryCandidate(record, now))) {
|
if (
|
||||||
|
records.some((record) =>
|
||||||
|
this.isOpenCodeDeliveryAdvisoryCandidate(record, now, observedAfterMs)
|
||||||
|
)
|
||||||
|
) {
|
||||||
memberKeysWithRecentErrors.add(memberKey);
|
memberKeysWithRecentErrors.add(memberKey);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -468,7 +532,13 @@ export class TeamMemberRuntimeAdvisoryService {
|
||||||
}
|
}
|
||||||
const originalName = activeMembersByKey.get(memberKey);
|
const originalName = activeMembersByKey.get(memberKey);
|
||||||
const advisory = originalName
|
const advisory = originalName
|
||||||
? this.buildOpenCodeDeliveryAdvisoryFromRecords(originalName, records, now, proofIndex)
|
? this.buildOpenCodeDeliveryAdvisoryFromRecords(
|
||||||
|
originalName,
|
||||||
|
records,
|
||||||
|
now,
|
||||||
|
proofIndex,
|
||||||
|
observedAfterMs
|
||||||
|
)
|
||||||
: null;
|
: null;
|
||||||
if (advisory && originalName) {
|
if (advisory && originalName) {
|
||||||
result.set(originalName, advisory);
|
result.set(originalName, advisory);
|
||||||
|
|
@ -489,7 +559,8 @@ export class TeamMemberRuntimeAdvisoryService {
|
||||||
memberName: string,
|
memberName: string,
|
||||||
records: readonly OpenCodePromptDeliveryLedgerRecord[],
|
records: readonly OpenCodePromptDeliveryLedgerRecord[],
|
||||||
now: number,
|
now: number,
|
||||||
proofIndex: OpenCodeRuntimeDeliveryProofIndex
|
proofIndex: OpenCodeRuntimeDeliveryProofIndex,
|
||||||
|
observedAfterMs: number | null
|
||||||
): MemberRuntimeAdvisory | null {
|
): MemberRuntimeAdvisory | null {
|
||||||
const ordered = records
|
const ordered = records
|
||||||
.slice()
|
.slice()
|
||||||
|
|
@ -499,7 +570,7 @@ export class TeamMemberRuntimeAdvisoryService {
|
||||||
getOpenCodeRuntimeDeliveryRecordTimeMs(left)
|
getOpenCodeRuntimeDeliveryRecordTimeMs(left)
|
||||||
);
|
);
|
||||||
const latestError = ordered.find((record) => {
|
const latestError = ordered.find((record) => {
|
||||||
return this.isOpenCodeDeliveryAdvisoryCandidate(record, now);
|
return this.isOpenCodeDeliveryAdvisoryCandidate(record, now, observedAfterMs);
|
||||||
});
|
});
|
||||||
if (!latestError) {
|
if (!latestError) {
|
||||||
return null;
|
return null;
|
||||||
|
|
@ -584,8 +655,13 @@ export class TeamMemberRuntimeAdvisoryService {
|
||||||
|
|
||||||
private isOpenCodeDeliveryAdvisoryCandidate(
|
private isOpenCodeDeliveryAdvisoryCandidate(
|
||||||
record: OpenCodePromptDeliveryLedgerRecord,
|
record: OpenCodePromptDeliveryLedgerRecord,
|
||||||
now: number
|
now: number,
|
||||||
|
observedAfterMs: number | null
|
||||||
): boolean {
|
): boolean {
|
||||||
|
const observedAt = getOpenCodeRuntimeDeliveryRecordTimeMs(record);
|
||||||
|
if (observedAfterMs != null && Number.isFinite(observedAt) && observedAt < observedAfterMs) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
if (!isPotentialOpenCodeRuntimeDeliveryError(record)) {
|
if (!isPotentialOpenCodeRuntimeDeliveryError(record)) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
@ -595,13 +671,13 @@ export class TeamMemberRuntimeAdvisoryService {
|
||||||
) {
|
) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
const observedAt = getOpenCodeRuntimeDeliveryRecordTimeMs(record);
|
|
||||||
return Number.isFinite(observedAt) && now - observedAt <= OPENCODE_DELIVERY_ERROR_LOOKBACK_MS;
|
return Number.isFinite(observedAt) && now - observedAt <= OPENCODE_DELIVERY_ERROR_LOOKBACK_MS;
|
||||||
}
|
}
|
||||||
|
|
||||||
private async findRecentMemberAdvisoriesFromBatchRefs(
|
private async findRecentMemberAdvisoriesFromBatchRefs(
|
||||||
teamName: string,
|
teamName: string,
|
||||||
memberNames: readonly string[]
|
memberNames: readonly string[],
|
||||||
|
observedAfterMs: number | null
|
||||||
): Promise<readonly (readonly [string, MemberRuntimeAdvisory | null])[]> {
|
): Promise<readonly (readonly [string, MemberRuntimeAdvisory | null])[]> {
|
||||||
const memberNamesByKey = new Map<string, string>();
|
const memberNamesByKey = new Map<string, string>();
|
||||||
for (const memberName of memberNames) {
|
for (const memberName of memberNames) {
|
||||||
|
|
@ -614,7 +690,7 @@ export class TeamMemberRuntimeAdvisoryService {
|
||||||
const refs = await this.logsFinder.findRecentMemberLogFileRefsByMember!(
|
const refs = await this.logsFinder.findRecentMemberLogFileRefsByMember!(
|
||||||
teamName,
|
teamName,
|
||||||
memberNames,
|
memberNames,
|
||||||
Date.now() - LOOKBACK_MS
|
Math.max(Date.now() - LOOKBACK_MS, observedAfterMs ?? 0)
|
||||||
);
|
);
|
||||||
const refsByMember = new Map<string, RuntimeAdvisoryLogFileRef[]>();
|
const refsByMember = new Map<string, RuntimeAdvisoryLogFileRef[]>();
|
||||||
for (const ref of refs) {
|
for (const ref of refs) {
|
||||||
|
|
@ -641,15 +717,19 @@ export class TeamMemberRuntimeAdvisoryService {
|
||||||
seenFilePaths.add(ref.filePath);
|
seenFilePaths.add(ref.filePath);
|
||||||
return [ref.filePath];
|
return [ref.filePath];
|
||||||
});
|
});
|
||||||
return [memberName, await this.findRecentMemberAdvisoryInFiles(filePaths)] as const;
|
return [
|
||||||
|
memberName,
|
||||||
|
await this.findRecentMemberAdvisoryInFiles(filePaths, observedAfterMs),
|
||||||
|
] as const;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
private async findRecentMemberAdvisoryInFiles(
|
private async findRecentMemberAdvisoryInFiles(
|
||||||
filePaths: readonly string[]
|
filePaths: readonly string[],
|
||||||
|
observedAfterMs: number | null
|
||||||
): Promise<MemberRuntimeAdvisory | null> {
|
): Promise<MemberRuntimeAdvisory | null> {
|
||||||
for (const filePath of filePaths) {
|
for (const filePath of filePaths) {
|
||||||
const advisory = await this.readRecentApiRetryAdvisory(filePath);
|
const advisory = await this.readRecentApiRetryAdvisory(filePath, observedAfterMs);
|
||||||
if (advisory) {
|
if (advisory) {
|
||||||
return advisory;
|
return advisory;
|
||||||
}
|
}
|
||||||
|
|
@ -658,7 +738,8 @@ export class TeamMemberRuntimeAdvisoryService {
|
||||||
}
|
}
|
||||||
|
|
||||||
private async readRecentApiRetryAdvisory(
|
private async readRecentApiRetryAdvisory(
|
||||||
filePath: string
|
filePath: string,
|
||||||
|
observedAfterMs: number | null = null
|
||||||
): Promise<MemberRuntimeAdvisory | null> {
|
): Promise<MemberRuntimeAdvisory | null> {
|
||||||
let handle: fs.FileHandle | null = null;
|
let handle: fs.FileHandle | null = null;
|
||||||
try {
|
try {
|
||||||
|
|
@ -682,7 +763,8 @@ export class TeamMemberRuntimeAdvisoryService {
|
||||||
for (let index = lines.length - 1; index >= 0; index -= 1) {
|
for (let index = lines.length - 1; index >= 0; index -= 1) {
|
||||||
const line = lines[index]?.trim() ?? '';
|
const line = lines[index]?.trim() ?? '';
|
||||||
const advisory =
|
const advisory =
|
||||||
this.extractApiRetryAdvisory(line, now) ?? this.extractApiErrorAdvisory(line, now);
|
this.extractApiRetryAdvisory(line, now, observedAfterMs) ??
|
||||||
|
this.extractApiErrorAdvisory(line, now, observedAfterMs);
|
||||||
if (advisory) {
|
if (advisory) {
|
||||||
return advisory;
|
return advisory;
|
||||||
}
|
}
|
||||||
|
|
@ -695,7 +777,11 @@ export class TeamMemberRuntimeAdvisoryService {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private extractApiRetryAdvisory(line: string, now = Date.now()): MemberRuntimeAdvisory | null {
|
private extractApiRetryAdvisory(
|
||||||
|
line: string,
|
||||||
|
now = Date.now(),
|
||||||
|
observedAfterMs: number | null = null
|
||||||
|
): MemberRuntimeAdvisory | null {
|
||||||
if (
|
if (
|
||||||
!line ||
|
!line ||
|
||||||
(!line.includes('"subtype":"api_error"') && !line.includes('"subtype": "api_error"'))
|
(!line.includes('"subtype":"api_error"') && !line.includes('"subtype": "api_error"'))
|
||||||
|
|
@ -735,6 +821,9 @@ export class TeamMemberRuntimeAdvisoryService {
|
||||||
if (!retryInMs || !Number.isFinite(observedAt)) {
|
if (!retryInMs || !Number.isFinite(observedAt)) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
if (observedAfterMs != null && observedAt < observedAfterMs) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
const retryUntil = observedAt + retryInMs;
|
const retryUntil = observedAt + retryInMs;
|
||||||
if (retryUntil <= now) {
|
if (retryUntil <= now) {
|
||||||
|
|
@ -760,7 +849,11 @@ export class TeamMemberRuntimeAdvisoryService {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private extractApiErrorAdvisory(line: string, now = Date.now()): MemberRuntimeAdvisory | null {
|
private extractApiErrorAdvisory(
|
||||||
|
line: string,
|
||||||
|
now = Date.now(),
|
||||||
|
observedAfterMs: number | null = null
|
||||||
|
): MemberRuntimeAdvisory | null {
|
||||||
if (
|
if (
|
||||||
!line ||
|
!line ||
|
||||||
(!line.includes('"isApiErrorMessage":true') &&
|
(!line.includes('"isApiErrorMessage":true') &&
|
||||||
|
|
@ -791,6 +884,9 @@ export class TeamMemberRuntimeAdvisoryService {
|
||||||
if (!Number.isFinite(observedAt) || observedAt < now - LOOKBACK_MS) {
|
if (!Number.isFinite(observedAt) || observedAt < now - LOOKBACK_MS) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
if (observedAfterMs != null && observedAt < observedAfterMs) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
const message = this.extractAssistantText(parsed.message?.content);
|
const message = this.extractAssistantText(parsed.message?.content);
|
||||||
if (!parsed.isApiErrorMessage && parsed.error !== 'authentication_failed') {
|
if (!parsed.isApiErrorMessage && parsed.error !== 'authentication_failed') {
|
||||||
|
|
|
||||||
|
|
@ -183,7 +183,11 @@ async function writeOpenCodeDeliveryFixture(input: {
|
||||||
const inboxDir = path.join(teamDir, 'inboxes');
|
const inboxDir = path.join(teamDir, 'inboxes');
|
||||||
await fs.mkdir(inboxDir, { recursive: true });
|
await fs.mkdir(inboxDir, { recursive: true });
|
||||||
for (const [inboxName, messages] of Object.entries(input.inboxes)) {
|
for (const [inboxName, messages] of Object.entries(input.inboxes)) {
|
||||||
await fs.writeFile(path.join(inboxDir, `${inboxName}.json`), JSON.stringify(messages), 'utf8');
|
await fs.writeFile(
|
||||||
|
path.join(inboxDir, `${inboxName}.json`),
|
||||||
|
JSON.stringify(messages),
|
||||||
|
'utf8'
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -367,6 +371,45 @@ describe('TeamMemberRuntimeAdvisoryService', () => {
|
||||||
expect(advisory?.message).toContain('auth_unavailable');
|
expect(advisory?.message).toContain('auth_unavailable');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('does not reuse API errors observed before the current launch floor', async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
vi.setSystemTime(new Date('2026-05-31T12:10:00.000Z'));
|
||||||
|
tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'claude-team-advisory-'));
|
||||||
|
const logPath = path.join(tmpDir, 'bob.jsonl');
|
||||||
|
await fs.writeFile(
|
||||||
|
logPath,
|
||||||
|
`${JSON.stringify({
|
||||||
|
type: 'assistant',
|
||||||
|
timestamp: '2026-05-31T12:01:00.000Z',
|
||||||
|
isApiErrorMessage: true,
|
||||||
|
error: 'unknown',
|
||||||
|
message: {
|
||||||
|
content: [
|
||||||
|
{
|
||||||
|
type: 'text',
|
||||||
|
text: 'API Error: 500 {"error":{"message":"old Codex API error","type":"server_error"}}',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
})}\n`,
|
||||||
|
'utf8'
|
||||||
|
);
|
||||||
|
|
||||||
|
const service = new TeamMemberRuntimeAdvisoryService({
|
||||||
|
findMemberLogs: vi.fn(async () => [{ filePath: logPath }]),
|
||||||
|
});
|
||||||
|
|
||||||
|
await expect(service.getMemberAdvisory('signal-ops', 'bob')).resolves.toMatchObject({
|
||||||
|
kind: 'api_error',
|
||||||
|
message: expect.stringContaining('old Codex API error'),
|
||||||
|
});
|
||||||
|
await expect(
|
||||||
|
service.getMemberAdvisory('signal-ops', 'bob', {
|
||||||
|
observedAfterMs: Date.parse('2026-05-31T12:05:00.000Z'),
|
||||||
|
})
|
||||||
|
).resolves.toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
it('treats Claude Code account access failures as auth errors', () => {
|
it('treats Claude Code account access failures as auth errors', () => {
|
||||||
const service = new TeamMemberRuntimeAdvisoryService({} as never);
|
const service = new TeamMemberRuntimeAdvisoryService({} as never);
|
||||||
const observedAt = '2099-04-09T10:00:00.000Z';
|
const observedAt = '2099-04-09T10:00:00.000Z';
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue