agent-ecosystem/src/main/services/team/AutoResumeService.ts

251 lines
7.8 KiB
TypeScript

import { createLogger } from '@shared/utils/logger';
import { parseRateLimitResetTime } from '@shared/utils/rateLimitDetector';
import { ConfigManager } from '../infrastructure/ConfigManager';
import type { TeamProvisioningService } from './TeamProvisioningService';
const logger = createLogger('Service:AutoResume');
const AUTO_RESUME_BUFFER_MS = 30 * 1000;
const AUTO_RESUME_MAX_DELAY_MS = 12 * 60 * 60 * 1000;
const AUTO_RESUME_HISTORY_FRESH_MS = 5 * 1000;
const AUTO_RESUME_MESSAGE =
'Your rate limit has reset. Please resume the work you were doing before the limit was hit.';
interface PendingAutoResumeEntry {
timer: NodeJS.Timeout;
fireAtMs: number;
sourceMessageAtMs: number;
sourceRunId: string | null;
}
export type RateLimitAutoResumePlan =
| {
kind: 'scheduled';
resetTime: Date;
delayMs: number;
fireAtMs: number;
rawDelayMs: number;
}
| {
kind: 'manual';
reason: 'disabled' | 'not_resumable' | 'reset_unparseable' | 'stale' | 'too_far';
};
export function planRateLimitAutoResume(input: {
enabled: boolean;
canAutoResume: boolean;
messageText: string;
observedAt: Date;
messageTimestamp?: Date;
}): RateLimitAutoResumePlan {
if (!input.enabled) return { kind: 'manual', reason: 'disabled' };
if (!input.canAutoResume) return { kind: 'manual', reason: 'not_resumable' };
const observedAtMs = input.observedAt.getTime();
const messageTimestamp = input.messageTimestamp ?? input.observedAt;
const messageAtMs = Number.isFinite(messageTimestamp.getTime())
? messageTimestamp.getTime()
: observedAtMs;
const parseReferenceTime = Number.isFinite(messageTimestamp.getTime())
? messageTimestamp
: input.observedAt;
const resetTime = parseRateLimitResetTime(input.messageText, parseReferenceTime);
if (!resetTime) return { kind: 'manual', reason: 'reset_unparseable' };
const resetAtMs = resetTime.getTime();
const rawDelayMs = resetAtMs - observedAtMs;
const targetFireAtMs = resetAtMs + AUTO_RESUME_BUFFER_MS;
const messageAgeMs = Math.max(0, observedAtMs - messageAtMs);
if (targetFireAtMs <= observedAtMs && messageAgeMs > AUTO_RESUME_HISTORY_FRESH_MS) {
return { kind: 'manual', reason: 'stale' };
}
const delayMs = Math.max(0, targetFireAtMs - observedAtMs);
if (delayMs > AUTO_RESUME_MAX_DELAY_MS) {
return { kind: 'manual', reason: 'too_far' };
}
return {
kind: 'scheduled',
resetTime,
delayMs,
fireAtMs: observedAtMs + delayMs,
rawDelayMs,
};
}
type AutoResumeProvisioning = Pick<
TeamProvisioningService,
'getCurrentRunId' | 'isTeamAlive' | 'sendMessageToTeam'
>;
type AutoResumeConfigReader = Pick<ConfigManager, 'getConfig'>;
export class AutoResumeService {
private readonly pendingTimers = new Map<string, PendingAutoResumeEntry>();
constructor(
private readonly provisioningService: AutoResumeProvisioning,
private readonly configManager: AutoResumeConfigReader = ConfigManager.getInstance()
) {}
handleRateLimitMessage(
teamName: string,
messageText: string,
observedAt: Date = new Date(),
messageTimestamp: Date = observedAt
): void {
const observedAtMs = observedAt.getTime();
const messageAtMs = Number.isFinite(messageTimestamp.getTime())
? messageTimestamp.getTime()
: observedAtMs;
const existing = this.pendingTimers.get(teamName);
const sourceRunId = this.provisioningService.getCurrentRunId(teamName);
const cfg = this.configManager.getConfig();
if (existing && messageAtMs < existing.sourceMessageAtMs) {
logger.info(
`[auto-resume] Ignoring older rate-limit message for "${teamName}" because a newer timer is already pending`
);
return;
}
const plan = planRateLimitAutoResume({
enabled: cfg.notifications.autoResumeOnRateLimit,
canAutoResume: true,
messageText,
observedAt,
messageTimestamp,
});
if (plan.kind === 'manual') {
if (plan.reason === 'too_far' && existing) {
clearTimeout(existing.timer);
this.pendingTimers.delete(teamName);
}
if (plan.reason === 'too_far') {
logger.warn(`[auto-resume] Parsed reset time for "${teamName}" exceeds ceiling - skipping`);
return;
}
logger.info(
`[auto-resume] Rate limit detected for "${teamName}" but auto-resume is manual (${plan.reason})`
);
return;
}
if (plan.rawDelayMs < 0) {
logger.warn(
`[auto-resume] Parsed reset time for "${teamName}" is ${Math.round(-plan.rawDelayMs / 1000)}s in the past - using remaining buffered delay`
);
}
if (
existing?.fireAtMs === plan.fireAtMs &&
existing.sourceMessageAtMs === messageAtMs &&
existing.sourceRunId === sourceRunId
) {
return;
}
if (existing) {
clearTimeout(existing.timer);
this.pendingTimers.delete(teamName);
logger.info(
`[auto-resume] Rescheduling resume for "${teamName}" to ${plan.resetTime.toISOString()}`
);
} else {
logger.info(
`[auto-resume] Scheduling resume for "${teamName}" at ${plan.resetTime.toISOString()} (in ${Math.round(plan.delayMs / 1000)}s)`
);
}
const timer = setTimeout(() => {
this.pendingTimers.delete(teamName);
void this.fireResumeNudge(teamName, sourceRunId);
}, plan.delayMs);
this.pendingTimers.set(teamName, {
timer,
fireAtMs: plan.fireAtMs,
sourceMessageAtMs: messageAtMs,
sourceRunId,
});
}
cancelPendingAutoResume(teamName: string): void {
const pending = this.pendingTimers.get(teamName);
if (!pending) return;
clearTimeout(pending.timer);
this.pendingTimers.delete(teamName);
}
clearAllPendingAutoResume(): void {
for (const pending of this.pendingTimers.values()) {
clearTimeout(pending.timer);
}
this.pendingTimers.clear();
}
private async fireResumeNudge(teamName: string, sourceRunId: string | null): Promise<void> {
const current = this.configManager.getConfig();
if (!current.notifications.autoResumeOnRateLimit) {
logger.info(
`[auto-resume] Config flag was disabled while timer was pending - skipping nudge for "${teamName}"`
);
return;
}
try {
if (!this.provisioningService.isTeamAlive(teamName)) {
logger.info(
`[auto-resume] Team "${teamName}" is no longer alive at fire time - skipping resume nudge`
);
return;
}
const currentRunId = this.provisioningService.getCurrentRunId(teamName);
if (sourceRunId && currentRunId !== sourceRunId) {
logger.info(
`[auto-resume] Team "${teamName}" advanced from run "${sourceRunId}" to "${currentRunId ?? 'none'}" before fire time - skipping stale resume nudge`
);
return;
}
await this.provisioningService.sendMessageToTeam(teamName, AUTO_RESUME_MESSAGE);
logger.info(`[auto-resume] Sent resume nudge to "${teamName}"`);
} catch (error) {
logger.error(
`[auto-resume] Failed to send resume nudge to "${teamName}": ${
error instanceof Error ? error.message : String(error)
}`
);
}
}
}
let autoResumeService: AutoResumeService | null = null;
export function initializeAutoResumeService(
provisioningService: AutoResumeProvisioning
): AutoResumeService {
autoResumeService?.clearAllPendingAutoResume();
autoResumeService = new AutoResumeService(provisioningService);
return autoResumeService;
}
export function getAutoResumeService(): AutoResumeService {
if (!autoResumeService) {
throw new Error('AutoResumeService is not initialized');
}
return autoResumeService;
}
export function peekAutoResumeService(): AutoResumeService | null {
return autoResumeService;
}
export function clearAutoResumeService(): void {
autoResumeService?.clearAllPendingAutoResume();
autoResumeService = null;
}