feat(team): auto-resume rate-limited teams when the limit resets
This commit is contained in:
parent
da9cb93e93
commit
a42ab3096f
21 changed files with 2581 additions and 78 deletions
|
|
@ -126,6 +126,8 @@ A local orchestration layer for AI agent teams across Claude and Codex.
|
|||
|
||||
- **Task creation with attachments** — send a message to the team lead with any attached images. The lead will automatically create a fully described task and attach your files directly to the task for complete context.
|
||||
|
||||
- **Auto-resume after rate limits** — when the lead hits a Claude rate limit and the reset time is known, the app can automatically nudge the lead to continue once the cooldown has passed
|
||||
|
||||
- **Deep session analysis** — detailed breakdown of what happened in each agent session: bash commands, reasoning, subprocesses
|
||||
|
||||
- **Smart task-to-log/changes matching** — automatically links session logs/changes to specific tasks
|
||||
|
|
|
|||
|
|
@ -100,6 +100,7 @@ import {
|
|||
type TeamReconcileTrigger,
|
||||
} from './services/team/TeamReconcileDrainScheduler';
|
||||
import { TeamSentMessagesStore } from './services/team/TeamSentMessagesStore';
|
||||
import { clearAutoResumeService } from './services/team/AutoResumeService';
|
||||
import { getAppIconPath } from './utils/appIcon';
|
||||
import { getProjectsBasePath, getTeamsBasePath, getTodosBasePath } from './utils/pathDecoder';
|
||||
import {
|
||||
|
|
@ -1070,6 +1071,11 @@ async function startHttpServer(
|
|||
function shutdownServices(): void {
|
||||
logger.info('Shutting down services...');
|
||||
|
||||
// Clear pending auto-resume timers before anything else — otherwise the
|
||||
// dangling setTimeout handles keep the event loop alive past shutdown and
|
||||
// may fire against a torn-down provisioning service.
|
||||
clearAutoResumeService();
|
||||
|
||||
// Kill all team CLI processes via SIGKILL BEFORE anything else.
|
||||
// This must happen before the OS closes stdin pipes (on app exit),
|
||||
// because stdin EOF triggers CLI's graceful shutdown which deletes team files.
|
||||
|
|
|
|||
|
|
@ -125,6 +125,7 @@ function validateNotificationsSection(
|
|||
'notifyOnCrossTeamMessage',
|
||||
'notifyOnTeamLaunched',
|
||||
'notifyOnToolApproval',
|
||||
'autoResumeOnRateLimit',
|
||||
'statusChangeOnlySolo',
|
||||
'statusChangeStatuses',
|
||||
'triggers',
|
||||
|
|
@ -219,6 +220,12 @@ function validateNotificationsSection(
|
|||
}
|
||||
result.notifyOnToolApproval = value;
|
||||
break;
|
||||
case 'autoResumeOnRateLimit':
|
||||
if (typeof value !== 'boolean') {
|
||||
return { valid: false, error: `notifications.${key} must be a boolean` };
|
||||
}
|
||||
result.autoResumeOnRateLimit = value;
|
||||
break;
|
||||
case 'statusChangeOnlySolo':
|
||||
if (typeof value !== 'boolean') {
|
||||
return { valid: false, error: `notifications.${key} must be a boolean` };
|
||||
|
|
|
|||
|
|
@ -99,6 +99,10 @@ import * as path from 'path';
|
|||
import { ConfigManager } from '../services/infrastructure/ConfigManager';
|
||||
import { NotificationManager } from '../services/infrastructure/NotificationManager';
|
||||
import { gitIdentityResolver } from '../services/parsing/GitIdentityResolver';
|
||||
import {
|
||||
getAutoResumeService,
|
||||
initializeAutoResumeService,
|
||||
} from '../services/team/AutoResumeService';
|
||||
import {
|
||||
buildActionModeAgentBlock,
|
||||
isAgentActionMode,
|
||||
|
|
@ -301,11 +305,25 @@ const SEEN_API_ERROR_KEYS_MAX = 500;
|
|||
* and NotificationManager dedupeKey (to prevent storage duplicates).
|
||||
*/
|
||||
function checkRateLimitMessages(
|
||||
messages: readonly { messageId?: string; from: string; text: string; timestamp: string }[],
|
||||
messages: readonly {
|
||||
messageId?: string;
|
||||
from: string;
|
||||
text: string;
|
||||
timestamp: string;
|
||||
to?: string;
|
||||
source?: string;
|
||||
leadSessionId?: string;
|
||||
}[],
|
||||
teamName: string,
|
||||
teamDisplayName: string,
|
||||
projectPath?: string
|
||||
projectPath?: string,
|
||||
teamIsAlive = true,
|
||||
currentLeadSessionId: string | null = null
|
||||
): void {
|
||||
const observedAt = new Date();
|
||||
const autoResumeEnabled =
|
||||
ConfigManager.getInstance().getConfig().notifications.autoResumeOnRateLimit;
|
||||
|
||||
for (const msg of messages) {
|
||||
if (msg.from === 'user') continue;
|
||||
if (!isRateLimitMessage(msg.text)) continue;
|
||||
|
|
@ -313,28 +331,55 @@ function checkRateLimitMessages(
|
|||
const rawKey = msg.messageId ?? `${msg.from}:${msg.timestamp}`;
|
||||
const dedupeKey = `rate-limit:${teamName}:${rawKey}`;
|
||||
|
||||
// In-memory guard: prevents resurrection after user deletes the notification
|
||||
if (seenRateLimitKeys.has(dedupeKey)) continue;
|
||||
seenRateLimitKeys.add(dedupeKey);
|
||||
// In-memory guard: prevents resurrection after user deletes the notification.
|
||||
if (!seenRateLimitKeys.has(dedupeKey)) {
|
||||
seenRateLimitKeys.add(dedupeKey);
|
||||
|
||||
// Evict oldest entries to prevent unbounded growth
|
||||
if (seenRateLimitKeys.size > SEEN_RATE_LIMIT_KEYS_MAX) {
|
||||
const first = seenRateLimitKeys.values().next().value;
|
||||
if (first) seenRateLimitKeys.delete(first);
|
||||
// Evict oldest entries to prevent unbounded growth
|
||||
if (seenRateLimitKeys.size > SEEN_RATE_LIMIT_KEYS_MAX) {
|
||||
const first = seenRateLimitKeys.values().next().value;
|
||||
if (first) seenRateLimitKeys.delete(first);
|
||||
}
|
||||
|
||||
void NotificationManager.getInstance()
|
||||
.addTeamNotification({
|
||||
teamEventType: 'rate_limit',
|
||||
teamName,
|
||||
teamDisplayName,
|
||||
from: msg.from,
|
||||
summary: `Rate limit: ${msg.from}`,
|
||||
body: msg.text.slice(0, 200),
|
||||
dedupeKey,
|
||||
projectPath,
|
||||
})
|
||||
.catch(() => undefined);
|
||||
}
|
||||
|
||||
void NotificationManager.getInstance()
|
||||
.addTeamNotification({
|
||||
teamEventType: 'rate_limit',
|
||||
// Only schedule auto-resume while a live team run currently exists.
|
||||
// Persisted history for an offline/stopped team may still contain the old
|
||||
// rate-limit message, but arming a new timer from that stale history would
|
||||
// resurrect the nudge into a later manual restart.
|
||||
const isLeadAutoResumeCandidate =
|
||||
!msg.to && (msg.source === 'lead_process' || msg.source === 'lead_session');
|
||||
|
||||
if (autoResumeEnabled && teamIsAlive && isLeadAutoResumeCandidate) {
|
||||
// Only let persisted lead_session history rebuild auto-resume when it
|
||||
// clearly belongs to the currently running lead session. Otherwise an old
|
||||
// rate-limit from a previous manual run can resurrect into a newer restart.
|
||||
if (msg.source === 'lead_session') {
|
||||
if (!currentLeadSessionId) continue;
|
||||
if (msg.leadSessionId !== currentLeadSessionId) continue;
|
||||
}
|
||||
|
||||
// Pass the original message timestamp so relative reset windows survive restarts
|
||||
// and old history does not rebuild a fresh auto-resume timer from "now".
|
||||
getAutoResumeService().handleRateLimitMessage(
|
||||
teamName,
|
||||
teamDisplayName,
|
||||
from: msg.from,
|
||||
summary: `Rate limit: ${msg.from}`,
|
||||
body: msg.text.slice(0, 200),
|
||||
dedupeKey,
|
||||
projectPath,
|
||||
})
|
||||
.catch(() => undefined);
|
||||
msg.text,
|
||||
observedAt,
|
||||
new Date(msg.timestamp)
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -436,6 +481,7 @@ export function initializeTeamHandlers(
|
|||
): void {
|
||||
teamDataService = service;
|
||||
teamProvisioningService = provisioningService;
|
||||
initializeAutoResumeService(provisioningService);
|
||||
teamMemberLogsFinder = logsFinder ?? null;
|
||||
memberStatsComputer = statsComputer ?? null;
|
||||
teamBackupService = backupService ?? null;
|
||||
|
|
@ -759,13 +805,21 @@ async function handleGetData(
|
|||
}
|
||||
const provisioning = getTeamProvisioningService();
|
||||
const isAlive = provisioning.isTeamAlive(tn);
|
||||
const currentLeadSessionId = provisioning.getCurrentLeadSessionId(tn);
|
||||
|
||||
const displayName = data.config.name || tn;
|
||||
const projectPath = data.config.projectPath;
|
||||
|
||||
const live = provisioning.getLiveLeadProcessMessages(tn);
|
||||
if (live.length === 0) {
|
||||
checkRateLimitMessages(data.messages, tn, displayName, projectPath);
|
||||
checkRateLimitMessages(
|
||||
data.messages,
|
||||
tn,
|
||||
displayName,
|
||||
projectPath,
|
||||
isAlive,
|
||||
currentLeadSessionId
|
||||
);
|
||||
checkApiErrorMessages(data.messages, tn, displayName, projectPath);
|
||||
return { success: true, data: { ...data, isAlive } };
|
||||
}
|
||||
|
|
@ -845,7 +899,7 @@ async function handleGetData(
|
|||
}
|
||||
merged.sort((a, b) => Date.parse(b.timestamp) - Date.parse(a.timestamp));
|
||||
|
||||
checkRateLimitMessages(merged, tn, displayName, projectPath);
|
||||
checkRateLimitMessages(merged, tn, displayName, projectPath, isAlive, currentLeadSessionId);
|
||||
checkApiErrorMessages(merged, tn, displayName, projectPath);
|
||||
return { success: true, data: { ...data, isAlive, messages: merged } };
|
||||
}
|
||||
|
|
@ -926,6 +980,7 @@ async function handleDeleteTeam(
|
|||
return { success: false, error: validated.error ?? 'Invalid teamName' };
|
||||
}
|
||||
return wrapTeamHandler('deleteTeam', async () => {
|
||||
getAutoResumeService().cancelPendingAutoResume(validated.value!);
|
||||
getTeamProvisioningService().stopTeam(validated.value!);
|
||||
await getTeamDataService().deleteTeam(validated.value!);
|
||||
});
|
||||
|
|
@ -951,6 +1006,7 @@ async function handlePermanentlyDeleteTeam(
|
|||
return { success: false, error: validated.error ?? 'Invalid teamName' };
|
||||
}
|
||||
return wrapTeamHandler('permanentlyDeleteTeam', async () => {
|
||||
getAutoResumeService().cancelPendingAutoResume(validated.value!);
|
||||
await getTeamDataService().permanentlyDeleteTeam(validated.value!);
|
||||
// Clean up app-owned data (attachments, task-attachments) that lives outside ~/.claude/
|
||||
const appData = getAppDataPath();
|
||||
|
|
@ -2733,6 +2789,7 @@ async function handleStopTeam(
|
|||
}
|
||||
return wrapTeamHandler('stop', async () => {
|
||||
addMainBreadcrumb('team', 'stop', { teamName: validated.value! });
|
||||
getAutoResumeService().cancelPendingAutoResume(validated.value!);
|
||||
getTeamProvisioningService().stopTeam(validated.value!);
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -62,6 +62,12 @@ export interface NotificationConfig {
|
|||
notifyOnTeamLaunched: boolean;
|
||||
/** Whether to show native OS notifications when a tool needs user approval */
|
||||
notifyOnToolApproval: boolean;
|
||||
/** Whether to automatically resume a rate-limited team when the limit resets.
|
||||
* When enabled, the app parses the reset time from Claude's rate-limit
|
||||
* message and schedules a nudge to the team lead once the limit expires.
|
||||
* Default is `false` — opt-in to avoid unexpected API usage after the reset.
|
||||
*/
|
||||
autoResumeOnRateLimit: boolean;
|
||||
/** Only notify on status changes in solo teams (no teammates) */
|
||||
statusChangeOnlySolo: boolean;
|
||||
/** Which target statuses to notify about (e.g. ['in_progress', 'completed']) */
|
||||
|
|
@ -306,6 +312,7 @@ const DEFAULT_CONFIG: AppConfig = {
|
|||
notifyOnCrossTeamMessage: true,
|
||||
notifyOnTeamLaunched: true,
|
||||
notifyOnToolApproval: true,
|
||||
autoResumeOnRateLimit: false,
|
||||
statusChangeOnlySolo: false,
|
||||
statusChangeStatuses: ['in_progress', 'completed'],
|
||||
triggers: DEFAULT_TRIGGERS,
|
||||
|
|
@ -502,8 +509,56 @@ export class ConfigManager {
|
|||
|
||||
return {
|
||||
notifications: {
|
||||
...DEFAULT_CONFIG.notifications,
|
||||
...loadedNotifications,
|
||||
enabled: loadedNotifications.enabled ?? DEFAULT_CONFIG.notifications.enabled,
|
||||
soundEnabled: loadedNotifications.soundEnabled ?? DEFAULT_CONFIG.notifications.soundEnabled,
|
||||
ignoredRegex: loadedNotifications.ignoredRegex ?? DEFAULT_CONFIG.notifications.ignoredRegex,
|
||||
ignoredRepositories:
|
||||
loadedNotifications.ignoredRepositories ??
|
||||
DEFAULT_CONFIG.notifications.ignoredRepositories,
|
||||
snoozedUntil:
|
||||
loadedNotifications.snoozedUntil ?? DEFAULT_CONFIG.notifications.snoozedUntil,
|
||||
snoozeMinutes:
|
||||
loadedNotifications.snoozeMinutes ?? DEFAULT_CONFIG.notifications.snoozeMinutes,
|
||||
includeSubagentErrors:
|
||||
loadedNotifications.includeSubagentErrors ??
|
||||
DEFAULT_CONFIG.notifications.includeSubagentErrors,
|
||||
notifyOnLeadInbox:
|
||||
loadedNotifications.notifyOnLeadInbox ?? DEFAULT_CONFIG.notifications.notifyOnLeadInbox,
|
||||
notifyOnUserInbox:
|
||||
loadedNotifications.notifyOnUserInbox ?? DEFAULT_CONFIG.notifications.notifyOnUserInbox,
|
||||
notifyOnClarifications:
|
||||
loadedNotifications.notifyOnClarifications ??
|
||||
DEFAULT_CONFIG.notifications.notifyOnClarifications,
|
||||
notifyOnStatusChange:
|
||||
loadedNotifications.notifyOnStatusChange ??
|
||||
DEFAULT_CONFIG.notifications.notifyOnStatusChange,
|
||||
notifyOnTaskComments:
|
||||
loadedNotifications.notifyOnTaskComments ??
|
||||
DEFAULT_CONFIG.notifications.notifyOnTaskComments,
|
||||
notifyOnTaskCreated:
|
||||
loadedNotifications.notifyOnTaskCreated ??
|
||||
DEFAULT_CONFIG.notifications.notifyOnTaskCreated,
|
||||
notifyOnAllTasksCompleted:
|
||||
loadedNotifications.notifyOnAllTasksCompleted ??
|
||||
DEFAULT_CONFIG.notifications.notifyOnAllTasksCompleted,
|
||||
notifyOnCrossTeamMessage:
|
||||
loadedNotifications.notifyOnCrossTeamMessage ??
|
||||
DEFAULT_CONFIG.notifications.notifyOnCrossTeamMessage,
|
||||
notifyOnTeamLaunched:
|
||||
loadedNotifications.notifyOnTeamLaunched ??
|
||||
DEFAULT_CONFIG.notifications.notifyOnTeamLaunched,
|
||||
notifyOnToolApproval:
|
||||
loadedNotifications.notifyOnToolApproval ??
|
||||
DEFAULT_CONFIG.notifications.notifyOnToolApproval,
|
||||
autoResumeOnRateLimit:
|
||||
loadedNotifications.autoResumeOnRateLimit ??
|
||||
DEFAULT_CONFIG.notifications.autoResumeOnRateLimit,
|
||||
statusChangeOnlySolo:
|
||||
loadedNotifications.statusChangeOnlySolo ??
|
||||
DEFAULT_CONFIG.notifications.statusChangeOnlySolo,
|
||||
statusChangeStatuses:
|
||||
loadedNotifications.statusChangeStatuses ??
|
||||
DEFAULT_CONFIG.notifications.statusChangeStatuses,
|
||||
triggers: mergedTriggers,
|
||||
},
|
||||
general: mergedGeneral,
|
||||
|
|
|
|||
209
src/main/services/team/AutoResumeService.ts
Normal file
209
src/main/services/team/AutoResumeService.ts
Normal file
|
|
@ -0,0 +1,209 @@
|
|||
import { createLogger } from '@shared/utils/logger';
|
||||
import { parseRateLimitResetTime } from '@shared/utils/rateLimitDetector';
|
||||
|
||||
import { ConfigManager } from '../infrastructure/ConfigManager';
|
||||
|
||||
import type { TeamProvisioningService } from './TeamProvisioningService';
|
||||
|
||||
const logger = createLogger('Service:AutoResume');
|
||||
|
||||
const AUTO_RESUME_BUFFER_MS = 30 * 1000;
|
||||
const AUTO_RESUME_MAX_DELAY_MS = 12 * 60 * 60 * 1000;
|
||||
const AUTO_RESUME_HISTORY_FRESH_MS = 5 * 1000;
|
||||
const AUTO_RESUME_MESSAGE =
|
||||
'Your rate limit has reset. Please resume the work you were doing before the limit was hit.';
|
||||
|
||||
interface PendingAutoResumeEntry {
|
||||
timer: NodeJS.Timeout;
|
||||
fireAtMs: number;
|
||||
sourceMessageAtMs: number;
|
||||
sourceRunId: string | null;
|
||||
}
|
||||
|
||||
type AutoResumeProvisioning = Pick<
|
||||
TeamProvisioningService,
|
||||
'getCurrentRunId' | 'isTeamAlive' | 'sendMessageToTeam'
|
||||
>;
|
||||
type AutoResumeConfigReader = Pick<ConfigManager, 'getConfig'>;
|
||||
|
||||
export class AutoResumeService {
|
||||
private readonly pendingTimers = new Map<string, PendingAutoResumeEntry>();
|
||||
|
||||
constructor(
|
||||
private readonly provisioningService: AutoResumeProvisioning,
|
||||
private readonly configManager: AutoResumeConfigReader = ConfigManager.getInstance()
|
||||
) {}
|
||||
|
||||
handleRateLimitMessage(
|
||||
teamName: string,
|
||||
messageText: string,
|
||||
observedAt: Date = new Date(),
|
||||
messageTimestamp: Date = observedAt
|
||||
): void {
|
||||
const cfg = this.configManager.getConfig();
|
||||
if (!cfg.notifications.autoResumeOnRateLimit) return;
|
||||
|
||||
const observedAtMs = observedAt.getTime();
|
||||
const messageAtMs = Number.isFinite(messageTimestamp.getTime())
|
||||
? messageTimestamp.getTime()
|
||||
: observedAtMs;
|
||||
const parseReferenceTime = Number.isFinite(messageTimestamp.getTime())
|
||||
? messageTimestamp
|
||||
: observedAt;
|
||||
|
||||
const resetTime = parseRateLimitResetTime(messageText, parseReferenceTime);
|
||||
if (!resetTime) {
|
||||
logger.info(
|
||||
`[auto-resume] Rate limit detected for "${teamName}" but reset time was not parseable - skipping auto-resume`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const resetAtMs = resetTime.getTime();
|
||||
const rawDelayMs = resetAtMs - observedAtMs;
|
||||
const targetFireAtMs = resetAtMs + AUTO_RESUME_BUFFER_MS;
|
||||
const messageAgeMs = Math.max(0, observedAtMs - messageAtMs);
|
||||
const existing = this.pendingTimers.get(teamName);
|
||||
const sourceRunId = this.provisioningService.getCurrentRunId(teamName);
|
||||
|
||||
if (existing && messageAtMs < existing.sourceMessageAtMs) {
|
||||
logger.info(
|
||||
`[auto-resume] Ignoring older rate-limit message for "${teamName}" because a newer timer is already pending`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (targetFireAtMs <= observedAtMs && messageAgeMs > AUTO_RESUME_HISTORY_FRESH_MS) {
|
||||
logger.info(
|
||||
`[auto-resume] Parsed reset time for "${teamName}" passed its buffered fire deadline ${Math.round((observedAtMs - targetFireAtMs) / 1000)}s ago - skipping stale history replay`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (rawDelayMs < 0) {
|
||||
logger.warn(
|
||||
`[auto-resume] Parsed reset time for "${teamName}" is ${Math.round(-rawDelayMs / 1000)}s in the past - using remaining buffered delay`
|
||||
);
|
||||
}
|
||||
|
||||
const delayMs = Math.max(0, targetFireAtMs - observedAtMs);
|
||||
const fireAtMs = observedAtMs + delayMs;
|
||||
|
||||
if (delayMs > AUTO_RESUME_MAX_DELAY_MS) {
|
||||
if (existing) {
|
||||
clearTimeout(existing.timer);
|
||||
this.pendingTimers.delete(teamName);
|
||||
}
|
||||
logger.warn(
|
||||
`[auto-resume] Parsed reset time for "${teamName}" is ${Math.round(delayMs / 60000)}m away - exceeds ceiling, skipping`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (
|
||||
existing?.fireAtMs === fireAtMs &&
|
||||
existing.sourceMessageAtMs === messageAtMs &&
|
||||
existing.sourceRunId === sourceRunId
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (existing) {
|
||||
clearTimeout(existing.timer);
|
||||
this.pendingTimers.delete(teamName);
|
||||
logger.info(
|
||||
`[auto-resume] Rescheduling resume for "${teamName}" to ${resetTime.toISOString()}`
|
||||
);
|
||||
} else {
|
||||
logger.info(
|
||||
`[auto-resume] Scheduling resume for "${teamName}" at ${resetTime.toISOString()} (in ${Math.round(delayMs / 1000)}s)`
|
||||
);
|
||||
}
|
||||
|
||||
const timer = setTimeout(() => {
|
||||
this.pendingTimers.delete(teamName);
|
||||
void this.fireResumeNudge(teamName, sourceRunId);
|
||||
}, delayMs);
|
||||
|
||||
this.pendingTimers.set(teamName, {
|
||||
timer,
|
||||
fireAtMs,
|
||||
sourceMessageAtMs: messageAtMs,
|
||||
sourceRunId,
|
||||
});
|
||||
}
|
||||
|
||||
cancelPendingAutoResume(teamName: string): void {
|
||||
const pending = this.pendingTimers.get(teamName);
|
||||
if (!pending) return;
|
||||
clearTimeout(pending.timer);
|
||||
this.pendingTimers.delete(teamName);
|
||||
}
|
||||
|
||||
clearAllPendingAutoResume(): void {
|
||||
for (const pending of this.pendingTimers.values()) {
|
||||
clearTimeout(pending.timer);
|
||||
}
|
||||
this.pendingTimers.clear();
|
||||
}
|
||||
|
||||
private async fireResumeNudge(teamName: string, sourceRunId: string | null): Promise<void> {
|
||||
const current = this.configManager.getConfig();
|
||||
if (!current.notifications.autoResumeOnRateLimit) {
|
||||
logger.info(
|
||||
`[auto-resume] Config flag was disabled while timer was pending - skipping nudge for "${teamName}"`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
if (!this.provisioningService.isTeamAlive(teamName)) {
|
||||
logger.info(
|
||||
`[auto-resume] Team "${teamName}" is no longer alive at fire time - skipping resume nudge`
|
||||
);
|
||||
return;
|
||||
}
|
||||
const currentRunId = this.provisioningService.getCurrentRunId(teamName);
|
||||
if (sourceRunId && currentRunId !== sourceRunId) {
|
||||
logger.info(
|
||||
`[auto-resume] Team "${teamName}" advanced from run "${sourceRunId}" to "${currentRunId ?? 'none'}" before fire time - skipping stale resume nudge`
|
||||
);
|
||||
return;
|
||||
}
|
||||
await this.provisioningService.sendMessageToTeam(teamName, AUTO_RESUME_MESSAGE);
|
||||
logger.info(`[auto-resume] Sent resume nudge to "${teamName}"`);
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
`[auto-resume] Failed to send resume nudge to "${teamName}": ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}`
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let autoResumeService: AutoResumeService | null = null;
|
||||
|
||||
export function initializeAutoResumeService(
|
||||
provisioningService: AutoResumeProvisioning
|
||||
): AutoResumeService {
|
||||
autoResumeService?.clearAllPendingAutoResume();
|
||||
autoResumeService = new AutoResumeService(provisioningService);
|
||||
return autoResumeService;
|
||||
}
|
||||
|
||||
export function getAutoResumeService(): AutoResumeService {
|
||||
if (!autoResumeService) {
|
||||
throw new Error('AutoResumeService is not initialized');
|
||||
}
|
||||
return autoResumeService;
|
||||
}
|
||||
|
||||
export function peekAutoResumeService(): AutoResumeService | null {
|
||||
return autoResumeService;
|
||||
}
|
||||
|
||||
export function clearAutoResumeService(): void {
|
||||
autoResumeService?.clearAllPendingAutoResume();
|
||||
autoResumeService = null;
|
||||
}
|
||||
|
|
@ -112,6 +112,7 @@ import { TeamMembersMetaStore } from './TeamMembersMetaStore';
|
|||
import { TeamMetaStore } from './TeamMetaStore';
|
||||
import { TeamSentMessagesStore } from './TeamSentMessagesStore';
|
||||
import { TeamTaskReader } from './TeamTaskReader';
|
||||
import { peekAutoResumeService } from './AutoResumeService';
|
||||
|
||||
/**
|
||||
* Kill a team CLI process using SIGKILL (uncatchable).
|
||||
|
|
@ -2342,6 +2343,40 @@ export class TeamProvisioningService {
|
|||
return this.getProvisioningRunId(teamName) ?? this.getAliveRunId(teamName);
|
||||
}
|
||||
|
||||
private clearSameTeamRetryTimers(teamName: string): void {
|
||||
for (const suffix of ['deferred', 'persist']) {
|
||||
const key = `same-team-${suffix}:${teamName}`;
|
||||
const timer = this.pendingTimeouts.get(key);
|
||||
if (timer) {
|
||||
clearTimeout(timer);
|
||||
this.pendingTimeouts.delete(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private resetTeamScopedTransientStateForNewRun(teamName: string): void {
|
||||
peekAutoResumeService()?.cancelPendingAutoResume(teamName);
|
||||
this.leadInboxRelayInFlight.delete(teamName);
|
||||
this.relayedLeadInboxMessageIds.delete(teamName);
|
||||
this.pendingCrossTeamFirstReplies.delete(teamName);
|
||||
this.recentCrossTeamLeadDeliveryMessageIds.delete(teamName);
|
||||
this.recentSameTeamNativeFingerprints.delete(teamName);
|
||||
this.clearSameTeamRetryTimers(teamName);
|
||||
|
||||
for (const key of Array.from(this.memberInboxRelayInFlight.keys())) {
|
||||
if (key.startsWith(`${teamName}:`)) {
|
||||
this.memberInboxRelayInFlight.delete(key);
|
||||
}
|
||||
}
|
||||
for (const key of Array.from(this.relayedMemberInboxMessageIds.keys())) {
|
||||
if (key.startsWith(`${teamName}:`)) {
|
||||
this.relayedMemberInboxMessageIds.delete(key);
|
||||
}
|
||||
}
|
||||
|
||||
this.liveLeadProcessMessages.delete(teamName);
|
||||
}
|
||||
|
||||
private appendCliLogs(run: ProvisioningRun, stream: 'stdout' | 'stderr', text: string): void {
|
||||
const nowMs = Date.now();
|
||||
run.claudeLogsUpdatedAt = new Date(nowMs).toISOString();
|
||||
|
|
@ -3074,7 +3109,61 @@ export class TeamProvisioningService {
|
|||
}
|
||||
|
||||
getLiveLeadProcessMessages(teamName: string): InboxMessage[] {
|
||||
return [...(this.liveLeadProcessMessages.get(teamName) ?? [])];
|
||||
const list = this.liveLeadProcessMessages.get(teamName) ?? [];
|
||||
const runId = this.getTrackedRunId(teamName);
|
||||
const sessionId = runId ? this.runs.get(runId)?.detectedSessionId : null;
|
||||
if (sessionId) {
|
||||
for (const message of list) {
|
||||
if (!message.leadSessionId && message.source === 'lead_process') {
|
||||
message.leadSessionId = sessionId;
|
||||
}
|
||||
}
|
||||
}
|
||||
return [...list];
|
||||
}
|
||||
|
||||
private pruneLiveLeadMessagesForCleanedRun(run: ProvisioningRun): void {
|
||||
const list = this.liveLeadProcessMessages.get(run.teamName);
|
||||
if (!list || list.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const runMessageIdPrefixes = [
|
||||
`lead-turn-${run.runId}-`,
|
||||
`lead-sendmsg-${run.runId}-`,
|
||||
`lead-process-${run.runId}-`,
|
||||
`compact-${run.runId}-`,
|
||||
];
|
||||
|
||||
const filtered = list.filter((message) => {
|
||||
const messageId = typeof message.messageId === 'string' ? message.messageId.trim() : '';
|
||||
if (messageId && runMessageIdPrefixes.some((prefix) => messageId.startsWith(prefix))) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (run.detectedSessionId && message.leadSessionId === run.detectedSessionId) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
});
|
||||
|
||||
if (filtered.length === 0) {
|
||||
this.liveLeadProcessMessages.delete(run.teamName);
|
||||
return;
|
||||
}
|
||||
|
||||
this.liveLeadProcessMessages.set(run.teamName, filtered);
|
||||
}
|
||||
|
||||
getCurrentLeadSessionId(teamName: string): string | null {
|
||||
const runId = this.getTrackedRunId(teamName);
|
||||
if (!runId) return null;
|
||||
return this.runs.get(runId)?.detectedSessionId ?? null;
|
||||
}
|
||||
|
||||
getCurrentRunId(teamName: string): string | null {
|
||||
return this.getAliveRunId(teamName);
|
||||
}
|
||||
|
||||
getLeadActivityState(teamName: string): {
|
||||
|
|
@ -4986,6 +5075,7 @@ export class TeamProvisioningService {
|
|||
},
|
||||
};
|
||||
|
||||
this.resetTeamScopedTransientStateForNewRun(request.teamName);
|
||||
this.runs.set(runId, run);
|
||||
this.provisioningRunByTeam.set(request.teamName, runId);
|
||||
run.onProgress(run.progress);
|
||||
|
|
@ -5531,7 +5621,7 @@ export class TeamProvisioningService {
|
|||
pendingInboxRelayCandidates: [],
|
||||
provisioningOutputParts: [],
|
||||
provisioningOutputIndexByMessageId: new Map(),
|
||||
detectedSessionId: null,
|
||||
detectedSessionId: previousSessionId ?? null,
|
||||
leadActivityState: 'active',
|
||||
leadContextUsage: null,
|
||||
authFailureRetried: false,
|
||||
|
|
@ -5571,6 +5661,7 @@ export class TeamProvisioningService {
|
|||
},
|
||||
};
|
||||
|
||||
this.resetTeamScopedTransientStateForNewRun(request.teamName);
|
||||
this.runs.set(runId, run);
|
||||
this.provisioningRunByTeam.set(request.teamName, runId);
|
||||
run.onProgress(run.progress);
|
||||
|
|
@ -5829,6 +5920,21 @@ export class TeamProvisioningService {
|
|||
throw new Error(`Team "${teamName}" process stdin is not writable`);
|
||||
}
|
||||
|
||||
await this.sendMessageToRun(run, message, attachments);
|
||||
}
|
||||
|
||||
private async sendMessageToRun(
|
||||
run: ProvisioningRun,
|
||||
message: string,
|
||||
attachments?: { data: string; mimeType: string; filename?: string }[]
|
||||
): Promise<void> {
|
||||
if (!this.isCurrentTrackedRun(run)) {
|
||||
throw new Error(`Team "${run.teamName}" run "${run.runId}" is no longer current`);
|
||||
}
|
||||
if (run.processKilled || run.cancelRequested || !run.child?.stdin?.writable) {
|
||||
throw new Error(`Team "${run.teamName}" process stdin is not writable`);
|
||||
}
|
||||
|
||||
const contentBlocks: Record<string, unknown>[] = [{ type: 'text', text: message }];
|
||||
if (attachments?.length) {
|
||||
for (const att of attachments) {
|
||||
|
|
@ -5948,7 +6054,7 @@ export class TeamProvisioningService {
|
|||
userText,
|
||||
].join('\n');
|
||||
|
||||
await this.sendMessageToTeam(teamName, message);
|
||||
await this.sendMessageToRun(run, message);
|
||||
}
|
||||
|
||||
async relayMemberInboxMessages(teamName: string, memberName: string): Promise<number> {
|
||||
|
|
@ -5970,6 +6076,8 @@ export class TeamProvisioningService {
|
|||
const run = this.runs.get(runId);
|
||||
if (!run?.child || run.processKilled || run.cancelRequested) return 0;
|
||||
if (!run.provisioningComplete) return 0;
|
||||
const isStaleRelayRun = (): boolean =>
|
||||
!this.isCurrentTrackedRun(run) || !run.child || run.processKilled || run.cancelRequested;
|
||||
|
||||
const relayedIds = this.relayedMemberInboxMessageIds.get(relayKey) ?? new Set<string>();
|
||||
|
||||
|
|
@ -5979,6 +6087,7 @@ export class TeamProvisioningService {
|
|||
} catch {
|
||||
return 0;
|
||||
}
|
||||
if (isStaleRelayRun()) return 0;
|
||||
|
||||
const unread = memberInboxMessages
|
||||
.filter((m): m is InboxMessage & { messageId: string } => {
|
||||
|
|
@ -6009,6 +6118,7 @@ export class TeamProvisioningService {
|
|||
.map(({ message }) => message);
|
||||
|
||||
const readOnlyIgnoredUnread = [...silentNoiseUnread, ...passiveIdleUnread];
|
||||
if (isStaleRelayRun()) return 0;
|
||||
|
||||
if (readOnlyIgnoredUnread.length > 0) {
|
||||
try {
|
||||
|
|
@ -6082,7 +6192,7 @@ export class TeamProvisioningService {
|
|||
].join('\n');
|
||||
|
||||
try {
|
||||
await this.sendMessageToTeam(teamName, message);
|
||||
await this.sendMessageToRun(run, message);
|
||||
} catch {
|
||||
this.forgetPendingInboxRelayCandidates(run, memberName, rememberedRelayIds);
|
||||
return 0;
|
||||
|
|
@ -6138,6 +6248,8 @@ export class TeamProvisioningService {
|
|||
if (!runId) return 0;
|
||||
const run = this.runs.get(runId);
|
||||
if (!run?.child || run.processKilled || run.cancelRequested) return 0;
|
||||
const isStaleRelayRun = (): boolean =>
|
||||
!this.isCurrentTrackedRun(run) || !run.child || run.processKilled || run.cancelRequested;
|
||||
|
||||
// Permission request scan runs even during provisioning — teammates may need
|
||||
// tool approval before the lead's first turn completes. CLI marks inbox messages
|
||||
|
|
@ -6148,10 +6260,12 @@ export class TeamProvisioningService {
|
|||
} catch {
|
||||
// config not ready yet during early provisioning — skip scan
|
||||
}
|
||||
if (isStaleRelayRun()) return 0;
|
||||
if (config) {
|
||||
const leadName = config.members?.find((m) => isLeadMember(m))?.name?.trim() || 'team-lead';
|
||||
try {
|
||||
const leadInboxMessages = await this.inboxReader.getMessagesFor(teamName, leadName);
|
||||
if (isStaleRelayRun()) return 0;
|
||||
const permMsgsToMarkRead: { messageId: string }[] = [];
|
||||
const runStartedAtMs = Date.parse(run.startedAt);
|
||||
for (const msg of leadInboxMessages) {
|
||||
|
|
@ -6196,6 +6310,7 @@ export class TeamProvisioningService {
|
|||
return 0;
|
||||
}
|
||||
}
|
||||
if (isStaleRelayRun()) return 0;
|
||||
if (!config) return 0;
|
||||
|
||||
const leadName = config.members?.find((m) => isLeadMember(m))?.name?.trim() || 'team-lead';
|
||||
|
|
@ -6205,8 +6320,10 @@ export class TeamProvisioningService {
|
|||
} catch {
|
||||
return 0;
|
||||
}
|
||||
if (isStaleRelayRun()) return 0;
|
||||
|
||||
await this.refreshMemberSpawnStatusesFromLeadInbox(run);
|
||||
if (isStaleRelayRun()) return 0;
|
||||
|
||||
const unread = leadInboxMessages
|
||||
.filter((m): m is InboxMessage & { messageId: string } => {
|
||||
|
|
@ -6344,6 +6461,7 @@ export class TeamProvisioningService {
|
|||
...passiveIdleUnread.map((m) => m.messageId),
|
||||
]);
|
||||
const remainingUnread = unread.filter((m) => !readOnlyIgnoredIds.has(m.messageId));
|
||||
if (isStaleRelayRun()) return 0;
|
||||
|
||||
// Category 2: same-team native delivery confirmation (one-to-one pairing).
|
||||
const { nativeMatchedMessageIds, persisted: sameTeamPersisted } =
|
||||
|
|
@ -6506,7 +6624,7 @@ export class TeamProvisioningService {
|
|||
});
|
||||
|
||||
try {
|
||||
await this.sendMessageToTeam(teamName, message);
|
||||
await this.sendMessageToRun(run, message);
|
||||
} catch {
|
||||
if (run.leadRelayCapture) {
|
||||
clearTimeout(run.leadRelayCapture.timeoutHandle);
|
||||
|
|
@ -7552,6 +7670,12 @@ export class TeamProvisioningService {
|
|||
if (result.deduplicated) {
|
||||
return;
|
||||
}
|
||||
if (this.getTrackedRunId(run.teamName) !== run.runId) {
|
||||
logger.debug(
|
||||
`[${run.teamName}] Skipping stale cross-team send result for old run ${run.runId}`
|
||||
);
|
||||
return;
|
||||
}
|
||||
const msg: InboxMessage = {
|
||||
from: leadName,
|
||||
to: recipient.startsWith('cross-team:')
|
||||
|
|
@ -7779,11 +7903,18 @@ export class TeamProvisioningService {
|
|||
private pushLiveLeadTextMessage(
|
||||
run: ProvisioningRun,
|
||||
cleanText: string,
|
||||
stableMessageId?: string
|
||||
stableMessageId?: string,
|
||||
messageTimestamp?: string
|
||||
): void {
|
||||
run.leadMsgSeq += 1;
|
||||
const leadName = this.getRunLeadName(run);
|
||||
const messageId = stableMessageId || `lead-turn-${run.runId}-${run.leadMsgSeq}`;
|
||||
const timestamp =
|
||||
typeof messageTimestamp === 'string' &&
|
||||
messageTimestamp.trim().length > 0 &&
|
||||
Number.isFinite(Date.parse(messageTimestamp))
|
||||
? messageTimestamp
|
||||
: nowIso();
|
||||
// Attach accumulated tool call details from preceding tool_use messages, then reset.
|
||||
const toolCalls = run.pendingToolCalls.length > 0 ? [...run.pendingToolCalls] : undefined;
|
||||
const toolSummary = toolCalls ? formatToolSummaryFromCalls(toolCalls) : undefined;
|
||||
|
|
@ -7791,7 +7922,7 @@ export class TeamProvisioningService {
|
|||
const leadMsg: InboxMessage = {
|
||||
from: leadName,
|
||||
text: cleanText,
|
||||
timestamp: nowIso(),
|
||||
timestamp,
|
||||
read: true,
|
||||
summary: cleanText.length > 60 ? cleanText.slice(0, 57) + '...' : cleanText,
|
||||
messageId,
|
||||
|
|
@ -8139,6 +8270,18 @@ export class TeamProvisioningService {
|
|||
// stream-json output has various message types:
|
||||
// {"type":"assistant","content":[{"type":"text","text":"..."},...]}
|
||||
// {"type":"result","subtype":"success",...}
|
||||
// Capture session_id as early as possible so live messages emitted during this
|
||||
// handler already carry the session identity used by merge/dedup paths.
|
||||
if (!run.detectedSessionId) {
|
||||
const sid = typeof msg.session_id === 'string' ? msg.session_id : undefined;
|
||||
if (sid && sid.trim().length > 0) {
|
||||
run.detectedSessionId = sid.trim();
|
||||
logger.info(
|
||||
`[${run.teamName}] Detected session ID from stream-json: ${run.detectedSessionId}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (msg.type === 'user') {
|
||||
// Check for permission_request in raw user message text BEFORE teammate-message parsing.
|
||||
// The permission_request may arrive as plain JSON without <teammate-message> wrapper,
|
||||
|
|
@ -8181,6 +8324,12 @@ export class TeamProvisioningService {
|
|||
.map((part) => part.text as string);
|
||||
if (textParts.length > 0) {
|
||||
const text = textParts.join('\n');
|
||||
const messageTimestamp =
|
||||
typeof msg.timestamp === 'string' &&
|
||||
msg.timestamp.trim().length > 0 &&
|
||||
Number.isFinite(Date.parse(msg.timestamp))
|
||||
? msg.timestamp
|
||||
: undefined;
|
||||
// Auth failures sometimes show up as assistant text (e.g. "401", "Please run /login")
|
||||
// rather than stderr or a result.subtype=error. Detect early to avoid false "ready".
|
||||
this.handleAuthFailureInOutput(run, text, 'assistant');
|
||||
|
|
@ -8223,7 +8372,8 @@ export class TeamProvisioningService {
|
|||
this.pushLiveLeadTextMessage(
|
||||
run,
|
||||
cleanText,
|
||||
this.getStableLeadThoughtMessageId(msg) ?? undefined
|
||||
this.getStableLeadThoughtMessageId(msg) ?? undefined,
|
||||
messageTimestamp
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -8236,7 +8386,8 @@ export class TeamProvisioningService {
|
|||
this.pushLiveLeadTextMessage(
|
||||
run,
|
||||
cleanText,
|
||||
this.getStableLeadThoughtMessageId(msg) ?? undefined
|
||||
this.getStableLeadThoughtMessageId(msg) ?? undefined,
|
||||
messageTimestamp
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -8320,17 +8471,6 @@ export class TeamProvisioningService {
|
|||
}
|
||||
}
|
||||
|
||||
// Capture session_id from any message type (first occurrence wins)
|
||||
if (!run.detectedSessionId) {
|
||||
const sid = typeof msg.session_id === 'string' ? msg.session_id : undefined;
|
||||
if (sid && sid.trim().length > 0) {
|
||||
run.detectedSessionId = sid.trim();
|
||||
logger.info(
|
||||
`[${run.teamName}] Detected session ID from stream-json: ${run.detectedSessionId}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (this.handleDeterministicBootstrapEvent(run, msg)) {
|
||||
return;
|
||||
}
|
||||
|
|
@ -9916,7 +10056,7 @@ export class TeamProvisioningService {
|
|||
`Не стартовали тиммейты: ${failedSpawnMembers.map((member) => `@${member.name}`).join(', ')}.`,
|
||||
`Не считай их доступными, пока их запуск не будет повторён успешно.`,
|
||||
].join(' ');
|
||||
await this.sendMessageToTeam(run.teamName, failureNotice).catch((error: unknown) =>
|
||||
await this.sendMessageToRun(run, failureNotice).catch((error: unknown) =>
|
||||
logger.warn(
|
||||
`[${run.teamName}] failed to send teammate-start failure notice to lead: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
|
|
@ -9964,7 +10104,7 @@ export class TeamProvisioningService {
|
|||
.filter(Boolean)
|
||||
.join('\n\n');
|
||||
|
||||
await this.sendMessageToTeam(run.teamName, message);
|
||||
await this.sendMessageToRun(run, message);
|
||||
} catch (error) {
|
||||
logger.warn(
|
||||
`[${run.teamName}] Failed to kick off solo task resumption: ${
|
||||
|
|
@ -10084,7 +10224,7 @@ export class TeamProvisioningService {
|
|||
`Не стартовали тиммейты: ${failedSpawnMembers.map((member) => `@${member.name}`).join(', ')}.`,
|
||||
`Не считай их доступными, пока их запуск не будет повторён успешно.`,
|
||||
].join(' ');
|
||||
await this.sendMessageToTeam(run.teamName, failureNotice).catch((error: unknown) =>
|
||||
await this.sendMessageToRun(run, failureNotice).catch((error: unknown) =>
|
||||
logger.warn(
|
||||
`[${run.teamName}] failed to send teammate-start failure notice to lead: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
|
|
@ -10404,7 +10544,14 @@ export class TeamProvisioningService {
|
|||
* Remove a run from tracking maps.
|
||||
*/
|
||||
private cleanupRun(run: ProvisioningRun): void {
|
||||
if (run.isLaunch && !run.provisioningComplete) {
|
||||
const currentTrackedRunId = this.getTrackedRunId(run.teamName);
|
||||
const hasNewerTrackedRun = currentTrackedRunId !== null && currentTrackedRunId !== run.runId;
|
||||
|
||||
if (!hasNewerTrackedRun) {
|
||||
peekAutoResumeService()?.cancelPendingAutoResume(run.teamName);
|
||||
}
|
||||
|
||||
if (!hasNewerTrackedRun && run.isLaunch && !run.provisioningComplete) {
|
||||
void this.persistLaunchStateSnapshot(run, 'finished');
|
||||
}
|
||||
this.resetRuntimeToolActivity(run);
|
||||
|
|
@ -10433,19 +10580,13 @@ export class TeamProvisioningService {
|
|||
if (this.aliveRunByTeam.get(run.teamName) === run.runId) {
|
||||
this.aliveRunByTeam.delete(run.teamName);
|
||||
}
|
||||
this.leadInboxRelayInFlight.delete(run.teamName);
|
||||
this.relayedLeadInboxMessageIds.delete(run.teamName);
|
||||
this.pendingCrossTeamFirstReplies.delete(run.teamName);
|
||||
this.recentCrossTeamLeadDeliveryMessageIds.delete(run.teamName);
|
||||
this.recentSameTeamNativeFingerprints.delete(run.teamName);
|
||||
// Clear same-team retry timers
|
||||
for (const suffix of ['deferred', 'persist']) {
|
||||
const key = `same-team-${suffix}:${run.teamName}`;
|
||||
const timer = this.pendingTimeouts.get(key);
|
||||
if (timer) {
|
||||
clearTimeout(timer);
|
||||
this.pendingTimeouts.delete(key);
|
||||
}
|
||||
if (!hasNewerTrackedRun) {
|
||||
this.leadInboxRelayInFlight.delete(run.teamName);
|
||||
this.relayedLeadInboxMessageIds.delete(run.teamName);
|
||||
this.pendingCrossTeamFirstReplies.delete(run.teamName);
|
||||
this.recentCrossTeamLeadDeliveryMessageIds.delete(run.teamName);
|
||||
this.recentSameTeamNativeFingerprints.delete(run.teamName);
|
||||
this.clearSameTeamRetryTimers(run.teamName);
|
||||
}
|
||||
for (const memberName of run.memberSpawnStatuses.keys()) {
|
||||
const key = this.getMemberLaunchGraceKey(run, memberName);
|
||||
|
|
@ -10457,17 +10598,21 @@ export class TeamProvisioningService {
|
|||
}
|
||||
run.activeCrossTeamReplyHints = [];
|
||||
run.pendingInboxRelayCandidates = [];
|
||||
for (const key of Array.from(this.memberInboxRelayInFlight.keys())) {
|
||||
if (key.startsWith(`${run.teamName}:`)) {
|
||||
this.memberInboxRelayInFlight.delete(key);
|
||||
if (!hasNewerTrackedRun) {
|
||||
for (const key of Array.from(this.memberInboxRelayInFlight.keys())) {
|
||||
if (key.startsWith(`${run.teamName}:`)) {
|
||||
this.memberInboxRelayInFlight.delete(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const key of Array.from(this.relayedMemberInboxMessageIds.keys())) {
|
||||
if (key.startsWith(`${run.teamName}:`)) {
|
||||
this.relayedMemberInboxMessageIds.delete(key);
|
||||
for (const key of Array.from(this.relayedMemberInboxMessageIds.keys())) {
|
||||
if (key.startsWith(`${run.teamName}:`)) {
|
||||
this.relayedMemberInboxMessageIds.delete(key);
|
||||
}
|
||||
}
|
||||
this.liveLeadProcessMessages.delete(run.teamName);
|
||||
} else {
|
||||
this.pruneLiveLeadMessagesForCleanedRun(run);
|
||||
}
|
||||
this.liveLeadProcessMessages.delete(run.teamName);
|
||||
// Dismiss any pending tool approvals for this run
|
||||
if (run.pendingApprovals.size > 0) {
|
||||
for (const requestId of run.pendingApprovals.keys()) {
|
||||
|
|
|
|||
|
|
@ -16,6 +16,12 @@ export { BoardTaskActivityService } from './taskLogs/activity/BoardTaskActivityS
|
|||
export { BoardTaskExactLogDetailService } from './taskLogs/exact/BoardTaskExactLogDetailService';
|
||||
export { BoardTaskExactLogsService } from './taskLogs/exact/BoardTaskExactLogsService';
|
||||
export { BoardTaskLogStreamService } from './taskLogs/stream/BoardTaskLogStreamService';
|
||||
export {
|
||||
AutoResumeService,
|
||||
clearAutoResumeService,
|
||||
getAutoResumeService,
|
||||
initializeAutoResumeService,
|
||||
} from './AutoResumeService';
|
||||
export { TeamAttachmentStore } from './TeamAttachmentStore';
|
||||
export { TeamBackupService } from './TeamBackupService';
|
||||
export { TeamConfigReader } from './TeamConfigReader';
|
||||
|
|
|
|||
|
|
@ -54,6 +54,7 @@ export interface SafeConfig {
|
|||
notifyOnCrossTeamMessage: boolean;
|
||||
notifyOnTeamLaunched: boolean;
|
||||
notifyOnToolApproval: boolean;
|
||||
autoResumeOnRateLimit: boolean;
|
||||
statusChangeOnlySolo: boolean;
|
||||
statusChangeStatuses: string[];
|
||||
triggers: AppConfig['notifications']['triggers'];
|
||||
|
|
@ -195,6 +196,7 @@ export function useSettingsConfig(): UseSettingsConfigReturn {
|
|||
notifyOnCrossTeamMessage: displayConfig?.notifications?.notifyOnCrossTeamMessage ?? true,
|
||||
notifyOnTeamLaunched: displayConfig?.notifications?.notifyOnTeamLaunched ?? true,
|
||||
notifyOnToolApproval: displayConfig?.notifications?.notifyOnToolApproval ?? true,
|
||||
autoResumeOnRateLimit: displayConfig?.notifications?.autoResumeOnRateLimit ?? false,
|
||||
statusChangeOnlySolo: displayConfig?.notifications?.statusChangeOnlySolo ?? true,
|
||||
statusChangeStatuses: displayConfig?.notifications?.statusChangeStatuses ?? [
|
||||
'in_progress',
|
||||
|
|
|
|||
|
|
@ -311,6 +311,7 @@ export function useSettingsHandlers({
|
|||
notifyOnCrossTeamMessage: true,
|
||||
notifyOnTeamLaunched: true,
|
||||
notifyOnToolApproval: true,
|
||||
autoResumeOnRateLimit: false,
|
||||
statusChangeOnlySolo: true,
|
||||
statusChangeStatuses: ['in_progress', 'completed'],
|
||||
triggers: defaultTriggers,
|
||||
|
|
|
|||
|
|
@ -76,6 +76,7 @@ interface NotificationsSectionProps {
|
|||
| 'notifyOnCrossTeamMessage'
|
||||
| 'notifyOnTeamLaunched'
|
||||
| 'notifyOnToolApproval'
|
||||
| 'autoResumeOnRateLimit'
|
||||
| 'statusChangeOnlySolo',
|
||||
value: boolean
|
||||
) => void;
|
||||
|
|
@ -360,6 +361,17 @@ export const NotificationsSection = ({
|
|||
disabled={saving || !safeConfig.notifications.enabled}
|
||||
/>
|
||||
</SettingRow>
|
||||
<SettingRow
|
||||
label="Auto-resume after rate limit"
|
||||
description="When Claude reports a reset time, schedule a follow-up nudge for the team lead after the limit resets"
|
||||
icon={<Clock className="size-4" />}
|
||||
>
|
||||
<SettingsToggle
|
||||
enabled={safeConfig.notifications.autoResumeOnRateLimit}
|
||||
onChange={(v) => onNotificationToggle('autoResumeOnRateLimit', v)}
|
||||
disabled={saving || !safeConfig.notifications.enabled}
|
||||
/>
|
||||
</SettingRow>
|
||||
|
||||
{/* Task Status Change Notifications — nested within team card */}
|
||||
<div className="last:*:border-b-0">
|
||||
|
|
|
|||
|
|
@ -291,6 +291,8 @@ export interface AppConfig {
|
|||
notifyOnTeamLaunched: boolean;
|
||||
/** Whether to show native OS notifications when a tool needs user approval (Allow/Deny) */
|
||||
notifyOnToolApproval: boolean;
|
||||
/** Whether to automatically nudge a rate-limited team after the limit resets */
|
||||
autoResumeOnRateLimit: boolean;
|
||||
/** Only notify on status changes in solo teams (no teammates) */
|
||||
statusChangeOnlySolo: boolean;
|
||||
/** Which target statuses to notify about (e.g. ['in_progress', 'completed']) */
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/**
|
||||
* Detects rate limit messages from Claude.
|
||||
* Detects rate limit messages from Claude and parses reset time from them.
|
||||
*/
|
||||
|
||||
const RATE_LIMIT_SUBSTRING = "You've hit your limit";
|
||||
|
|
@ -10,3 +10,229 @@ const RATE_LIMIT_SUBSTRING = "You've hit your limit";
|
|||
export function isRateLimitMessage(text: string): boolean {
|
||||
return text.includes(RATE_LIMIT_SUBSTRING);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Reset-time parsing
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Maps known Claude timezone abbreviations to fixed UTC offsets in minutes.
|
||||
* We only include zones Claude's API has been observed to emit. When the
|
||||
* message contains an explicit parenthesized timezone that is NOT in this
|
||||
* map, the parser returns `null` rather than guessing. When no timezone is
|
||||
* present at all, the hour:minute is treated as user-local time.
|
||||
*/
|
||||
const TIMEZONE_OFFSETS_MIN: Record<string, number> = {
|
||||
UTC: 0,
|
||||
GMT: 0,
|
||||
// North America — standard times
|
||||
EST: -5 * 60,
|
||||
CST: -6 * 60,
|
||||
MST: -7 * 60,
|
||||
PST: -8 * 60,
|
||||
// North America — daylight times
|
||||
EDT: -4 * 60,
|
||||
CDT: -5 * 60,
|
||||
MDT: -6 * 60,
|
||||
PDT: -7 * 60,
|
||||
};
|
||||
|
||||
/**
|
||||
* Attempts to parse the reset time from a Claude rate-limit message.
|
||||
*
|
||||
* Supported formats (case-insensitive):
|
||||
* - "limit will reset at 3pm (PST)"
|
||||
* - "limit will reset at 3:30 pm (PST)"
|
||||
* - "limit will reset at 15:30 UTC"
|
||||
* - "resets at 3pm" (local time assumed)
|
||||
* - "resets in 2 hours"
|
||||
* - "resets in 45 minutes"
|
||||
*
|
||||
* Returns `null` when the reset time cannot be extracted reliably. Also returns
|
||||
* null for text that does not look like a rate-limit message, so the parser is
|
||||
* safe to call on arbitrary strings.
|
||||
*
|
||||
* @param text the full rate-limit message text
|
||||
* @param now reference "now" used to resolve wall-clock times and relative
|
||||
* offsets (exposed for testability; defaults to `new Date()`)
|
||||
*/
|
||||
export function parseRateLimitResetTime(text: string, now: Date = new Date()): Date | null {
|
||||
if (!text) return null;
|
||||
// Defensive gate: only parse text that actually looks like a rate-limit
|
||||
// message. Prevents false positives from unrelated prose containing
|
||||
// words like "reset" (e.g. "reset the 5pm meeting").
|
||||
if (!isRateLimitMessage(text)) return null;
|
||||
|
||||
const relative = parseRelativeResetDuration(text);
|
||||
if (relative !== null) {
|
||||
return new Date(now.getTime() + relative);
|
||||
}
|
||||
|
||||
return parseAbsoluteResetClockTime(text, now);
|
||||
}
|
||||
|
||||
/**
|
||||
* Matches trailing qualifiers that shift the reset to a different day.
|
||||
* When present, we can't reliably resolve the date without more context, so
|
||||
* the parser bails out. Example: "reset at 3pm (PST) next week" — the naive
|
||||
* "today or tomorrow" rollover would fire in hours instead of a week.
|
||||
*/
|
||||
const DAY_SHIFT_QUALIFIER_RE =
|
||||
/\b(?:next\s+week|next\s+month|tomorrow|yesterday|on\s+(?:mon|tue|wed|thu|fri|sat|sun)[a-z]*)\b/i;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Relative durations: "resets in 2 hours", "resets in 45 minutes"
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const RESET_VERB_RE = /\breset(?:s|ting)?\b/i;
|
||||
const LEADING_FILLER_RE = /^(?:about|around)\s+/i;
|
||||
const LEADING_TIME_VALUE_RE = /^(\d+(?:\.\d+)?)\s*([a-z]+)\b/i;
|
||||
|
||||
function parseRelativeResetDuration(text: string): number | null {
|
||||
const resetVerbMatch = RESET_VERB_RE.exec(text);
|
||||
if (!resetVerbMatch) return null;
|
||||
|
||||
const afterVerb = text.slice(resetVerbMatch.index + resetVerbMatch[0].length).trimStart();
|
||||
if (!afterVerb.toLowerCase().startsWith('in')) return null;
|
||||
|
||||
let tail = afterVerb.slice(2).trimStart();
|
||||
if (tail.startsWith('~')) {
|
||||
tail = tail.slice(1).trimStart();
|
||||
}
|
||||
tail = tail.replace(LEADING_FILLER_RE, '');
|
||||
|
||||
const match = LEADING_TIME_VALUE_RE.exec(tail);
|
||||
if (!match) return null;
|
||||
|
||||
const amount = Number.parseFloat(match[1]!);
|
||||
if (!Number.isFinite(amount) || amount < 0) return null;
|
||||
|
||||
const unit = match[2]!.toLowerCase();
|
||||
if (['second', 'seconds', 'sec', 'secs', 's'].includes(unit)) {
|
||||
return Math.round(amount * 1000);
|
||||
}
|
||||
if (['minute', 'minutes', 'min', 'mins', 'm'].includes(unit)) {
|
||||
return Math.round(amount * 60 * 1000);
|
||||
}
|
||||
if (['hour', 'hours', 'hr', 'hrs', 'h'].includes(unit)) {
|
||||
return Math.round(amount * 60 * 60 * 1000);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Absolute clock times: "resets at 3pm (PST)", "resets at 15:30 UTC"
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Captures the clock time + optional timezone abbreviation from phrases like
|
||||
* "reset at 3pm (PST)" or "resets at 15:30 UTC".
|
||||
*/
|
||||
const LEADING_CLOCK_RE = /^(\d{1,2})(?::(\d{2}))?\s*(am|pm)?\b/i;
|
||||
const PAREN_TZ_RE = /^\(([A-Za-z]{2,5})\)/;
|
||||
const TRAILING_TZ_RE = /^([A-Za-z]{2,5})\b/;
|
||||
|
||||
function parseAbsoluteResetClockTime(text: string, now: Date): Date | null {
|
||||
const resetVerbMatch = RESET_VERB_RE.exec(text);
|
||||
if (!resetVerbMatch) return null;
|
||||
|
||||
let tail = text.slice(resetVerbMatch.index + resetVerbMatch[0].length).trimStart();
|
||||
if (tail.toLowerCase().startsWith('at ')) {
|
||||
tail = tail.slice(3).trimStart();
|
||||
}
|
||||
|
||||
const match = LEADING_CLOCK_RE.exec(tail);
|
||||
if (!match) return null;
|
||||
|
||||
tail = tail.slice(match[0].length).trimStart();
|
||||
const parenthesizedTzMatch = PAREN_TZ_RE.exec(tail);
|
||||
const bareWordMatch = parenthesizedTzMatch ? null : TRAILING_TZ_RE.exec(tail);
|
||||
const bareTzMatch =
|
||||
bareWordMatch && bareWordMatch[1].toUpperCase() in TIMEZONE_OFFSETS_MIN ? bareWordMatch : null;
|
||||
const tzTokenLength = parenthesizedTzMatch?.[0].length ?? bareTzMatch?.[0].length ?? 0;
|
||||
|
||||
// If the text contains a day-shift qualifier ("next week", "on Tuesday",
|
||||
// etc.), the "today or tomorrow" rollover below would produce a materially
|
||||
// wrong time. Bail out and let the caller fall back to no auto-resume.
|
||||
const afterMatch = tail.slice(tzTokenLength);
|
||||
if (DAY_SHIFT_QUALIFIER_RE.test(afterMatch)) return null;
|
||||
|
||||
const hourRaw = Number.parseInt(match[1]!, 10);
|
||||
const minuteRaw = match[2] ? Number.parseInt(match[2], 10) : 0;
|
||||
const ampm = match[3]?.toLowerCase() ?? null;
|
||||
const parenthesizedTz = parenthesizedTzMatch?.[1]?.toUpperCase() ?? '';
|
||||
const trailingTz = bareTzMatch?.[1]?.toUpperCase() ?? '';
|
||||
|
||||
if (!Number.isFinite(hourRaw) || !Number.isFinite(minuteRaw)) return null;
|
||||
if (minuteRaw < 0 || minuteRaw > 59) return null;
|
||||
|
||||
let hour = hourRaw;
|
||||
if (ampm === 'pm' && hour < 12) hour += 12;
|
||||
else if (ampm === 'am' && hour === 12) hour = 0;
|
||||
|
||||
if (hour < 0 || hour > 23) return null;
|
||||
|
||||
// Timezone resolution treats parenthesized vs bare tokens differently.
|
||||
//
|
||||
// "reset at 3pm (PST)" — parenthesized, authoritative. Unknown zone
|
||||
// here means the sender meant a specific zone
|
||||
// we don't model; bail out rather than guess.
|
||||
// "reset at 3pm PST" — bare known abbreviation, same effect.
|
||||
// "reset at 3pm today" — bare unknown word ("TODAY"). This is just a
|
||||
// trailing word, not a real TZ claim; fall
|
||||
// back to local time instead of suppressing.
|
||||
// "reset at 3pm" — no token. Treat as user-local.
|
||||
let tzOffset: number | null;
|
||||
if (parenthesizedTz) {
|
||||
if (!(parenthesizedTz in TIMEZONE_OFFSETS_MIN)) return null;
|
||||
tzOffset = TIMEZONE_OFFSETS_MIN[parenthesizedTz]!;
|
||||
} else if (trailingTz && trailingTz in TIMEZONE_OFFSETS_MIN) {
|
||||
tzOffset = TIMEZONE_OFFSETS_MIN[trailingTz]!;
|
||||
} else {
|
||||
tzOffset = null;
|
||||
}
|
||||
|
||||
const candidateSeed =
|
||||
tzOffset === null
|
||||
? buildLocalToday(now, hour, minuteRaw)
|
||||
: buildUtcTodayWithOffset(now, hour, minuteRaw, tzOffset);
|
||||
let candidate: Date = candidateSeed;
|
||||
|
||||
// If the computed time is materially in the past (e.g. "3pm" parsed while
|
||||
// it's already 4pm), roll forward by one day. A small tolerance prevents
|
||||
// near-present timestamps — stale messages, clock skew, sub-second drift —
|
||||
// from being bumped 24 h forward, which would then trip the scheduler's
|
||||
// 12 h ceiling and silently drop auto-resume altogether. Timestamps within
|
||||
// `ROLLOVER_TOLERANCE_MS` of now fire immediately after the scheduler's
|
||||
// own 30 s buffer and `Math.max(0, rawDelayMs)` clamp.
|
||||
if (candidate.getTime() <= now.getTime() - ROLLOVER_TOLERANCE_MS) {
|
||||
candidate = new Date(candidate.getTime() + 24 * 60 * 60 * 1000);
|
||||
}
|
||||
return candidate;
|
||||
}
|
||||
|
||||
const ROLLOVER_TOLERANCE_MS = 60 * 1000;
|
||||
|
||||
function buildLocalToday(now: Date, hour: number, minute: number): Date {
|
||||
const d = new Date(now);
|
||||
d.setHours(hour, minute, 0, 0);
|
||||
return d;
|
||||
}
|
||||
|
||||
function buildUtcTodayWithOffset(
|
||||
now: Date,
|
||||
hour: number,
|
||||
minute: number,
|
||||
offsetMinutes: number
|
||||
): Date {
|
||||
// The caller's "hour:minute" is expressed in the target zone. Anchor the
|
||||
// calendar date in that zone too — not in UTC — otherwise we get a 24h
|
||||
// error when the zone-local day differs from UTC's day (e.g. 01:00 UTC is
|
||||
// still "yesterday" for any negative-offset zone like PST).
|
||||
const zoned = new Date(now.getTime() + offsetMinutes * 60 * 1000);
|
||||
const offsetMs = offsetMinutes * 60 * 1000;
|
||||
return new Date(
|
||||
Date.UTC(zoned.getUTCFullYear(), zoned.getUTCMonth(), zoned.getUTCDate(), hour, minute, 0, 0) -
|
||||
offsetMs
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -115,6 +115,7 @@ describe('configValidation', () => {
|
|||
'notifyOnClarifications',
|
||||
'notifyOnStatusChange',
|
||||
'notifyOnTeamLaunched',
|
||||
'autoResumeOnRateLimit',
|
||||
'statusChangeOnlySolo',
|
||||
] as const)('accepts boolean %s toggle', (key) => {
|
||||
const resultOn = validateConfigUpdatePayload('notifications', { [key]: true });
|
||||
|
|
@ -136,6 +137,7 @@ describe('configValidation', () => {
|
|||
'notifyOnClarifications',
|
||||
'notifyOnStatusChange',
|
||||
'notifyOnTeamLaunched',
|
||||
'autoResumeOnRateLimit',
|
||||
'statusChangeOnlySolo',
|
||||
] as const)('rejects non-boolean %s', (key) => {
|
||||
const result = validateConfigUpdatePayload('notifications', { [key]: 'yes' });
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import * as os from 'os';
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import type {
|
||||
BoardTaskActivityDetailResult,
|
||||
BoardTaskActivityEntry,
|
||||
|
|
@ -116,6 +116,7 @@ import {
|
|||
registerTeamHandlers,
|
||||
removeTeamHandlers,
|
||||
} from '../../../src/main/ipc/teams';
|
||||
import { ConfigManager } from '../../../src/main/services/infrastructure/ConfigManager';
|
||||
|
||||
describe('ipc teams handlers', () => {
|
||||
const handlers = new Map<string, (...args: unknown[]) => Promise<unknown>>();
|
||||
|
|
@ -192,10 +193,12 @@ describe('ipc teams handlers', () => {
|
|||
launchTeam: vi.fn(async () => ({ runId: 'run-2' })),
|
||||
sendMessageToTeam: vi.fn(async () => undefined),
|
||||
isTeamAlive: vi.fn(() => true),
|
||||
getCurrentRunId: vi.fn(() => 'run-2' as string | null),
|
||||
pushLiveLeadProcessMessage: vi.fn(),
|
||||
relayLeadInboxMessages: vi.fn(async () => 0),
|
||||
relayMemberInboxMessages: vi.fn(async () => 0),
|
||||
getLiveLeadProcessMessages: vi.fn(() => [] as InboxMessage[]),
|
||||
getCurrentLeadSessionId: vi.fn(() => null as string | null),
|
||||
getAliveTeams: vi.fn(() => ['my-team']),
|
||||
getLeadActivityState: vi.fn(() => 'idle'),
|
||||
stopTeam: vi.fn(() => undefined),
|
||||
|
|
@ -249,6 +252,10 @@ describe('ipc teams handlers', () => {
|
|||
registerTeamHandlers(ipcMain as never);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it('registers all expected handlers', () => {
|
||||
expect(handlers.has(TEAM_LIST)).toBe(true);
|
||||
expect(handlers.has(TEAM_GET_DATA)).toBe(true);
|
||||
|
|
@ -799,6 +806,81 @@ describe('ipc teams handlers', () => {
|
|||
expect(sources.filter((s) => s === 'lead_session')).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('does not let a live duplicate of the same session rate-limit reply delay auto-resume', async () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date('2026-04-17T12:00:30.000Z'));
|
||||
const configManager = ConfigManager.getInstance();
|
||||
const actualConfig = configManager.getConfig();
|
||||
const getConfigSpy = vi.spyOn(configManager, 'getConfig').mockImplementation(
|
||||
() =>
|
||||
({
|
||||
...actualConfig,
|
||||
notifications: {
|
||||
...actualConfig.notifications,
|
||||
autoResumeOnRateLimit: true,
|
||||
},
|
||||
}) as never
|
||||
);
|
||||
|
||||
try {
|
||||
provisioningService.isTeamAlive.mockReturnValue(true);
|
||||
provisioningService.getCurrentLeadSessionId.mockReturnValue('sess-123');
|
||||
provisioningService.sendMessageToTeam.mockResolvedValue(undefined);
|
||||
service.getTeamData.mockResolvedValueOnce({
|
||||
teamName: 'my-team',
|
||||
config: { name: 'My Team' },
|
||||
tasks: [],
|
||||
members: [],
|
||||
messages: [
|
||||
{
|
||||
from: 'team-lead',
|
||||
text: "You've hit your limit. Resets in 5 minutes.",
|
||||
timestamp: '2026-04-17T12:00:00.000Z',
|
||||
read: true,
|
||||
source: 'lead_session' as const,
|
||||
messageId: 'persisted-rate-limit-1',
|
||||
leadSessionId: 'sess-123',
|
||||
},
|
||||
],
|
||||
kanbanState: { teamName: 'my-team', reviewers: [], tasks: {} },
|
||||
processes: [],
|
||||
});
|
||||
provisioningService.getLiveLeadProcessMessages.mockReturnValueOnce([
|
||||
{
|
||||
from: 'team-lead',
|
||||
text: "You've hit your limit. Resets in 5 minutes.",
|
||||
timestamp: '2026-04-17T12:00:02.000Z',
|
||||
read: true,
|
||||
source: 'lead_process' as const,
|
||||
messageId: 'live-rate-limit-1',
|
||||
leadSessionId: 'sess-123',
|
||||
},
|
||||
]);
|
||||
|
||||
const getDataHandler = handlers.get(TEAM_GET_DATA)!;
|
||||
const result = (await getDataHandler({} as never, 'my-team')) as {
|
||||
success: boolean;
|
||||
data: { messages: Array<{ source?: string; messageId?: string }> };
|
||||
};
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.data.messages).toEqual([
|
||||
expect.objectContaining({
|
||||
source: 'lead_session',
|
||||
messageId: 'persisted-rate-limit-1',
|
||||
}),
|
||||
]);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(4 * 60 * 1000 + 59 * 1000);
|
||||
expect(provisioningService.sendMessageToTeam).not.toHaveBeenCalled();
|
||||
|
||||
await vi.advanceTimersByTimeAsync(1100);
|
||||
expect(provisioningService.sendMessageToTeam).toHaveBeenCalledTimes(1);
|
||||
} finally {
|
||||
getConfigSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('merges early live messages before durable lead_session backfill exists', async () => {
|
||||
// Simulate: team just became readable but lead_session JSONL hasn't been written yet.
|
||||
// Only live in-memory messages exist from the provisioning process.
|
||||
|
|
@ -846,6 +928,357 @@ describe('ipc teams handlers', () => {
|
|||
expect(result.data.messages[1].text).toBe('Команда создана. Запускаю тиммейтов.');
|
||||
});
|
||||
|
||||
it('rebuilds only the remaining auto-resume delay from persisted rate-limit history', async () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date('2026-04-17T12:02:00.000Z'));
|
||||
const configManager = ConfigManager.getInstance();
|
||||
const actualConfig = configManager.getConfig();
|
||||
const getConfigSpy = vi.spyOn(configManager, 'getConfig').mockImplementation(
|
||||
() =>
|
||||
({
|
||||
...actualConfig,
|
||||
notifications: {
|
||||
...actualConfig.notifications,
|
||||
autoResumeOnRateLimit: true,
|
||||
},
|
||||
}) as never
|
||||
);
|
||||
|
||||
try {
|
||||
provisioningService.isTeamAlive.mockReturnValue(true);
|
||||
provisioningService.getCurrentLeadSessionId.mockReturnValue('sess-live');
|
||||
provisioningService.sendMessageToTeam.mockResolvedValue(undefined);
|
||||
service.getTeamData.mockResolvedValueOnce({
|
||||
teamName: 'my-team',
|
||||
config: { name: 'My Team' },
|
||||
tasks: [],
|
||||
members: [],
|
||||
messages: [
|
||||
{
|
||||
from: 'team-lead',
|
||||
text: "You've hit your limit. Resets in 5 minutes.",
|
||||
timestamp: '2026-04-17T12:00:00.000Z',
|
||||
read: true,
|
||||
source: 'lead_session' as const,
|
||||
leadSessionId: 'sess-live',
|
||||
messageId: 'rate-limit-1',
|
||||
},
|
||||
],
|
||||
kanbanState: { teamName: 'my-team', reviewers: [], tasks: {} },
|
||||
processes: [],
|
||||
});
|
||||
|
||||
const getDataHandler = handlers.get(TEAM_GET_DATA)!;
|
||||
const result = (await getDataHandler({} as never, 'my-team')) as {
|
||||
success: boolean;
|
||||
data: { messages: { source?: string; text: string }[] };
|
||||
};
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(3 * 60 * 1000 + 29 * 1000);
|
||||
expect(provisioningService.sendMessageToTeam).not.toHaveBeenCalled();
|
||||
|
||||
await vi.advanceTimersByTimeAsync(1100);
|
||||
expect(provisioningService.sendMessageToTeam).toHaveBeenCalledTimes(1);
|
||||
} finally {
|
||||
getConfigSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('can schedule auto-resume when the setting is enabled after an earlier history scan', async () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date('2026-04-17T12:02:00.000Z'));
|
||||
const configManager = ConfigManager.getInstance();
|
||||
const actualConfig = configManager.getConfig();
|
||||
let autoResumeEnabled = false;
|
||||
const getConfigSpy = vi.spyOn(configManager, 'getConfig').mockImplementation(
|
||||
() =>
|
||||
({
|
||||
...actualConfig,
|
||||
notifications: {
|
||||
...actualConfig.notifications,
|
||||
autoResumeOnRateLimit: autoResumeEnabled,
|
||||
},
|
||||
}) as never
|
||||
);
|
||||
|
||||
try {
|
||||
provisioningService.isTeamAlive.mockReturnValue(true);
|
||||
provisioningService.getCurrentLeadSessionId.mockReturnValue('sess-live');
|
||||
provisioningService.sendMessageToTeam.mockResolvedValue(undefined);
|
||||
service.getTeamData.mockResolvedValue({
|
||||
teamName: 'my-team',
|
||||
config: { name: 'My Team' },
|
||||
tasks: [],
|
||||
members: [],
|
||||
messages: [
|
||||
{
|
||||
from: 'team-lead',
|
||||
text: "You've hit your limit. Resets in 5 minutes.",
|
||||
timestamp: '2026-04-17T12:00:00.000Z',
|
||||
read: true,
|
||||
source: 'lead_session' as const,
|
||||
leadSessionId: 'sess-live',
|
||||
messageId: 'rate-limit-enable-later',
|
||||
},
|
||||
],
|
||||
kanbanState: { teamName: 'my-team', reviewers: [], tasks: {} },
|
||||
processes: [],
|
||||
});
|
||||
|
||||
const getDataHandler = handlers.get(TEAM_GET_DATA)!;
|
||||
|
||||
const firstResult = (await getDataHandler({} as never, 'my-team')) as {
|
||||
success: boolean;
|
||||
};
|
||||
expect(firstResult.success).toBe(true);
|
||||
expect(provisioningService.sendMessageToTeam).not.toHaveBeenCalled();
|
||||
|
||||
autoResumeEnabled = true;
|
||||
|
||||
const secondResult = (await getDataHandler({} as never, 'my-team')) as {
|
||||
success: boolean;
|
||||
};
|
||||
expect(secondResult.success).toBe(true);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(3 * 60 * 1000 + 29 * 1000);
|
||||
expect(provisioningService.sendMessageToTeam).not.toHaveBeenCalled();
|
||||
|
||||
await vi.advanceTimersByTimeAsync(1100);
|
||||
expect(provisioningService.sendMessageToTeam).toHaveBeenCalledTimes(1);
|
||||
} finally {
|
||||
getConfigSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('retries a previously over-ceiling history message once it becomes schedulable', async () => {
|
||||
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined);
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date('2026-04-17T00:00:00.000Z'));
|
||||
const configManager = ConfigManager.getInstance();
|
||||
const actualConfig = configManager.getConfig();
|
||||
const getConfigSpy = vi.spyOn(configManager, 'getConfig').mockImplementation(
|
||||
() =>
|
||||
({
|
||||
...actualConfig,
|
||||
notifications: {
|
||||
...actualConfig.notifications,
|
||||
autoResumeOnRateLimit: true,
|
||||
},
|
||||
}) as never
|
||||
);
|
||||
|
||||
try {
|
||||
provisioningService.isTeamAlive.mockReturnValue(true);
|
||||
provisioningService.getCurrentLeadSessionId.mockReturnValue('sess-live');
|
||||
provisioningService.sendMessageToTeam.mockResolvedValue(undefined);
|
||||
service.getTeamData.mockResolvedValue({
|
||||
teamName: 'my-team',
|
||||
config: { name: 'My Team' },
|
||||
tasks: [],
|
||||
members: [],
|
||||
messages: [
|
||||
{
|
||||
from: 'team-lead',
|
||||
text: "You've hit your limit. Resets at 12:20 UTC.",
|
||||
timestamp: '2026-04-17T00:00:00.000Z',
|
||||
read: true,
|
||||
source: 'lead_session' as const,
|
||||
leadSessionId: 'sess-live',
|
||||
messageId: 'rate-limit-over-ceiling',
|
||||
},
|
||||
],
|
||||
kanbanState: { teamName: 'my-team', reviewers: [], tasks: {} },
|
||||
processes: [],
|
||||
});
|
||||
|
||||
const getDataHandler = handlers.get(TEAM_GET_DATA)!;
|
||||
|
||||
const firstResult = (await getDataHandler({} as never, 'my-team')) as {
|
||||
success: boolean;
|
||||
};
|
||||
expect(firstResult.success).toBe(true);
|
||||
expect(provisioningService.sendMessageToTeam).not.toHaveBeenCalled();
|
||||
|
||||
vi.setSystemTime(new Date('2026-04-17T12:20:00.000Z'));
|
||||
|
||||
const secondResult = (await getDataHandler({} as never, 'my-team')) as {
|
||||
success: boolean;
|
||||
};
|
||||
expect(secondResult.success).toBe(true);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(29 * 1000);
|
||||
expect(provisioningService.sendMessageToTeam).not.toHaveBeenCalled();
|
||||
|
||||
await vi.advanceTimersByTimeAsync(1500);
|
||||
expect(provisioningService.sendMessageToTeam).toHaveBeenCalledTimes(1);
|
||||
} finally {
|
||||
warnSpy.mockRestore();
|
||||
getConfigSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('does not rebuild auto-resume from persisted history while the team is offline', async () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date('2026-04-17T12:02:00.000Z'));
|
||||
const configManager = ConfigManager.getInstance();
|
||||
const actualConfig = configManager.getConfig();
|
||||
const getConfigSpy = vi.spyOn(configManager, 'getConfig').mockImplementation(
|
||||
() =>
|
||||
({
|
||||
...actualConfig,
|
||||
notifications: {
|
||||
...actualConfig.notifications,
|
||||
autoResumeOnRateLimit: true,
|
||||
},
|
||||
}) as never
|
||||
);
|
||||
|
||||
try {
|
||||
provisioningService.isTeamAlive.mockReturnValue(false);
|
||||
provisioningService.sendMessageToTeam.mockResolvedValue(undefined);
|
||||
service.getTeamData.mockResolvedValue({
|
||||
teamName: 'my-team',
|
||||
config: { name: 'My Team' },
|
||||
tasks: [],
|
||||
members: [],
|
||||
messages: [
|
||||
{
|
||||
from: 'team-lead',
|
||||
text: "You've hit your limit. Resets in 5 minutes.",
|
||||
timestamp: '2026-04-17T12:00:00.000Z',
|
||||
read: true,
|
||||
source: 'lead_session' as const,
|
||||
messageId: 'rate-limit-offline-history',
|
||||
},
|
||||
],
|
||||
kanbanState: { teamName: 'my-team', reviewers: [], tasks: {} },
|
||||
processes: [],
|
||||
});
|
||||
|
||||
const getDataHandler = handlers.get(TEAM_GET_DATA)!;
|
||||
const result = (await getDataHandler({} as never, 'my-team')) as {
|
||||
success: boolean;
|
||||
};
|
||||
expect(result.success).toBe(true);
|
||||
|
||||
// Simulate the user manually starting a fresh run later; stale persisted history
|
||||
// should not have armed an auto-resume timer while the team was offline.
|
||||
provisioningService.isTeamAlive.mockReturnValue(true);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(3 * 60 * 1000 + 31 * 1000);
|
||||
expect(provisioningService.sendMessageToTeam).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
getConfigSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('does not rebuild auto-resume from an older lead session after the team was manually restarted', async () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date('2026-04-17T12:02:00.000Z'));
|
||||
const configManager = ConfigManager.getInstance();
|
||||
const actualConfig = configManager.getConfig();
|
||||
const getConfigSpy = vi.spyOn(configManager, 'getConfig').mockImplementation(
|
||||
() =>
|
||||
({
|
||||
...actualConfig,
|
||||
notifications: {
|
||||
...actualConfig.notifications,
|
||||
autoResumeOnRateLimit: true,
|
||||
},
|
||||
}) as never
|
||||
);
|
||||
|
||||
try {
|
||||
provisioningService.isTeamAlive.mockReturnValue(true);
|
||||
provisioningService.getCurrentLeadSessionId.mockReturnValue('sess-new');
|
||||
provisioningService.sendMessageToTeam.mockResolvedValue(undefined);
|
||||
service.getTeamData.mockResolvedValue({
|
||||
teamName: 'my-team',
|
||||
config: { name: 'My Team' },
|
||||
tasks: [],
|
||||
members: [],
|
||||
messages: [
|
||||
{
|
||||
from: 'team-lead',
|
||||
text: "You've hit your limit. Resets in 5 minutes.",
|
||||
timestamp: '2026-04-17T12:00:00.000Z',
|
||||
read: true,
|
||||
source: 'lead_session' as const,
|
||||
leadSessionId: 'sess-old',
|
||||
messageId: 'rate-limit-old-session',
|
||||
},
|
||||
],
|
||||
kanbanState: { teamName: 'my-team', reviewers: [], tasks: {} },
|
||||
processes: [],
|
||||
});
|
||||
|
||||
const getDataHandler = handlers.get(TEAM_GET_DATA)!;
|
||||
const result = (await getDataHandler({} as never, 'my-team')) as {
|
||||
success: boolean;
|
||||
};
|
||||
expect(result.success).toBe(true);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(3 * 60 * 1000 + 31 * 1000);
|
||||
expect(provisioningService.sendMessageToTeam).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
getConfigSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('does not arm lead auto-resume from a teammate inbox rate-limit message', async () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date('2026-04-17T12:02:00.000Z'));
|
||||
const configManager = ConfigManager.getInstance();
|
||||
const actualConfig = configManager.getConfig();
|
||||
const getConfigSpy = vi.spyOn(configManager, 'getConfig').mockImplementation(
|
||||
() =>
|
||||
({
|
||||
...actualConfig,
|
||||
notifications: {
|
||||
...actualConfig.notifications,
|
||||
autoResumeOnRateLimit: true,
|
||||
},
|
||||
}) as never
|
||||
);
|
||||
|
||||
try {
|
||||
provisioningService.isTeamAlive.mockReturnValue(true);
|
||||
provisioningService.getCurrentLeadSessionId.mockReturnValue('sess-live');
|
||||
provisioningService.sendMessageToTeam.mockResolvedValue(undefined);
|
||||
service.getTeamData.mockResolvedValue({
|
||||
teamName: 'my-team',
|
||||
config: { name: 'My Team' },
|
||||
tasks: [],
|
||||
members: [],
|
||||
messages: [
|
||||
{
|
||||
from: 'alice',
|
||||
to: 'team-lead',
|
||||
text: "You've hit your limit. Resets in 5 minutes.",
|
||||
timestamp: '2026-04-17T12:00:00.000Z',
|
||||
read: false,
|
||||
messageId: 'member-rate-limit-1',
|
||||
},
|
||||
],
|
||||
kanbanState: { teamName: 'my-team', reviewers: [], tasks: {} },
|
||||
processes: [],
|
||||
});
|
||||
|
||||
const getDataHandler = handlers.get(TEAM_GET_DATA)!;
|
||||
const result = (await getDataHandler({} as never, 'my-team')) as {
|
||||
success: boolean;
|
||||
};
|
||||
expect(result.success).toBe(true);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(3 * 60 * 1000 + 31 * 1000);
|
||||
expect(provisioningService.sendMessageToTeam).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
getConfigSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('keeps TEAM_GET_DATA read-only and never triggers reconcile side effects', async () => {
|
||||
const getDataHandler = handlers.get(TEAM_GET_DATA)!;
|
||||
const result = (await getDataHandler({} as never, 'my-team')) as {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,46 @@
|
|||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
describe('ConfigManager notification config shape', () => {
|
||||
let overrideRoot: string | null = null;
|
||||
|
||||
afterEach(async () => {
|
||||
if (overrideRoot) {
|
||||
fs.rmSync(overrideRoot, { recursive: true, force: true });
|
||||
overrideRoot = null;
|
||||
}
|
||||
vi.resetModules();
|
||||
const pathDecoder = await import('../../../../src/main/utils/pathDecoder');
|
||||
pathDecoder.setClaudeBasePathOverride(null);
|
||||
});
|
||||
|
||||
it('strips unknown notification keys while keeping autoResumeOnRateLimit', async () => {
|
||||
vi.resetModules();
|
||||
|
||||
overrideRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'config-notifications-'));
|
||||
const pathDecoder = await import('../../../../src/main/utils/pathDecoder');
|
||||
pathDecoder.setClaudeBasePathOverride(overrideRoot);
|
||||
|
||||
fs.writeFileSync(
|
||||
path.join(overrideRoot, 'claude-devtools-config.json'),
|
||||
JSON.stringify({
|
||||
notifications: {
|
||||
notifyOnInboxMessages: true,
|
||||
autoResumeOnRateLimit: true,
|
||||
notifyOnTeamLaunched: false,
|
||||
},
|
||||
})
|
||||
);
|
||||
|
||||
const { configManager } = await import(
|
||||
'../../../../src/main/services/infrastructure/ConfigManager'
|
||||
);
|
||||
const config = configManager.getConfig();
|
||||
|
||||
expect(config.notifications.autoResumeOnRateLimit).toBe(true);
|
||||
expect(config.notifications.notifyOnTeamLaunched).toBe(false);
|
||||
expect('notifyOnInboxMessages' in config.notifications).toBe(false);
|
||||
});
|
||||
});
|
||||
313
test/main/services/team/AutoResumeService.test.ts
Normal file
313
test/main/services/team/AutoResumeService.test.ts
Normal file
|
|
@ -0,0 +1,313 @@
|
|||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { AutoResumeService } from '../../../../src/main/services/team/AutoResumeService';
|
||||
|
||||
import type { ConfigManager } from '../../../../src/main/services/infrastructure/ConfigManager';
|
||||
|
||||
const TEAM = 'test-team';
|
||||
const RATE_LIMIT_MSG = "You've hit your limit. Resets in 5 minutes.";
|
||||
|
||||
describe('AutoResumeService', () => {
|
||||
const mockConfig = { autoResumeOnRateLimit: false };
|
||||
const configManagerMock = {
|
||||
getConfig: vi.fn(() => ({
|
||||
notifications: {
|
||||
autoResumeOnRateLimit: mockConfig.autoResumeOnRateLimit,
|
||||
},
|
||||
})),
|
||||
};
|
||||
const configManager = configManagerMock as unknown as Pick<ConfigManager, 'getConfig'>;
|
||||
const provisioningService = {
|
||||
getCurrentRunId: vi.fn<(teamName: string) => string | null>(),
|
||||
isTeamAlive: vi.fn<(teamName: string) => boolean>(),
|
||||
sendMessageToTeam: vi.fn<(teamName: string, text: string) => Promise<void>>(),
|
||||
};
|
||||
|
||||
let service: AutoResumeService;
|
||||
|
||||
beforeEach(() => {
|
||||
mockConfig.autoResumeOnRateLimit = false;
|
||||
provisioningService.getCurrentRunId.mockReset();
|
||||
provisioningService.isTeamAlive.mockReset();
|
||||
provisioningService.sendMessageToTeam.mockReset();
|
||||
configManagerMock.getConfig.mockClear();
|
||||
provisioningService.getCurrentRunId.mockReturnValue('run-1');
|
||||
service = new AutoResumeService(provisioningService, configManager);
|
||||
vi.useFakeTimers();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
service.clearAllPendingAutoResume();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it('does nothing when the feature flag is off', () => {
|
||||
const now = new Date('2026-04-17T12:00:00Z');
|
||||
|
||||
service.handleRateLimitMessage(TEAM, RATE_LIMIT_MSG, now);
|
||||
|
||||
vi.advanceTimersByTime(24 * 60 * 60 * 1000);
|
||||
expect(provisioningService.sendMessageToTeam).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not schedule when the reset time is unparseable', () => {
|
||||
mockConfig.autoResumeOnRateLimit = true;
|
||||
const now = new Date('2026-04-17T12:00:00Z');
|
||||
|
||||
service.handleRateLimitMessage(TEAM, "You've hit your limit.", now);
|
||||
|
||||
vi.advanceTimersByTime(24 * 60 * 60 * 1000);
|
||||
expect(provisioningService.sendMessageToTeam).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('reschedules when a later rate-limit message changes the reset time', async () => {
|
||||
mockConfig.autoResumeOnRateLimit = true;
|
||||
provisioningService.isTeamAlive.mockReturnValue(true);
|
||||
provisioningService.sendMessageToTeam.mockResolvedValue(undefined);
|
||||
const now = new Date('2026-04-17T12:00:00Z');
|
||||
|
||||
service.handleRateLimitMessage(TEAM, `You've hit your limit. Resets in 1 minute.`, now);
|
||||
service.handleRateLimitMessage(TEAM, `You've hit your limit. Resets in 10 minutes.`, now);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(2 * 60 * 1000);
|
||||
expect(provisioningService.sendMessageToTeam).not.toHaveBeenCalled();
|
||||
|
||||
await vi.advanceTimersByTimeAsync(8 * 60 * 1000 + 30 * 1000 + 100);
|
||||
expect(provisioningService.sendMessageToTeam).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('ignores an older rate-limit message when a newer timer is already pending', async () => {
|
||||
mockConfig.autoResumeOnRateLimit = true;
|
||||
provisioningService.isTeamAlive.mockReturnValue(true);
|
||||
provisioningService.sendMessageToTeam.mockResolvedValue(undefined);
|
||||
|
||||
const observedAt = new Date('2026-04-17T12:01:30Z');
|
||||
const newerMessageAt = new Date('2026-04-17T12:01:00Z');
|
||||
const olderMessageAt = new Date('2026-04-17T12:00:00Z');
|
||||
|
||||
service.handleRateLimitMessage(
|
||||
TEAM,
|
||||
`You've hit your limit. Resets in 10 minutes.`,
|
||||
observedAt,
|
||||
newerMessageAt
|
||||
);
|
||||
service.handleRateLimitMessage(
|
||||
TEAM,
|
||||
`You've hit your limit. Resets in 15 minutes.`,
|
||||
observedAt,
|
||||
olderMessageAt
|
||||
);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(9 * 60 * 1000 + 59 * 1000);
|
||||
expect(provisioningService.sendMessageToTeam).not.toHaveBeenCalled();
|
||||
|
||||
await vi.advanceTimersByTimeAsync(1200);
|
||||
expect(provisioningService.sendMessageToTeam).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('keeps only one timer when the same reset time is reported again', async () => {
|
||||
mockConfig.autoResumeOnRateLimit = true;
|
||||
provisioningService.isTeamAlive.mockReturnValue(true);
|
||||
provisioningService.sendMessageToTeam.mockResolvedValue(undefined);
|
||||
const now = new Date('2026-04-17T12:00:00Z');
|
||||
|
||||
service.handleRateLimitMessage(TEAM, RATE_LIMIT_MSG, now);
|
||||
service.handleRateLimitMessage(TEAM, RATE_LIMIT_MSG, now);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(5 * 60 * 1000 + 30 * 1000 + 100);
|
||||
expect(provisioningService.sendMessageToTeam).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('clears a stale pending timer when a newer reset exceeds the ceiling', async () => {
|
||||
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined);
|
||||
|
||||
mockConfig.autoResumeOnRateLimit = true;
|
||||
provisioningService.isTeamAlive.mockReturnValue(true);
|
||||
provisioningService.sendMessageToTeam.mockResolvedValue(undefined);
|
||||
const now = new Date('2026-04-17T16:00:00Z');
|
||||
|
||||
service.handleRateLimitMessage(TEAM, `You've hit your limit. Resets in 5 minutes.`, now);
|
||||
service.handleRateLimitMessage(TEAM, `You've hit your limit. Resets at 15:00 UTC.`, now);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(10 * 60 * 1000);
|
||||
expect(provisioningService.sendMessageToTeam).not.toHaveBeenCalled();
|
||||
expect(warnSpy).toHaveBeenCalledWith(
|
||||
expect.any(String),
|
||||
expect.stringContaining('exceeds ceiling')
|
||||
);
|
||||
warnSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('reconstructs the remaining delay from a persisted rate-limit message timestamp', async () => {
|
||||
mockConfig.autoResumeOnRateLimit = true;
|
||||
provisioningService.isTeamAlive.mockReturnValue(true);
|
||||
provisioningService.sendMessageToTeam.mockResolvedValue(undefined);
|
||||
|
||||
const observedAt = new Date('2026-04-17T12:02:00Z');
|
||||
const messageAt = new Date('2026-04-17T12:00:00Z');
|
||||
|
||||
service.handleRateLimitMessage(TEAM, RATE_LIMIT_MSG, observedAt, messageAt);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(3 * 60 * 1000 + 29 * 1000);
|
||||
expect(provisioningService.sendMessageToTeam).not.toHaveBeenCalled();
|
||||
|
||||
await vi.advanceTimersByTimeAsync(1100);
|
||||
expect(provisioningService.sendMessageToTeam).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('uses only the remaining buffer when the reset already happened shortly before replay', async () => {
|
||||
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined);
|
||||
mockConfig.autoResumeOnRateLimit = true;
|
||||
provisioningService.isTeamAlive.mockReturnValue(true);
|
||||
provisioningService.sendMessageToTeam.mockResolvedValue(undefined);
|
||||
|
||||
const observedAt = new Date('2026-04-17T12:05:20Z');
|
||||
const messageAt = new Date('2026-04-17T12:00:00Z');
|
||||
|
||||
service.handleRateLimitMessage(TEAM, RATE_LIMIT_MSG, observedAt, messageAt);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(9 * 1000);
|
||||
expect(provisioningService.sendMessageToTeam).not.toHaveBeenCalled();
|
||||
|
||||
await vi.advanceTimersByTimeAsync(1500);
|
||||
expect(provisioningService.sendMessageToTeam).toHaveBeenCalledTimes(1);
|
||||
warnSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('skips stale persisted history once the parsed reset is materially in the past', async () => {
|
||||
mockConfig.autoResumeOnRateLimit = true;
|
||||
provisioningService.isTeamAlive.mockReturnValue(true);
|
||||
provisioningService.sendMessageToTeam.mockResolvedValue(undefined);
|
||||
|
||||
const observedAt = new Date('2026-04-17T12:05:40Z');
|
||||
const messageAt = new Date('2026-04-17T11:00:00Z');
|
||||
|
||||
service.handleRateLimitMessage(TEAM, RATE_LIMIT_MSG, observedAt, messageAt);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(10 * 60 * 1000);
|
||||
expect(provisioningService.sendMessageToTeam).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('skips replay after the buffered fire deadline already passed', async () => {
|
||||
mockConfig.autoResumeOnRateLimit = true;
|
||||
provisioningService.isTeamAlive.mockReturnValue(true);
|
||||
provisioningService.sendMessageToTeam.mockResolvedValue(undefined);
|
||||
|
||||
const observedAt = new Date('2026-04-17T12:05:40Z');
|
||||
const messageAt = new Date('2026-04-17T12:00:00Z');
|
||||
|
||||
service.handleRateLimitMessage(TEAM, RATE_LIMIT_MSG, observedAt, messageAt);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(60 * 1000);
|
||||
expect(provisioningService.sendMessageToTeam).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('sends the resume nudge when the team is alive at fire time', async () => {
|
||||
mockConfig.autoResumeOnRateLimit = true;
|
||||
provisioningService.isTeamAlive.mockReturnValue(true);
|
||||
provisioningService.sendMessageToTeam.mockResolvedValue(undefined);
|
||||
const now = new Date('2026-04-17T12:00:00Z');
|
||||
|
||||
service.handleRateLimitMessage(TEAM, RATE_LIMIT_MSG, now);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(5 * 60 * 1000 + 30 * 1000 + 100);
|
||||
|
||||
expect(provisioningService.isTeamAlive).toHaveBeenCalledWith(TEAM);
|
||||
expect(provisioningService.sendMessageToTeam).toHaveBeenCalledTimes(1);
|
||||
expect(provisioningService.sendMessageToTeam.mock.calls[0]![0]).toBe(TEAM);
|
||||
expect(provisioningService.sendMessageToTeam.mock.calls[0]![1]).toMatch(
|
||||
/Your rate limit has reset/
|
||||
);
|
||||
});
|
||||
|
||||
it('skips the nudge when the team is no longer alive at fire time', async () => {
|
||||
mockConfig.autoResumeOnRateLimit = true;
|
||||
provisioningService.isTeamAlive.mockReturnValue(false);
|
||||
const now = new Date('2026-04-17T12:00:00Z');
|
||||
|
||||
service.handleRateLimitMessage(TEAM, RATE_LIMIT_MSG, now);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(5 * 60 * 1000 + 30 * 1000 + 100);
|
||||
|
||||
expect(provisioningService.isTeamAlive).toHaveBeenCalledWith(TEAM);
|
||||
expect(provisioningService.sendMessageToTeam).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('skips the nudge when the team has moved to a newer run before fire time', async () => {
|
||||
mockConfig.autoResumeOnRateLimit = true;
|
||||
provisioningService.getCurrentRunId.mockReturnValue('run-1');
|
||||
provisioningService.isTeamAlive.mockReturnValue(true);
|
||||
provisioningService.sendMessageToTeam.mockResolvedValue(undefined);
|
||||
const now = new Date('2026-04-17T12:00:00Z');
|
||||
|
||||
service.handleRateLimitMessage(TEAM, RATE_LIMIT_MSG, now);
|
||||
provisioningService.getCurrentRunId.mockReturnValue('run-2');
|
||||
|
||||
await vi.advanceTimersByTimeAsync(5 * 60 * 1000 + 30 * 1000 + 100);
|
||||
|
||||
expect(provisioningService.isTeamAlive).toHaveBeenCalledWith(TEAM);
|
||||
expect(provisioningService.getCurrentRunId).toHaveBeenLastCalledWith(TEAM);
|
||||
expect(provisioningService.sendMessageToTeam).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('re-checks the config flag at fire time and aborts when toggled off', async () => {
|
||||
mockConfig.autoResumeOnRateLimit = true;
|
||||
provisioningService.isTeamAlive.mockReturnValue(true);
|
||||
const now = new Date('2026-04-17T12:00:00Z');
|
||||
|
||||
service.handleRateLimitMessage(TEAM, RATE_LIMIT_MSG, now);
|
||||
mockConfig.autoResumeOnRateLimit = false;
|
||||
|
||||
await vi.advanceTimersByTimeAsync(5 * 60 * 1000 + 30 * 1000 + 100);
|
||||
|
||||
expect(provisioningService.sendMessageToTeam).not.toHaveBeenCalled();
|
||||
expect(provisioningService.isTeamAlive).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('swallows errors from sendMessageToTeam without crashing', async () => {
|
||||
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined);
|
||||
|
||||
mockConfig.autoResumeOnRateLimit = true;
|
||||
provisioningService.isTeamAlive.mockReturnValue(true);
|
||||
provisioningService.sendMessageToTeam.mockRejectedValue(new Error('stdin closed'));
|
||||
const now = new Date('2026-04-17T12:00:00Z');
|
||||
|
||||
service.handleRateLimitMessage(TEAM, RATE_LIMIT_MSG, now);
|
||||
|
||||
await expect(
|
||||
vi.advanceTimersByTimeAsync(5 * 60 * 1000 + 30 * 1000 + 100)
|
||||
).resolves.not.toThrow();
|
||||
expect(provisioningService.sendMessageToTeam).toHaveBeenCalledTimes(1);
|
||||
expect(errorSpy).toHaveBeenCalledWith(
|
||||
expect.any(String),
|
||||
expect.stringContaining('Failed to send resume nudge')
|
||||
);
|
||||
errorSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('clears a pending timer so the nudge never fires', async () => {
|
||||
mockConfig.autoResumeOnRateLimit = true;
|
||||
provisioningService.isTeamAlive.mockReturnValue(true);
|
||||
const now = new Date('2026-04-17T12:00:00Z');
|
||||
|
||||
service.handleRateLimitMessage(TEAM, RATE_LIMIT_MSG, now);
|
||||
service.cancelPendingAutoResume(TEAM);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(10 * 60 * 1000);
|
||||
expect(provisioningService.sendMessageToTeam).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('cancels every pending timer across teams', async () => {
|
||||
mockConfig.autoResumeOnRateLimit = true;
|
||||
provisioningService.isTeamAlive.mockReturnValue(true);
|
||||
const now = new Date('2026-04-17T12:00:00Z');
|
||||
|
||||
service.handleRateLimitMessage('team-a', RATE_LIMIT_MSG, now);
|
||||
service.handleRateLimitMessage('team-b', `You've hit your limit. Resets in 10 minutes.`, now);
|
||||
|
||||
service.clearAllPendingAutoResume();
|
||||
|
||||
await vi.advanceTimersByTimeAsync(15 * 60 * 1000);
|
||||
expect(provisioningService.sendMessageToTeam).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
|
@ -51,6 +51,11 @@ vi.mock('@main/utils/pathDecoder', async (importOriginal) => {
|
|||
});
|
||||
|
||||
import { TeamProvisioningService } from '@main/services/team/TeamProvisioningService';
|
||||
import {
|
||||
clearAutoResumeService,
|
||||
getAutoResumeService,
|
||||
initializeAutoResumeService,
|
||||
} from '@main/services/team/AutoResumeService';
|
||||
import { createPersistedLaunchSnapshot } from '@main/services/team/TeamLaunchStateEvaluator';
|
||||
import { getTeamLaunchStatePath } from '@main/services/team/TeamLaunchStateStore';
|
||||
import { ClaudeBinaryResolver } from '@main/services/team/ClaudeBinaryResolver';
|
||||
|
|
@ -160,6 +165,7 @@ describe('TeamProvisioningService', () => {
|
|||
});
|
||||
|
||||
afterEach(() => {
|
||||
clearAutoResumeService();
|
||||
vi.useRealTimers();
|
||||
try {
|
||||
fs.rmSync(tempClaudeRoot, { recursive: true, force: true });
|
||||
|
|
@ -674,6 +680,141 @@ describe('TeamProvisioningService', () => {
|
|||
expect(launchArgs).toContain(leadSessionId);
|
||||
});
|
||||
|
||||
it('seeds the current lead session id immediately when launch resumes an existing session', async () => {
|
||||
allowConsoleLogs();
|
||||
const teamName = 'resume-seed-session-team';
|
||||
const leadSessionId = 'lead-session-seeded';
|
||||
writeLaunchConfig(teamName, tempClaudeRoot, leadSessionId, ['alice']);
|
||||
|
||||
vi.mocked(ClaudeBinaryResolver.resolve).mockResolvedValue('/mock/claude');
|
||||
const child = createRunningChild();
|
||||
vi.mocked(spawnCli).mockReturnValue(child as any);
|
||||
|
||||
const svc = new TeamProvisioningService(undefined, undefined, undefined, undefined, {
|
||||
writeConfigFile: vi.fn(async () => '/mock/mcp-config-launch.json'),
|
||||
removeConfigFile: vi.fn(async () => {}),
|
||||
} as any);
|
||||
(svc as any).buildProvisioningEnv = vi.fn(async () => ({
|
||||
env: { ANTHROPIC_API_KEY: 'test' },
|
||||
authSource: 'anthropic_api_key',
|
||||
}));
|
||||
(svc as any).resolveLaunchExpectedMembers = vi.fn(async () => ({
|
||||
members: [{ name: 'alice' }],
|
||||
source: 'members-meta',
|
||||
warning: undefined,
|
||||
}));
|
||||
(svc as any).normalizeTeamConfigForLaunch = vi.fn(async () => {});
|
||||
(svc as any).assertConfigLeadOnlyForLaunch = vi.fn(async () => {});
|
||||
(svc as any).updateConfigProjectPath = vi.fn(async () => {});
|
||||
(svc as any).restorePrelaunchConfig = vi.fn(async () => {});
|
||||
(svc as any).validateAgentTeamsMcpRuntime = vi.fn(async () => {});
|
||||
(svc as any).persistLaunchStateSnapshot = vi.fn(async () => {});
|
||||
(svc as any).startFilesystemMonitor = vi.fn();
|
||||
(svc as any).pathExists = vi.fn(async (targetPath: string) =>
|
||||
targetPath.endsWith(`${leadSessionId}.jsonl`)
|
||||
);
|
||||
|
||||
const { runId } = await svc.launchTeam({ teamName, cwd: tempClaudeRoot }, () => {});
|
||||
|
||||
expect(svc.getCurrentLeadSessionId(teamName)).toBe(leadSessionId);
|
||||
|
||||
await svc.cancelProvisioning(runId);
|
||||
});
|
||||
|
||||
it('clears stale team-scoped transient state before starting a new launch run', async () => {
|
||||
allowConsoleLogs();
|
||||
vi.useFakeTimers();
|
||||
|
||||
const teamName = 'launch-clears-stale-runtime-state';
|
||||
const leadSessionId = 'lead-session-stale-state';
|
||||
writeLaunchConfig(teamName, tempClaudeRoot, leadSessionId, ['alice']);
|
||||
|
||||
vi.mocked(ClaudeBinaryResolver.resolve).mockResolvedValue('/mock/claude');
|
||||
vi.mocked(spawnCli).mockImplementation(() => {
|
||||
throw new Error('launch spawn EINVAL');
|
||||
});
|
||||
|
||||
const svc = new TeamProvisioningService(undefined, undefined, undefined, undefined, {
|
||||
writeConfigFile: vi.fn(async () => '/mock/mcp-config-launch.json'),
|
||||
removeConfigFile: vi.fn(async () => {}),
|
||||
} as any);
|
||||
(svc as any).buildProvisioningEnv = vi.fn(async () => ({
|
||||
env: { ANTHROPIC_API_KEY: 'test' },
|
||||
authSource: 'anthropic_api_key',
|
||||
}));
|
||||
(svc as any).resolveLaunchExpectedMembers = vi.fn(async () => ({
|
||||
members: [{ name: 'alice' }],
|
||||
source: 'members-meta',
|
||||
warning: undefined,
|
||||
}));
|
||||
(svc as any).normalizeTeamConfigForLaunch = vi.fn(async () => {});
|
||||
(svc as any).assertConfigLeadOnlyForLaunch = vi.fn(async () => {});
|
||||
(svc as any).updateConfigProjectPath = vi.fn(async () => {});
|
||||
(svc as any).restorePrelaunchConfig = vi.fn(async () => {});
|
||||
(svc as any).validateAgentTeamsMcpRuntime = vi.fn(async () => {});
|
||||
(svc as any).pathExists = vi.fn(async (targetPath: string) =>
|
||||
targetPath.endsWith(`${leadSessionId}.jsonl`)
|
||||
);
|
||||
|
||||
const autoResumeProvisioning = {
|
||||
getCurrentRunId: vi.fn(() => 'run-1' as string | null),
|
||||
isTeamAlive: vi.fn(() => true),
|
||||
sendMessageToTeam: vi.fn(async () => undefined),
|
||||
};
|
||||
initializeAutoResumeService(autoResumeProvisioning);
|
||||
|
||||
const configManagerModule = await import('@main/services/infrastructure/ConfigManager');
|
||||
const configManager = configManagerModule.ConfigManager.getInstance();
|
||||
const actualConfig = configManager.getConfig();
|
||||
const getConfigSpy = vi.spyOn(configManager, 'getConfig').mockImplementation(
|
||||
() =>
|
||||
({
|
||||
...actualConfig,
|
||||
notifications: {
|
||||
...actualConfig.notifications,
|
||||
autoResumeOnRateLimit: true,
|
||||
},
|
||||
}) as never
|
||||
);
|
||||
|
||||
try {
|
||||
getAutoResumeService().handleRateLimitMessage(
|
||||
teamName,
|
||||
"You've hit your limit. Resets in 5 minutes.",
|
||||
new Date('2026-04-17T12:00:00.000Z')
|
||||
);
|
||||
|
||||
(svc as any).relayedLeadInboxMessageIds.set(teamName, new Set(['stale-msg']));
|
||||
(svc as any).liveLeadProcessMessages.set(teamName, [
|
||||
{
|
||||
from: 'team-lead',
|
||||
text: 'Old transient message',
|
||||
timestamp: '2026-04-17T12:00:00.000Z',
|
||||
read: true,
|
||||
source: 'lead_process',
|
||||
messageId: 'lead-turn-old-run-1',
|
||||
},
|
||||
]);
|
||||
(svc as any).pendingTimeouts.set(
|
||||
`same-team-deferred:${teamName}`,
|
||||
setTimeout(() => undefined, 60_000)
|
||||
);
|
||||
|
||||
await expect(svc.launchTeam({ teamName, cwd: tempClaudeRoot }, () => {})).rejects.toThrow(
|
||||
'launch spawn EINVAL'
|
||||
);
|
||||
|
||||
expect((svc as any).relayedLeadInboxMessageIds.has(teamName)).toBe(false);
|
||||
expect((svc as any).liveLeadProcessMessages.has(teamName)).toBe(false);
|
||||
expect((svc as any).pendingTimeouts.has(`same-team-deferred:${teamName}`)).toBe(false);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(5 * 60 * 1000 + 30 * 1000 + 100);
|
||||
expect(autoResumeProvisioning.sendMessageToTeam).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
getConfigSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('marks persisted bootstrap as failed when member transcript shows an unsupported model error', async () => {
|
||||
allowConsoleLogs();
|
||||
const teamName = 'zz-unit-bootstrap-unsupported-model';
|
||||
|
|
|
|||
|
|
@ -113,6 +113,12 @@ vi.mock('agent-teams-controller', () => ({
|
|||
}));
|
||||
|
||||
import type { TeamChangeEvent } from '@shared/types/team';
|
||||
import { ConfigManager } from '../../../../src/main/services/infrastructure/ConfigManager';
|
||||
import {
|
||||
clearAutoResumeService,
|
||||
getAutoResumeService,
|
||||
initializeAutoResumeService,
|
||||
} from '../../../../src/main/services/team/AutoResumeService';
|
||||
import { TeamProvisioningService } from '../../../../src/main/services/team/TeamProvisioningService';
|
||||
|
||||
function seedConfig(teamName: string): void {
|
||||
|
|
@ -133,6 +139,7 @@ interface RunLike {
|
|||
runId: string;
|
||||
teamName: string;
|
||||
provisioningComplete: boolean;
|
||||
detectedSessionId?: string | null;
|
||||
leadMsgSeq: number;
|
||||
pendingToolCalls: { name: string; preview: string }[];
|
||||
activeToolCalls: Map<string, unknown>;
|
||||
|
|
@ -150,6 +157,8 @@ interface RunLike {
|
|||
request: { members: { name: string; role?: string }[] };
|
||||
activeCrossTeamReplyHints?: Array<{ toTeam: string; conversationId: string }>;
|
||||
pendingInboxRelayCandidates?: unknown[];
|
||||
memberSpawnStatuses: Map<string, unknown>;
|
||||
pendingApprovals: Map<string, unknown>;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -159,13 +168,14 @@ interface RunLike {
|
|||
function attachRun(
|
||||
service: TeamProvisioningService,
|
||||
teamName: string,
|
||||
opts?: { provisioningComplete?: boolean }
|
||||
opts?: { provisioningComplete?: boolean; runId?: string; detectedSessionId?: string | null }
|
||||
): RunLike {
|
||||
const runId = 'run-1';
|
||||
const runId = opts?.runId ?? 'run-1';
|
||||
const run: RunLike = {
|
||||
runId,
|
||||
teamName,
|
||||
provisioningComplete: opts?.provisioningComplete ?? false,
|
||||
detectedSessionId: opts?.detectedSessionId ?? null,
|
||||
leadMsgSeq: 0,
|
||||
pendingToolCalls: [],
|
||||
activeToolCalls: new Map(),
|
||||
|
|
@ -180,6 +190,8 @@ function attachRun(
|
|||
provisioningOutputParts: [],
|
||||
request: { members: [{ name: 'team-lead', role: 'Team Lead' }] },
|
||||
activeCrossTeamReplyHints: [],
|
||||
memberSpawnStatuses: new Map(),
|
||||
pendingApprovals: new Map(),
|
||||
};
|
||||
|
||||
(service as unknown as { aliveRunByTeam: Map<string, string> }).aliveRunByTeam.set(
|
||||
|
|
@ -227,6 +239,64 @@ describe('TeamProvisioningService pre-ready live messages', () => {
|
|||
expect(run.provisioningOutputParts).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('attaches leadSessionId to a live message when the same assistant payload carries session_id', () => {
|
||||
const service = new TeamProvisioningService();
|
||||
seedConfig('my-team');
|
||||
const run = attachRun(service, 'my-team', { provisioningComplete: false });
|
||||
|
||||
callHandleStreamJsonMessage(service, run, {
|
||||
type: 'assistant',
|
||||
session_id: 'sess-123',
|
||||
content: [{ type: 'text', text: 'Команда создана. Запускаю всех тиммейтов параллельно.' }],
|
||||
});
|
||||
|
||||
const live = service.getLiveLeadProcessMessages('my-team');
|
||||
expect(live).toHaveLength(1);
|
||||
expect(live[0].leadSessionId).toBe('sess-123');
|
||||
});
|
||||
|
||||
it('makes leadSessionId visible to synchronous lead-message listeners in the same turn', () => {
|
||||
const service = new TeamProvisioningService();
|
||||
seedConfig('my-team');
|
||||
const seenSessionIds: Array<string | undefined> = [];
|
||||
service.setTeamChangeEmitter((event) => {
|
||||
if (event.type === 'lead-message') {
|
||||
seenSessionIds.push(service.getLiveLeadProcessMessages('my-team')[0]?.leadSessionId);
|
||||
}
|
||||
});
|
||||
const run = attachRun(service, 'my-team', { provisioningComplete: false });
|
||||
|
||||
callHandleStreamJsonMessage(service, run, {
|
||||
type: 'assistant',
|
||||
session_id: 'sess-sync',
|
||||
content: [{ type: 'text', text: 'Команда создана. Запускаю всех тиммейтов параллельно.' }],
|
||||
});
|
||||
|
||||
expect(seenSessionIds).toEqual(['sess-sync']);
|
||||
});
|
||||
|
||||
it('retrofits leadSessionId onto earlier live messages after session detection', () => {
|
||||
const service = new TeamProvisioningService();
|
||||
seedConfig('my-team');
|
||||
const run = attachRun(service, 'my-team', { provisioningComplete: false });
|
||||
|
||||
callHandleStreamJsonMessage(service, run, {
|
||||
type: 'assistant',
|
||||
content: [{ type: 'text', text: 'Команда создана. Запускаю всех тиммейтов параллельно.' }],
|
||||
});
|
||||
expect(service.getLiveLeadProcessMessages('my-team')[0]?.leadSessionId).toBeUndefined();
|
||||
|
||||
callHandleStreamJsonMessage(service, run, {
|
||||
type: 'assistant',
|
||||
session_id: 'sess-456',
|
||||
content: [],
|
||||
});
|
||||
|
||||
const live = service.getLiveLeadProcessMessages('my-team');
|
||||
expect(live).toHaveLength(1);
|
||||
expect(live[0].leadSessionId).toBe('sess-456');
|
||||
});
|
||||
|
||||
it('emits lead-message event type (not inbox)', () => {
|
||||
const service = new TeamProvisioningService();
|
||||
seedConfig('my-team');
|
||||
|
|
@ -547,6 +617,82 @@ describe('TeamProvisioningService pre-ready live messages', () => {
|
|||
expect(hoisted.appendSentMessage).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('ignores stale cross-team send completions from an older run after a new run starts', async () => {
|
||||
const service = new TeamProvisioningService();
|
||||
seedConfig('my-team');
|
||||
|
||||
let resolveSend: ((value: { deliveredToInbox: boolean; messageId: string }) => void) | null =
|
||||
null;
|
||||
const crossTeamSender = vi.fn(
|
||||
() =>
|
||||
new Promise<{ deliveredToInbox: boolean; messageId: string }>((resolve) => {
|
||||
resolveSend = resolve;
|
||||
})
|
||||
);
|
||||
service.setCrossTeamSender(crossTeamSender);
|
||||
|
||||
const oldRun = attachRun(service, 'my-team', {
|
||||
provisioningComplete: true,
|
||||
runId: 'run-old',
|
||||
detectedSessionId: 'sess-old',
|
||||
});
|
||||
oldRun.activeCrossTeamReplyHints = [{ toTeam: 'team-best', conversationId: 'conv-old' }];
|
||||
|
||||
callHandleStreamJsonMessage(service, oldRun, {
|
||||
type: 'assistant',
|
||||
content: [
|
||||
{
|
||||
type: 'tool_use',
|
||||
name: 'SendMessage',
|
||||
input: {
|
||||
type: 'message',
|
||||
recipient: 'team-best.user',
|
||||
content: 'Old run cross-team reply.',
|
||||
summary: 'Old run reply',
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
await vi.waitFor(() => {
|
||||
expect(crossTeamSender).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
const newRun = attachRun(service, 'my-team', {
|
||||
provisioningComplete: true,
|
||||
runId: 'run-new',
|
||||
detectedSessionId: 'sess-new',
|
||||
});
|
||||
service.pushLiveLeadProcessMessage('my-team', {
|
||||
from: 'team-lead',
|
||||
text: 'Current run is active.',
|
||||
timestamp: '2026-04-17T12:00:10.000Z',
|
||||
read: true,
|
||||
source: 'lead_process',
|
||||
messageId: 'lead-turn-run-new-1',
|
||||
leadSessionId: 'sess-new',
|
||||
});
|
||||
|
||||
expect(resolveSend).not.toBeNull();
|
||||
const finishSend = resolveSend as unknown as ((
|
||||
value: { deliveredToInbox: boolean; messageId: string }
|
||||
) => void);
|
||||
finishSend({ deliveredToInbox: true, messageId: 'cross-stale-old-run' });
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
|
||||
expect(service.getLiveLeadProcessMessages('my-team')).toEqual([
|
||||
expect.objectContaining({
|
||||
text: 'Current run is active.',
|
||||
messageId: 'lead-turn-run-new-1',
|
||||
leadSessionId: 'sess-new',
|
||||
}),
|
||||
]);
|
||||
|
||||
(service as unknown as { cleanupRun: (runLike: unknown) => void }).cleanupRun(oldRun);
|
||||
(service as unknown as { cleanupRun: (runLike: unknown) => void }).cleanupRun(newRun);
|
||||
});
|
||||
|
||||
it('upgrades pseudo cross-team recipients into cross-team sends', async () => {
|
||||
const service = new TeamProvisioningService();
|
||||
seedConfig('my-team');
|
||||
|
|
@ -964,3 +1110,238 @@ describe('TeamProvisioningService pre-ready live messages', () => {
|
|||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('TeamProvisioningService auto-resume cleanup', () => {
|
||||
beforeEach(() => {
|
||||
hoisted.files.clear();
|
||||
hoisted.appendSentMessage.mockClear();
|
||||
hoisted.sendInboxMessage.mockClear();
|
||||
clearAutoResumeService();
|
||||
vi.useFakeTimers();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
clearAutoResumeService();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it('cancels pending auto-resume timers when a run is cleaned up', async () => {
|
||||
const service = new TeamProvisioningService();
|
||||
seedConfig('my-team');
|
||||
const run = attachRun(service, 'my-team', { provisioningComplete: true });
|
||||
|
||||
const autoResumeProvisioning = {
|
||||
getCurrentRunId: vi.fn(() => 'run-1' as string | null),
|
||||
isTeamAlive: vi.fn(() => true),
|
||||
sendMessageToTeam: vi.fn(async () => undefined),
|
||||
};
|
||||
initializeAutoResumeService(autoResumeProvisioning);
|
||||
|
||||
const configManager = ConfigManager.getInstance();
|
||||
const actualConfig = configManager.getConfig();
|
||||
const getConfigSpy = vi.spyOn(configManager, 'getConfig').mockImplementation(
|
||||
() =>
|
||||
({
|
||||
...actualConfig,
|
||||
notifications: {
|
||||
...actualConfig.notifications,
|
||||
autoResumeOnRateLimit: true,
|
||||
},
|
||||
}) as never
|
||||
);
|
||||
|
||||
try {
|
||||
getAutoResumeService().handleRateLimitMessage(
|
||||
'my-team',
|
||||
"You've hit your limit. Resets in 5 minutes.",
|
||||
new Date('2026-04-17T12:00:00.000Z')
|
||||
);
|
||||
|
||||
(service as unknown as { cleanupRun: (runLike: unknown) => void }).cleanupRun(run);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(5 * 60 * 1000 + 30 * 1000 + 100);
|
||||
expect(autoResumeProvisioning.sendMessageToTeam).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
getConfigSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('does not let stale cleanup from an older run cancel the current run state', async () => {
|
||||
const service = new TeamProvisioningService();
|
||||
seedConfig('my-team');
|
||||
const oldRun = attachRun(service, 'my-team', {
|
||||
provisioningComplete: true,
|
||||
runId: 'run-old',
|
||||
detectedSessionId: 'sess-old',
|
||||
});
|
||||
const newRun = attachRun(service, 'my-team', {
|
||||
provisioningComplete: true,
|
||||
runId: 'run-new',
|
||||
detectedSessionId: 'sess-new',
|
||||
});
|
||||
|
||||
const autoResumeProvisioning = {
|
||||
getCurrentRunId: vi.fn(() => 'run-1' as string | null),
|
||||
isTeamAlive: vi.fn(() => true),
|
||||
sendMessageToTeam: vi.fn(async () => undefined),
|
||||
};
|
||||
initializeAutoResumeService(autoResumeProvisioning);
|
||||
|
||||
const configManager = ConfigManager.getInstance();
|
||||
const actualConfig = configManager.getConfig();
|
||||
const getConfigSpy = vi.spyOn(configManager, 'getConfig').mockImplementation(
|
||||
() =>
|
||||
({
|
||||
...actualConfig,
|
||||
notifications: {
|
||||
...actualConfig.notifications,
|
||||
autoResumeOnRateLimit: true,
|
||||
},
|
||||
}) as never
|
||||
);
|
||||
|
||||
try {
|
||||
getAutoResumeService().handleRateLimitMessage(
|
||||
'my-team',
|
||||
"You've hit your limit. Resets in 5 minutes.",
|
||||
new Date('2026-04-17T12:00:00.000Z')
|
||||
);
|
||||
|
||||
service.pushLiveLeadProcessMessage('my-team', {
|
||||
from: 'team-lead',
|
||||
text: 'Current run is active.',
|
||||
timestamp: '2026-04-17T12:00:01.000Z',
|
||||
read: true,
|
||||
source: 'lead_process',
|
||||
messageId: 'live-new-run',
|
||||
});
|
||||
expect(service.getLiveLeadProcessMessages('my-team')).toHaveLength(1);
|
||||
|
||||
(service as unknown as { cleanupRun: (runLike: unknown) => void }).cleanupRun(oldRun);
|
||||
|
||||
expect(service.getLiveLeadProcessMessages('my-team')).toHaveLength(1);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(5 * 60 * 1000 + 30 * 1000 + 100);
|
||||
expect(autoResumeProvisioning.sendMessageToTeam).toHaveBeenCalledTimes(1);
|
||||
expect(autoResumeProvisioning.sendMessageToTeam).toHaveBeenCalledWith(
|
||||
'my-team',
|
||||
expect.stringContaining('rate limit has reset')
|
||||
);
|
||||
} finally {
|
||||
getConfigSpy.mockRestore();
|
||||
(service as unknown as { cleanupRun: (runLike: unknown) => void }).cleanupRun(newRun);
|
||||
}
|
||||
});
|
||||
|
||||
it('removes stale live lead messages from an older run while preserving the current run', () => {
|
||||
const service = new TeamProvisioningService();
|
||||
seedConfig('my-team');
|
||||
const oldRun = attachRun(service, 'my-team', {
|
||||
provisioningComplete: true,
|
||||
runId: 'run-old',
|
||||
detectedSessionId: 'sess-old',
|
||||
});
|
||||
|
||||
service.pushLiveLeadProcessMessage('my-team', {
|
||||
from: 'team-lead',
|
||||
text: "You've hit your limit. Resets in 5 minutes.",
|
||||
timestamp: '2026-04-17T12:00:00.000Z',
|
||||
read: true,
|
||||
source: 'lead_process',
|
||||
messageId: 'lead-turn-run-old-1',
|
||||
leadSessionId: 'sess-old',
|
||||
});
|
||||
|
||||
const newRun = attachRun(service, 'my-team', {
|
||||
provisioningComplete: true,
|
||||
runId: 'run-new',
|
||||
detectedSessionId: 'sess-new',
|
||||
});
|
||||
|
||||
service.pushLiveLeadProcessMessage('my-team', {
|
||||
from: 'team-lead',
|
||||
text: 'Current run is active.',
|
||||
timestamp: '2026-04-17T12:00:10.000Z',
|
||||
read: true,
|
||||
source: 'lead_process',
|
||||
messageId: 'lead-turn-run-new-1',
|
||||
leadSessionId: 'sess-new',
|
||||
});
|
||||
|
||||
expect(service.getLiveLeadProcessMessages('my-team')).toHaveLength(2);
|
||||
|
||||
(service as unknown as { cleanupRun: (runLike: unknown) => void }).cleanupRun(oldRun);
|
||||
|
||||
expect(service.getLiveLeadProcessMessages('my-team')).toEqual([
|
||||
expect.objectContaining({
|
||||
text: 'Current run is active.',
|
||||
messageId: 'lead-turn-run-new-1',
|
||||
leadSessionId: 'sess-new',
|
||||
}),
|
||||
]);
|
||||
|
||||
(service as unknown as { cleanupRun: (runLike: unknown) => void }).cleanupRun(newRun);
|
||||
});
|
||||
|
||||
it('preserves the canonical assistant timestamp for live rate-limit messages', async () => {
|
||||
vi.setSystemTime(new Date('2026-04-17T12:00:20.000Z'));
|
||||
|
||||
const service = new TeamProvisioningService();
|
||||
seedConfig('my-team');
|
||||
const run = attachRun(service, 'my-team', {
|
||||
provisioningComplete: true,
|
||||
detectedSessionId: 'sess-live',
|
||||
});
|
||||
|
||||
const autoResumeProvisioning = {
|
||||
getCurrentRunId: vi.fn(() => 'run-1' as string | null),
|
||||
isTeamAlive: vi.fn(() => true),
|
||||
sendMessageToTeam: vi.fn(async () => undefined),
|
||||
};
|
||||
initializeAutoResumeService(autoResumeProvisioning);
|
||||
|
||||
const configManager = ConfigManager.getInstance();
|
||||
const actualConfig = configManager.getConfig();
|
||||
const getConfigSpy = vi.spyOn(configManager, 'getConfig').mockImplementation(
|
||||
() =>
|
||||
({
|
||||
...actualConfig,
|
||||
notifications: {
|
||||
...actualConfig.notifications,
|
||||
autoResumeOnRateLimit: true,
|
||||
},
|
||||
}) as never
|
||||
);
|
||||
|
||||
try {
|
||||
callHandleStreamJsonMessage(service, run, {
|
||||
type: 'assistant',
|
||||
timestamp: '2026-04-17T12:00:00.000Z',
|
||||
content: [{ type: 'text', text: "You've hit your limit. Resets in 5 minutes." }],
|
||||
});
|
||||
|
||||
const live = service.getLiveLeadProcessMessages('my-team');
|
||||
expect(live).toHaveLength(1);
|
||||
expect(live[0].timestamp).toBe('2026-04-17T12:00:00.000Z');
|
||||
|
||||
getAutoResumeService().handleRateLimitMessage(
|
||||
'my-team',
|
||||
live[0].text,
|
||||
new Date('2026-04-17T12:00:20.000Z'),
|
||||
new Date(live[0].timestamp)
|
||||
);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(5 * 60 * 1000 + 9 * 1000);
|
||||
expect(autoResumeProvisioning.sendMessageToTeam).not.toHaveBeenCalled();
|
||||
|
||||
await vi.advanceTimersByTimeAsync(1500);
|
||||
expect(autoResumeProvisioning.sendMessageToTeam).toHaveBeenCalledTimes(1);
|
||||
expect(autoResumeProvisioning.sendMessageToTeam).toHaveBeenCalledWith(
|
||||
'my-team',
|
||||
expect.stringContaining('rate limit has reset')
|
||||
);
|
||||
} finally {
|
||||
getConfigSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -151,12 +151,26 @@ function seedMemberInbox(teamName: string, memberName: string, messages: unknown
|
|||
hoisted.files.set(`/mock/teams/${teamName}/inboxes/${memberName}.json`, JSON.stringify(messages));
|
||||
}
|
||||
|
||||
function createDeferred<T>(): {
|
||||
promise: Promise<T>;
|
||||
resolve: (value: T) => void;
|
||||
reject: (error: unknown) => void;
|
||||
} {
|
||||
let resolve!: (value: T) => void;
|
||||
let reject!: (error: unknown) => void;
|
||||
const promise = new Promise<T>((res, rej) => {
|
||||
resolve = res;
|
||||
reject = rej;
|
||||
});
|
||||
return { promise, resolve, reject };
|
||||
}
|
||||
|
||||
function attachAliveRun(
|
||||
service: TeamProvisioningService,
|
||||
teamName: string,
|
||||
opts?: { writable?: boolean }
|
||||
): { writeSpy: ReturnType<typeof vi.fn> } {
|
||||
const runId = 'run-1';
|
||||
opts?: { writable?: boolean; runId?: string; provisioningComplete?: boolean }
|
||||
): { writeSpy: ReturnType<typeof vi.fn>; runId: string } {
|
||||
const runId = opts?.runId ?? 'run-1';
|
||||
const writeSpy = vi.fn((_data: unknown, cb?: (err?: Error | null) => void) => {
|
||||
if (typeof cb === 'function') cb(null);
|
||||
return true;
|
||||
|
|
@ -174,6 +188,7 @@ function attachAliveRun(
|
|||
teamName,
|
||||
members: [{ name: 'team-lead', role: 'team-lead' }],
|
||||
},
|
||||
startedAt: '2026-02-23T09:59:00.000Z',
|
||||
leadMsgSeq: 0,
|
||||
pendingToolCalls: [],
|
||||
activeToolCalls: new Map(),
|
||||
|
|
@ -181,6 +196,8 @@ function attachAliveRun(
|
|||
lastLeadTextEmitMs: 0,
|
||||
activeCrossTeamReplyHints: [],
|
||||
pendingInboxRelayCandidates: [],
|
||||
pendingApprovals: new Map(),
|
||||
processedPermissionRequestIds: new Set(),
|
||||
silentUserDmForward: null,
|
||||
silentUserDmForwardClearHandle: null,
|
||||
child: {
|
||||
|
|
@ -191,11 +208,11 @@ function attachAliveRun(
|
|||
},
|
||||
processKilled: false,
|
||||
cancelRequested: false,
|
||||
provisioningComplete: true,
|
||||
provisioningComplete: opts?.provisioningComplete ?? true,
|
||||
leadRelayCapture: null,
|
||||
});
|
||||
|
||||
return { writeSpy };
|
||||
return { writeSpy, runId };
|
||||
}
|
||||
|
||||
async function waitForCapture(service: TeamProvisioningService): Promise<any> {
|
||||
|
|
@ -435,6 +452,111 @@ describe('TeamProvisioningService relayLeadInboxMessages', () => {
|
|||
expect(writeSpy).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('does not let stale lead inbox relay work write into a newer run', async () => {
|
||||
const service = new TeamProvisioningService();
|
||||
const teamName = 'my-team';
|
||||
const inboxMessages = [
|
||||
{
|
||||
from: 'bob',
|
||||
text: 'Please pick this up.',
|
||||
timestamp: '2026-02-23T10:00:00.000Z',
|
||||
read: false,
|
||||
messageId: 'm-stale-lead-1',
|
||||
},
|
||||
];
|
||||
seedConfig(teamName);
|
||||
seedLeadInbox(teamName, inboxMessages);
|
||||
|
||||
const { writeSpy: oldWriteSpy, runId: oldRunId } = attachAliveRun(service, teamName, {
|
||||
runId: 'run-old',
|
||||
});
|
||||
const inboxDeferred = createDeferred<typeof inboxMessages>();
|
||||
const inboxReader = (service as unknown as {
|
||||
inboxReader: { getMessagesFor: (team: string, member: string) => Promise<typeof inboxMessages> };
|
||||
}).inboxReader;
|
||||
const inboxSpy = vi
|
||||
.spyOn(inboxReader, 'getMessagesFor')
|
||||
.mockImplementationOnce(async () => await inboxDeferred.promise)
|
||||
.mockImplementation(async () => inboxMessages);
|
||||
|
||||
const relayPromise = service.relayLeadInboxMessages(teamName);
|
||||
await Promise.resolve();
|
||||
|
||||
const oldRun = (service as unknown as { runs: Map<string, any> }).runs.get(oldRunId);
|
||||
oldRun.processKilled = true;
|
||||
oldRun.cancelRequested = true;
|
||||
oldRun.child.stdin.writable = false;
|
||||
|
||||
const { writeSpy: newWriteSpy } = attachAliveRun(service, teamName, { runId: 'run-new' });
|
||||
inboxDeferred.resolve(inboxMessages);
|
||||
|
||||
await expect(relayPromise).resolves.toBe(0);
|
||||
expect(oldWriteSpy).not.toHaveBeenCalled();
|
||||
expect(newWriteSpy).not.toHaveBeenCalled();
|
||||
inboxSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('does not let stale lead relay consume a newer run permission_request', async () => {
|
||||
const service = new TeamProvisioningService();
|
||||
const teamName = 'my-team';
|
||||
const permissionMessage = {
|
||||
from: 'alice',
|
||||
text: JSON.stringify({
|
||||
type: 'permission_request',
|
||||
request_id: 'perm-new-run-1',
|
||||
agent_id: 'alice',
|
||||
tool_name: 'Bash',
|
||||
input: { command: 'git status' },
|
||||
}),
|
||||
timestamp: '2026-02-23T10:00:30.000Z',
|
||||
read: false,
|
||||
messageId: 'perm-inbox-1',
|
||||
};
|
||||
seedConfig(teamName);
|
||||
seedLeadInbox(teamName, [permissionMessage]);
|
||||
|
||||
const { runId: oldRunId } = attachAliveRun(service, teamName, { runId: 'run-old' });
|
||||
const inboxDeferred = createDeferred<[typeof permissionMessage]>();
|
||||
const inboxReader = (service as unknown as {
|
||||
inboxReader: {
|
||||
getMessagesFor: (
|
||||
team: string,
|
||||
member: string
|
||||
) => Promise<[typeof permissionMessage]>;
|
||||
};
|
||||
}).inboxReader;
|
||||
const inboxSpy = vi
|
||||
.spyOn(inboxReader, 'getMessagesFor')
|
||||
.mockImplementationOnce(async () => await inboxDeferred.promise)
|
||||
.mockImplementation(async () => [permissionMessage]);
|
||||
|
||||
const relayPromise = service.relayLeadInboxMessages(teamName);
|
||||
await Promise.resolve();
|
||||
|
||||
const oldRun = (service as unknown as { runs: Map<string, any> }).runs.get(oldRunId);
|
||||
oldRun.processKilled = true;
|
||||
oldRun.cancelRequested = true;
|
||||
oldRun.child.stdin.writable = false;
|
||||
|
||||
attachAliveRun(service, teamName, { runId: 'run-new' });
|
||||
inboxDeferred.resolve([permissionMessage]);
|
||||
|
||||
await expect(relayPromise).resolves.toBe(0);
|
||||
|
||||
const inbox = JSON.parse(
|
||||
hoisted.files.get(`/mock/teams/${teamName}/inboxes/team-lead.json`) ?? '[]'
|
||||
) as Array<{ messageId?: string; read?: boolean }>;
|
||||
expect(inbox).toEqual([
|
||||
expect.objectContaining({
|
||||
messageId: 'perm-inbox-1',
|
||||
read: false,
|
||||
}),
|
||||
]);
|
||||
expect(oldRun.pendingApprovals.size).toBe(0);
|
||||
expect(oldRun.processedPermissionRequestIds.size).toBe(0);
|
||||
inboxSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('relays legacy lead inbox rows with generated messageId', async () => {
|
||||
const service = new TeamProvisioningService();
|
||||
const teamName = 'my-team';
|
||||
|
|
@ -910,6 +1032,50 @@ describe('TeamProvisioningService relayLeadInboxMessages', () => {
|
|||
expect(payload).toContain('Please review my changes');
|
||||
});
|
||||
|
||||
it('does not let stale member inbox relay work write into a newer run', async () => {
|
||||
const service = new TeamProvisioningService();
|
||||
const teamName = 'my-team';
|
||||
const inboxMessages = [
|
||||
{
|
||||
from: 'user',
|
||||
text: 'Please sync with Alice.',
|
||||
timestamp: '2026-02-23T10:00:00.000Z',
|
||||
read: false,
|
||||
messageId: 'm-stale-member-1',
|
||||
},
|
||||
];
|
||||
seedConfig(teamName);
|
||||
seedMemberInbox(teamName, 'alice', inboxMessages);
|
||||
|
||||
const { writeSpy: oldWriteSpy, runId: oldRunId } = attachAliveRun(service, teamName, {
|
||||
runId: 'run-old',
|
||||
});
|
||||
const inboxDeferred = createDeferred<typeof inboxMessages>();
|
||||
const inboxReader = (service as unknown as {
|
||||
inboxReader: { getMessagesFor: (team: string, member: string) => Promise<typeof inboxMessages> };
|
||||
}).inboxReader;
|
||||
const inboxSpy = vi
|
||||
.spyOn(inboxReader, 'getMessagesFor')
|
||||
.mockImplementationOnce(async () => await inboxDeferred.promise)
|
||||
.mockImplementation(async () => inboxMessages);
|
||||
|
||||
const relayPromise = service.relayMemberInboxMessages(teamName, 'alice');
|
||||
await Promise.resolve();
|
||||
|
||||
const oldRun = (service as unknown as { runs: Map<string, any> }).runs.get(oldRunId);
|
||||
oldRun.processKilled = true;
|
||||
oldRun.cancelRequested = true;
|
||||
oldRun.child.stdin.writable = false;
|
||||
|
||||
const { writeSpy: newWriteSpy } = attachAliveRun(service, teamName, { runId: 'run-new' });
|
||||
inboxDeferred.resolve(inboxMessages);
|
||||
|
||||
await expect(relayPromise).resolves.toBe(0);
|
||||
expect(oldWriteSpy).not.toHaveBeenCalled();
|
||||
expect(newWriteSpy).not.toHaveBeenCalled();
|
||||
inboxSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('marks pure member heartbeat idle as read without relaying it', async () => {
|
||||
const service = new TeamProvisioningService();
|
||||
const teamName = 'my-team';
|
||||
|
|
|
|||
291
test/shared/utils/rateLimitDetector.test.ts
Normal file
291
test/shared/utils/rateLimitDetector.test.ts
Normal file
|
|
@ -0,0 +1,291 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import {
|
||||
isRateLimitMessage,
|
||||
parseRateLimitResetTime,
|
||||
} from '../../../src/shared/utils/rateLimitDetector';
|
||||
|
||||
// Helper: every production rate-limit message starts with this substring.
|
||||
// Prefix test inputs so they clear the parser's rate-limit-context gate.
|
||||
const RL = "You've hit your limit. ";
|
||||
|
||||
describe('isRateLimitMessage', () => {
|
||||
it('detects the canonical substring', () => {
|
||||
expect(isRateLimitMessage("You've hit your limit")).toBe(true);
|
||||
expect(
|
||||
isRateLimitMessage("You've hit your limit. Your limit will reset at 3pm (PST).")
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it('returns false for unrelated text', () => {
|
||||
expect(isRateLimitMessage('All good here')).toBe(false);
|
||||
expect(isRateLimitMessage('hit the limit')).toBe(false); // missing "You've"
|
||||
expect(isRateLimitMessage('')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('parseRateLimitResetTime', () => {
|
||||
// ---------------------------------------------------------------------
|
||||
// Rate-limit context gate
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
it('returns null for text that is not a rate-limit message', () => {
|
||||
// Even if the text contains a parseable "reset at X" clause, the parser
|
||||
// must refuse to interpret it when the rate-limit context is absent.
|
||||
// Protects against false positives like "reset at 3pm (PST)" appearing
|
||||
// in unrelated prose.
|
||||
const now = new Date('2026-04-17T12:00:00Z');
|
||||
expect(
|
||||
parseRateLimitResetTime('Please reset your expectations at 3pm (PST).', now)
|
||||
).toBeNull();
|
||||
expect(parseRateLimitResetTime('Resets in 2 hours.', now)).toBeNull();
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Relative durations
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
it('parses "resets in N hours"', () => {
|
||||
const now = new Date('2026-04-17T12:00:00Z');
|
||||
const result = parseRateLimitResetTime(`${RL}Resets in 2 hours.`, now);
|
||||
expect(result?.toISOString()).toBe('2026-04-17T14:00:00.000Z');
|
||||
});
|
||||
|
||||
it('parses "resets in N minutes"', () => {
|
||||
const now = new Date('2026-04-17T12:00:00Z');
|
||||
const result = parseRateLimitResetTime(`${RL}Will reset in 45 minutes.`, now);
|
||||
expect(result?.toISOString()).toBe('2026-04-17T12:45:00.000Z');
|
||||
});
|
||||
|
||||
it('parses "resets in N seconds"', () => {
|
||||
const now = new Date('2026-04-17T12:00:00Z');
|
||||
const result = parseRateLimitResetTime(`${RL}Resets in 90 seconds.`, now);
|
||||
expect(result?.toISOString()).toBe('2026-04-17T12:01:30.000Z');
|
||||
});
|
||||
|
||||
it('parses "hrs" and "mins" abbreviations', () => {
|
||||
const now = new Date('2026-04-17T12:00:00Z');
|
||||
expect(
|
||||
parseRateLimitResetTime(`${RL}Resets in 3 hrs.`, now)?.toISOString()
|
||||
).toBe('2026-04-17T15:00:00.000Z');
|
||||
expect(
|
||||
parseRateLimitResetTime(`${RL}Resets in 15 mins.`, now)?.toISOString()
|
||||
).toBe('2026-04-17T12:15:00.000Z');
|
||||
});
|
||||
|
||||
it('parses bare "h" / "m" / "s" single-letter units', () => {
|
||||
const now = new Date('2026-04-17T12:00:00Z');
|
||||
expect(parseRateLimitResetTime(`${RL}Resets in 2 h.`, now)?.toISOString()).toBe(
|
||||
'2026-04-17T14:00:00.000Z'
|
||||
);
|
||||
expect(parseRateLimitResetTime(`${RL}Resets in 30 m.`, now)?.toISOString()).toBe(
|
||||
'2026-04-17T12:30:00.000Z'
|
||||
);
|
||||
expect(parseRateLimitResetTime(`${RL}Resets in 45 s.`, now)?.toISOString()).toBe(
|
||||
'2026-04-17T12:00:45.000Z'
|
||||
);
|
||||
});
|
||||
|
||||
it('parses "resets in about 30 minutes" with filler words', () => {
|
||||
const now = new Date('2026-04-17T12:00:00Z');
|
||||
const result = parseRateLimitResetTime(
|
||||
`${RL}Your limit will reset in about 30 minutes.`,
|
||||
now
|
||||
);
|
||||
expect(result?.toISOString()).toBe('2026-04-17T12:30:00.000Z');
|
||||
});
|
||||
|
||||
it('parses "around" and "~" filler variants', () => {
|
||||
const now = new Date('2026-04-17T12:00:00Z');
|
||||
expect(
|
||||
parseRateLimitResetTime(`${RL}Your limit will reset in around 30 minutes.`, now)?.toISOString()
|
||||
).toBe('2026-04-17T12:30:00.000Z');
|
||||
expect(
|
||||
parseRateLimitResetTime(`${RL}Your limit will reset in ~ 45 seconds.`, now)?.toISOString()
|
||||
).toBe('2026-04-17T12:00:45.000Z');
|
||||
});
|
||||
|
||||
it('parses fractional hours', () => {
|
||||
const now = new Date('2026-04-17T12:00:00Z');
|
||||
const result = parseRateLimitResetTime(`${RL}Resets in 1.5 hours.`, now);
|
||||
expect(result?.toISOString()).toBe('2026-04-17T13:30:00.000Z');
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Absolute clock times with timezone
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
it('parses "resets at 3pm (PST)"', () => {
|
||||
// 3pm PST = 23:00 UTC (PST = UTC-8)
|
||||
const now = new Date('2026-04-17T12:00:00Z'); // earlier than 23:00 UTC
|
||||
const result = parseRateLimitResetTime(
|
||||
`${RL}Your limit will reset at 3pm (PST).`,
|
||||
now
|
||||
);
|
||||
expect(result?.toISOString()).toBe('2026-04-17T23:00:00.000Z');
|
||||
});
|
||||
|
||||
it('parses "resets at 3:30 pm (PST)"', () => {
|
||||
const now = new Date('2026-04-17T12:00:00Z');
|
||||
const result = parseRateLimitResetTime(
|
||||
`${RL}Your limit will reset at 3:30 pm (PST).`,
|
||||
now
|
||||
);
|
||||
expect(result?.toISOString()).toBe('2026-04-17T23:30:00.000Z');
|
||||
});
|
||||
|
||||
it('parses 24-hour time with UTC', () => {
|
||||
const now = new Date('2026-04-17T12:00:00Z');
|
||||
const result = parseRateLimitResetTime(
|
||||
`${RL}Your limit will reset at 15:30 UTC.`,
|
||||
now
|
||||
);
|
||||
expect(result?.toISOString()).toBe('2026-04-17T15:30:00.000Z');
|
||||
});
|
||||
|
||||
it('parses bare timezone abbreviation without parentheses', () => {
|
||||
// Regex group 5 path: "3pm PST" (no parens) should parse same as "(PST)".
|
||||
const now = new Date('2026-04-17T12:00:00Z');
|
||||
const result = parseRateLimitResetTime(
|
||||
`${RL}Your limit will reset at 3pm PST.`,
|
||||
now
|
||||
);
|
||||
expect(result?.toISOString()).toBe('2026-04-17T23:00:00.000Z');
|
||||
});
|
||||
|
||||
it('parses non-PST North American timezones', () => {
|
||||
// Cover each zone in the whitelist — regression guard against map typos.
|
||||
const now = new Date('2026-04-17T02:00:00Z');
|
||||
// 3am EST = UTC-5 → 08:00 UTC
|
||||
expect(
|
||||
parseRateLimitResetTime(`${RL}Resets at 3am (EST).`, now)?.toISOString()
|
||||
).toBe('2026-04-17T08:00:00.000Z');
|
||||
// 3am EDT = UTC-4 → 07:00 UTC
|
||||
expect(
|
||||
parseRateLimitResetTime(`${RL}Resets at 3am (EDT).`, now)?.toISOString()
|
||||
).toBe('2026-04-17T07:00:00.000Z');
|
||||
// 3am CST = UTC-6 → 09:00 UTC
|
||||
expect(
|
||||
parseRateLimitResetTime(`${RL}Resets at 3am (CST).`, now)?.toISOString()
|
||||
).toBe('2026-04-17T09:00:00.000Z');
|
||||
// 3am MDT = UTC-6 → 09:00 UTC
|
||||
expect(
|
||||
parseRateLimitResetTime(`${RL}Resets at 3am (MDT).`, now)?.toISOString()
|
||||
).toBe('2026-04-17T09:00:00.000Z');
|
||||
});
|
||||
|
||||
it('rolls forward to tomorrow when the time has already passed today', () => {
|
||||
// 3pm PST = 23:00 UTC; if "now" is 23:30 UTC, the parsed 23:00 should
|
||||
// roll to tomorrow rather than return a time in the past.
|
||||
const now = new Date('2026-04-17T23:30:00Z');
|
||||
const result = parseRateLimitResetTime(`${RL}Resets at 3pm (PST).`, now);
|
||||
expect(result?.toISOString()).toBe('2026-04-18T23:00:00.000Z');
|
||||
});
|
||||
|
||||
it('does NOT roll forward for near-present timestamps (within the 1-minute tolerance)', () => {
|
||||
// Parsed time is 20s in the past (stale message / clock skew). A full
|
||||
// 24h rollover here would trip the scheduler's 12h ceiling and silently
|
||||
// drop auto-resume. Instead, the parser returns the near-past time and
|
||||
// lets the scheduler's buffer + Math.max(0, ...) clamp take over.
|
||||
const now = new Date('2026-04-17T23:00:20Z');
|
||||
const result = parseRateLimitResetTime(`${RL}Resets at 3pm (PST).`, now);
|
||||
// 3pm PST = 23:00 UTC (today) — stays in the past, not rolled.
|
||||
expect(result?.toISOString()).toBe('2026-04-17T23:00:00.000Z');
|
||||
});
|
||||
|
||||
it('resolves the zone-local calendar date when UTC and zone disagree on the day', () => {
|
||||
// now = 2026-04-18T01:00:00Z which is still 2026-04-17 17:00 PST.
|
||||
// "8pm (PST)" on that PST day = 2026-04-17T20:00 PST = 2026-04-18T04:00Z.
|
||||
// A naive UTC-anchored build would emit 2026-04-19T04:00Z (24h off).
|
||||
const now = new Date('2026-04-18T01:00:00Z');
|
||||
const result = parseRateLimitResetTime(`${RL}Resets at 8pm (PST).`, now);
|
||||
expect(result?.toISOString()).toBe('2026-04-18T04:00:00.000Z');
|
||||
});
|
||||
|
||||
it('handles the mirror case for positive offsets crossing the UTC day', () => {
|
||||
// 02:00 UTC today is already in the past vs 23:00 UTC → roll to tomorrow.
|
||||
const now = new Date('2026-04-17T23:00:00Z');
|
||||
const result = parseRateLimitResetTime(`${RL}Resets at 02:00 UTC.`, now);
|
||||
expect(result?.toISOString()).toBe('2026-04-18T02:00:00.000Z');
|
||||
});
|
||||
|
||||
it('handles 12am (midnight) correctly', () => {
|
||||
const now = new Date('2026-04-17T12:00:00Z');
|
||||
const result = parseRateLimitResetTime(`${RL}Resets at 12am UTC.`, now);
|
||||
// Same day midnight is already in the past relative to noon; rolls to next day.
|
||||
expect(result?.toISOString()).toBe('2026-04-18T00:00:00.000Z');
|
||||
});
|
||||
|
||||
it('handles 12pm (noon) correctly', () => {
|
||||
const now = new Date('2026-04-17T06:00:00Z');
|
||||
const result = parseRateLimitResetTime(`${RL}Resets at 12pm UTC.`, now);
|
||||
expect(result?.toISOString()).toBe('2026-04-17T12:00:00.000Z');
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Day-shift qualifiers — should bail out rather than guess today/tomorrow
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
it('returns null when the reset is qualified with "next week"', () => {
|
||||
const now = new Date('2026-04-17T12:00:00Z');
|
||||
expect(
|
||||
parseRateLimitResetTime(`${RL}Reset at 3pm (PST) next week.`, now)
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
it('returns null when the reset is qualified with "tomorrow"', () => {
|
||||
const now = new Date('2026-04-17T12:00:00Z');
|
||||
expect(
|
||||
parseRateLimitResetTime(`${RL}Reset at 9am UTC tomorrow.`, now)
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
it('returns null when the reset is qualified with a day of week', () => {
|
||||
const now = new Date('2026-04-17T12:00:00Z');
|
||||
expect(
|
||||
parseRateLimitResetTime(`${RL}Reset at 3pm (PST) on Tuesday.`, now)
|
||||
).toBeNull();
|
||||
expect(
|
||||
parseRateLimitResetTime(`${RL}Reset at 9am UTC on Mon.`, now)
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Unparseable / ambiguous cases
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
it('returns null when no reset time is present', () => {
|
||||
const now = new Date('2026-04-17T12:00:00Z');
|
||||
expect(parseRateLimitResetTime("You've hit your limit.", now)).toBeNull();
|
||||
expect(parseRateLimitResetTime('', now)).toBeNull();
|
||||
});
|
||||
|
||||
it('returns null for unknown parenthesized timezone abbreviations', () => {
|
||||
// Parenthesized TZ is authoritative — unknown means "sender meant a
|
||||
// specific zone we don't model"; bail out rather than guess.
|
||||
const now = new Date('2026-04-17T12:00:00Z');
|
||||
expect(parseRateLimitResetTime(`${RL}Resets at 3pm (CEST).`, now)).toBeNull();
|
||||
});
|
||||
|
||||
it('falls back to local time when a trailing word looks like a TZ but is not one', () => {
|
||||
// "3pm today" used to capture "TODAY" as an unknown TZ and suppress
|
||||
// the whole message. Now the parser ignores the bare token and treats
|
||||
// "3pm" as user-local. Assert a parse happens (non-null result) rather
|
||||
// than pinning the UTC value, since local time depends on the runner.
|
||||
const now = new Date('2026-04-17T06:00:00Z');
|
||||
const result = parseRateLimitResetTime(`${RL}Reset at 3pm today.`, now);
|
||||
expect(result).not.toBeNull();
|
||||
});
|
||||
|
||||
it('returns null for invalid clock values', () => {
|
||||
const now = new Date('2026-04-17T12:00:00Z');
|
||||
expect(parseRateLimitResetTime(`${RL}Resets at 25:00 UTC.`, now)).toBeNull();
|
||||
expect(parseRateLimitResetTime(`${RL}Resets at 10:99 UTC.`, now)).toBeNull();
|
||||
});
|
||||
|
||||
it('returns null for negative relative durations', () => {
|
||||
const now = new Date('2026-04-17T12:00:00Z');
|
||||
// Regex requires \d+ so "-2" won't match; we'd get null anyway, but verify.
|
||||
expect(parseRateLimitResetTime(`${RL}Resets in -2 hours.`, now)).toBeNull();
|
||||
});
|
||||
});
|
||||
Loading…
Reference in a new issue