fix: 简化健康检查逻辑,支持热更新interval,修复provider-tag点击问题
- 移除checkHealth per-instance flag,简化逻辑为只依赖providerTypes勾选 - performHealthChecks启动检查也遵循providerTypes过滤 - 优化健康检查日志:显示耗时、通过/失败计数 - 支持修改interval后热更新,无需重启(globalThis.reloadHealthCheckTimer) - 移除openai-iflow选项(未注册的provider) - 修复config-manager.js中scheduledHealthCheckProviders点击不生效问题 - providerTypes配置修改后下次检查自动生效
This commit is contained in:
parent
df9a36291c
commit
9172401a50
8 changed files with 160 additions and 122 deletions
|
|
@ -1674,6 +1674,10 @@ export class ProviderPoolManager {
|
|||
* This method would typically be called periodically (e.g., via cron job).
|
||||
*/
|
||||
async performHealthChecks(isInit = false) {
|
||||
const scheduledConfig = this.globalConfig?.SCHEDULED_HEALTH_CHECK;
|
||||
const selectedProviderTypes = scheduledConfig?.providerTypes || [];
|
||||
const checkAllTypes = selectedProviderTypes.length === 0;
|
||||
|
||||
this._log('info', 'Performing health checks on all providers...');
|
||||
const now = new Date();
|
||||
|
||||
|
|
@ -1681,6 +1685,11 @@ export class ProviderPoolManager {
|
|||
this._checkAndRecoverScheduledProviders();
|
||||
|
||||
for (const providerType in this.providerStatus) {
|
||||
// Filter by selected provider types if specified (same logic as scheduled health check)
|
||||
if (!checkAllTypes && !selectedProviderTypes.includes(providerType)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const providerStatus of this.providerStatus[providerType]) {
|
||||
const providerConfig = providerStatus.config;
|
||||
|
||||
|
|
@ -1749,7 +1758,8 @@ export class ProviderPoolManager {
|
|||
* It respects provider-level isDisabled and checkHealth flags.
|
||||
*/
|
||||
async performScheduledHealthChecks() {
|
||||
const scheduledConfig = globalThis.CONFIG?.SCHEDULED_HEALTH_CHECK;
|
||||
const scheduledConfig = this.globalConfig?.SCHEDULED_HEALTH_CHECK;
|
||||
const checkStartTime = Date.now();
|
||||
|
||||
// Check if scheduled health checks are disabled
|
||||
if (!scheduledConfig?.enabled) {
|
||||
|
|
@ -1757,11 +1767,20 @@ export class ProviderPoolManager {
|
|||
return;
|
||||
}
|
||||
|
||||
this._log('info', '[ScheduledHealthCheck] Starting scheduled health checks on all providers...');
|
||||
|
||||
// Get selected provider types, if empty/undefined then check all
|
||||
const selectedProviderTypes = scheduledConfig?.providerTypes;
|
||||
const checkAllTypes = !selectedProviderTypes || selectedProviderTypes.length === 0;
|
||||
let selectedProviderTypes = scheduledConfig?.providerTypes;
|
||||
|
||||
// Validate providerTypes is an array to prevent TypeError
|
||||
if (!Array.isArray(selectedProviderTypes)) {
|
||||
this._log('warn', '[ScheduledHealthCheck] providerTypes is not an array, treating as check all');
|
||||
selectedProviderTypes = [];
|
||||
}
|
||||
|
||||
const checkAllTypes = selectedProviderTypes.length === 0;
|
||||
|
||||
// Count providers to be checked
|
||||
let totalProviders = 0;
|
||||
let providersToCheck = [];
|
||||
|
||||
for (const providerType in this.providerStatus) {
|
||||
// Filter by selected provider types if specified
|
||||
|
|
@ -1777,39 +1796,58 @@ export class ProviderPoolManager {
|
|||
continue;
|
||||
}
|
||||
|
||||
// Skip providers with checkHealth disabled
|
||||
if (provider.config.checkHealth === false) {
|
||||
this._log('debug', `[ScheduledHealthCheck] Skipping ${provider.config.uuid} (${providerType}): checkHealth is false`);
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
// Perform health check with forceCheck=false (respects checkHealth flag)
|
||||
const result = await this._checkProviderHealth(providerType, provider.config, false);
|
||||
|
||||
// result === null means checkHealth was false (already handled above) or not implemented
|
||||
if (result === null) {
|
||||
this._log('debug', `[ScheduledHealthCheck] Health check for ${provider.config.uuid} (${providerType}) skipped: not implemented`);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!result.success) {
|
||||
// Provider is unhealthy
|
||||
this._log('warn', `[ScheduledHealthCheck] Health check failed for ${provider.config.uuid} (${providerType}): ${result.errorMessage || 'Provider is not responding correctly.'}`);
|
||||
this.markProviderUnhealthyImmediately(providerType, provider.config, result.errorMessage);
|
||||
} else {
|
||||
// Provider is healthy
|
||||
this._log('debug', `[ScheduledHealthCheck] Health check passed for ${provider.config.uuid} (${providerType})`);
|
||||
this.markProviderHealthy(providerType, provider.config, true, result.modelName);
|
||||
}
|
||||
} catch (error) {
|
||||
this._log('error', `[ScheduledHealthCheck] Health check exception for ${provider.config.uuid} (${providerType}): ${error.message}`);
|
||||
this.markProviderUnhealthyImmediately(providerType, provider.config, error.message);
|
||||
}
|
||||
totalProviders++;
|
||||
providersToCheck.push({ providerType, provider, uuid: provider.config.uuid, customName: provider.config.customName });
|
||||
}
|
||||
}
|
||||
|
||||
this._log('info', '[ScheduledHealthCheck] Completed');
|
||||
this._log('info', `[ScheduledHealthCheck] Starting scheduled health checks: ${totalProviders} provider(s) to check (interval: ${scheduledConfig.interval}ms, types: ${checkAllTypes ? 'all' : selectedProviderTypes.join(', ')})`);
|
||||
|
||||
let successCount = 0;
|
||||
let failCount = 0;
|
||||
|
||||
for (const { providerType, provider, uuid, customName } of providersToCheck) {
|
||||
// Skip if provider became disabled during iteration
|
||||
if (provider.config.isDisabled === true) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const checkStartTime = Date.now();
|
||||
const checkModelName = provider.config.checkModelName || ProviderPoolManager.DEFAULT_HEALTH_CHECK_MODELS[providerType] || 'unknown';
|
||||
const displayName = customName || uuid.substring(0, 8);
|
||||
|
||||
try {
|
||||
// Perform health check (forceCheck=true to bypass per-instance checkHealth flag)
|
||||
const result = await this._checkProviderHealth(providerType, provider.config);
|
||||
const checkDuration = Date.now() - checkStartTime;
|
||||
|
||||
// result === null means check not implemented for this provider type
|
||||
if (result === null) {
|
||||
this._log('info', `[ScheduledHealthCheck] ${displayName} (${providerType}): check skipped - not implemented (${checkDuration}ms)`);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!result.success) {
|
||||
// Provider is unhealthy
|
||||
failCount++;
|
||||
this._log('warn', `[ScheduledHealthCheck] ${displayName} (${providerType}) FAILED: ${result.errorMessage || 'Provider is not responding correctly.'} (${checkDuration}ms)`);
|
||||
this.markProviderUnhealthyImmediately(providerType, provider.config, result.errorMessage);
|
||||
} else {
|
||||
// Provider is healthy
|
||||
successCount++;
|
||||
this._log('info', `[ScheduledHealthCheck] ${displayName} (${providerType}) PASSED: model=${result.modelName || checkModelName} (${checkDuration}ms)`);
|
||||
this.markProviderHealthy(providerType, provider.config, false, result.modelName);
|
||||
}
|
||||
} catch (error) {
|
||||
const checkDuration = Date.now() - checkStartTime;
|
||||
failCount++;
|
||||
this._log('error', `[ScheduledHealthCheck] ${displayName} (${providerType}) EXCEPTION: ${error.message} (${checkDuration}ms)`);
|
||||
this.markProviderUnhealthyImmediately(providerType, provider.config, error.message);
|
||||
}
|
||||
}
|
||||
|
||||
const totalDuration = Date.now() - checkStartTime;
|
||||
this._log('info', `[ScheduledHealthCheck] Completed: ${successCount} passed, ${failCount} failed, ${totalDuration}ms total`);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -1864,15 +1902,9 @@ export class ProviderPoolManager {
|
|||
* Performs an actual health check for a specific provider.
|
||||
* @param {string} providerType - The type of the provider.
|
||||
* @param {object} providerConfig - The configuration of the provider to check.
|
||||
* @param {boolean} forceCheck - If true, ignore checkHealth config and force the check.
|
||||
* @returns {Promise<{success: boolean, modelName: string, errorMessage: string}|null>} - Health check result object or null if check not implemented.
|
||||
*/
|
||||
async _checkProviderHealth(providerType, providerConfig, forceCheck = false) {
|
||||
// 如果未启用健康检查且不是强制检查,返回 null(提前返回,避免不必要的计算)
|
||||
if (!providerConfig.checkHealth && !forceCheck) {
|
||||
return null;
|
||||
}
|
||||
|
||||
async _checkProviderHealth(providerType, providerConfig) {
|
||||
// 确定健康检查使用的模型名称
|
||||
const modelName = providerConfig.checkModelName ||
|
||||
ProviderPoolManager.DEFAULT_HEALTH_CHECK_MODELS[providerType];
|
||||
|
|
@ -1907,8 +1939,6 @@ export class ProviderPoolManager {
|
|||
const timeoutId = setTimeout(() => abortController.abort(), healthCheckTimeout);
|
||||
|
||||
try {
|
||||
this._log('debug', `Health check attempt ${i + 1}/${healthCheckRequests.length} for ${modelName}: ${JSON.stringify(healthCheckRequest)}`);
|
||||
|
||||
// 尝试将 signal 注入请求体,供支持的适配器使用
|
||||
const requestWithSignal = {
|
||||
...healthCheckRequest,
|
||||
|
|
@ -1918,16 +1948,17 @@ export class ProviderPoolManager {
|
|||
await serviceAdapter.generateContent(modelName, requestWithSignal);
|
||||
|
||||
clearTimeout(timeoutId);
|
||||
// 将健康检查计入使用量(resetUsageCount=false 只会递增,不会重置)
|
||||
this.markProviderHealthy(providerType, providerConfig, false, modelName);
|
||||
return { success: true, modelName, errorMessage: null };
|
||||
} catch (error) {
|
||||
clearTimeout(timeoutId);
|
||||
lastError = error;
|
||||
this._log('debug', `Health check attempt ${i + 1} failed for ${providerType}: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// 所有尝试都失败
|
||||
this._log('error', `Health check failed for ${providerType} after ${healthCheckRequests.length} attempts: ${lastError?.message}`);
|
||||
this._log('warn', `[HealthCheck] ${providerType} failed after ${healthCheckRequests.length} attempts: ${lastError?.message}`);
|
||||
return { success: false, modelName, errorMessage: lastError?.message || 'All health check attempts failed' };
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -364,30 +364,71 @@ async function startServer() {
|
|||
// 定时健康检查
|
||||
const scheduledConfig = CONFIG.SCHEDULED_HEALTH_CHECK;
|
||||
if (scheduledConfig?.enabled) {
|
||||
const interval = scheduledConfig.interval || CONFIG.CRON_NEAR_MINUTES * 60 * 1000;
|
||||
// Validate interval is within acceptable range (minimum 60000ms, no maximum)
|
||||
const DEFAULT_INTERVAL = CONFIG.CRON_NEAR_MINUTES * 60 * 1000;
|
||||
let interval = scheduledConfig.interval;
|
||||
if (typeof interval !== 'number' || interval < 60000) {
|
||||
logger.warn(`[ScheduledHealthCheck] Invalid interval ${interval}, using default ${DEFAULT_INTERVAL}`);
|
||||
interval = DEFAULT_INTERVAL;
|
||||
}
|
||||
|
||||
// 启动时运行健康检查
|
||||
if (scheduledConfig.startupRun !== false) {
|
||||
logger.info('[ScheduledHealthCheck] Running scheduled health check on startup...');
|
||||
setTimeout(async () => {
|
||||
// 使用 setImmediate 确保在事件循环的下一阶段执行,此时服务器已完全就绪
|
||||
setImmediate(async () => {
|
||||
try {
|
||||
await poolManager.performScheduledHealthChecks();
|
||||
} catch (error) {
|
||||
logger.error('[ScheduledHealthCheck] Startup run error:', error);
|
||||
}
|
||||
}, 100); // 延迟100ms确保服务已完全就绪
|
||||
});
|
||||
}
|
||||
|
||||
let isHealthCheckRunning = false;
|
||||
let healthCheckTimerId = null;
|
||||
|
||||
// 定时健康检查函数
|
||||
const runHealthCheckTimer = (interval) => {
|
||||
// 清除旧的 timer
|
||||
if (healthCheckTimerId) {
|
||||
clearInterval(healthCheckTimerId);
|
||||
}
|
||||
// 设置定时任务
|
||||
healthCheckTimerId = setInterval(async () => {
|
||||
if (isHealthCheckRunning) {
|
||||
logger.debug('[ScheduledHealthCheck] Skipping - previous run still in progress');
|
||||
return;
|
||||
}
|
||||
isHealthCheckRunning = true;
|
||||
try {
|
||||
await poolManager.performScheduledHealthChecks();
|
||||
} catch (error) {
|
||||
logger.error('[ScheduledHealthCheck] Error:', error);
|
||||
} finally {
|
||||
isHealthCheckRunning = false;
|
||||
}
|
||||
}, interval);
|
||||
logger.info(`[ScheduledHealthCheck] Scheduled every ${interval}ms`);
|
||||
};
|
||||
|
||||
// 启动时运行健康检查
|
||||
if (scheduledConfig.startupRun !== false) {
|
||||
logger.info('[ScheduledHealthCheck] Running scheduled health check on startup...');
|
||||
setImmediate(async () => {
|
||||
try {
|
||||
await poolManager.performScheduledHealthChecks();
|
||||
} catch (error) {
|
||||
logger.error('[ScheduledHealthCheck] Startup run error:', error);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// 设置定时任务
|
||||
setInterval(async () => {
|
||||
try {
|
||||
await poolManager.performScheduledHealthChecks();
|
||||
} catch (error) {
|
||||
logger.error('[ScheduledHealthCheck] Error:', error);
|
||||
}
|
||||
}, interval);
|
||||
runHealthCheckTimer(interval);
|
||||
|
||||
logger.info(`[ScheduledHealthCheck] Scheduled every ${interval}ms`);
|
||||
// 导出重载函数供外部调用
|
||||
globalThis.reloadHealthCheckTimer = runHealthCheckTimer;
|
||||
}
|
||||
|
||||
// 如果是子进程,通知主进程已就绪
|
||||
|
|
|
|||
|
|
@ -117,16 +117,22 @@ export async function handleUpdateConfig(req, res, currentConfig) {
|
|||
|
||||
// Scheduled Health Check settings
|
||||
if (newConfig.SCHEDULED_HEALTH_CHECK !== undefined) {
|
||||
const incoming = newConfig.SCHEDULED_HEALTH_CHECK;
|
||||
currentConfig.SCHEDULED_HEALTH_CHECK = {
|
||||
enabled: incoming?.enabled === true,
|
||||
startupRun: incoming?.startupRun !== false,
|
||||
interval: (() => {
|
||||
const val = Number(incoming?.interval);
|
||||
return isNaN(val) ? 600000 : Math.max(60000, Math.min(3600000, val));
|
||||
})(),
|
||||
providerTypes: Array.isArray(incoming?.providerTypes) ? incoming.providerTypes : []
|
||||
};
|
||||
const incoming = newConfig.SCHEDULED_HEALTH_CHECK;
|
||||
const newInterval = (() => {
|
||||
const val = Number(incoming?.interval);
|
||||
return isNaN(val) ? 600000 : Math.max(60000, val);
|
||||
})();
|
||||
currentConfig.SCHEDULED_HEALTH_CHECK = {
|
||||
enabled: incoming?.enabled === true,
|
||||
startupRun: incoming?.startupRun !== false,
|
||||
interval: newInterval,
|
||||
providerTypes: Array.isArray(incoming?.providerTypes) ? incoming.providerTypes : []
|
||||
};
|
||||
|
||||
// 如果定时器已存在且 enabled,重新加载 timer(interval 变化时)
|
||||
if (globalThis.reloadHealthCheckTimer && currentConfig.SCHEDULED_HEALTH_CHECK.enabled) {
|
||||
globalThis.reloadHealthCheckTimer(newInterval);
|
||||
}
|
||||
}
|
||||
|
||||
// Handle system prompt update
|
||||
|
|
|
|||
|
|
@ -710,8 +710,7 @@ export async function handleHealthCheck(req, res, currentConfig, providerPoolMan
|
|||
}
|
||||
|
||||
try {
|
||||
// 传递 forceCheck = true 强制执行健康检查,忽略 checkHealth 配置
|
||||
const healthResult = await providerPoolManager._checkProviderHealth(providerType, providerConfig, true);
|
||||
const healthResult = await providerPoolManager._checkProviderHealth(providerType, providerConfig);
|
||||
|
||||
if (healthResult === null) {
|
||||
results.push({
|
||||
|
|
|
|||
|
|
@ -41,6 +41,7 @@ export const PROVIDER_MAPPINGS = [
|
|||
providerType: 'openai-qwen-oauth',
|
||||
credPathKey: 'QWEN_OAUTH_CREDS_FILE_PATH',
|
||||
defaultCheckModel: 'qwen3-coder-plus',
|
||||
defaultCheckHealth: true,
|
||||
displayName: 'Qwen OAuth',
|
||||
needsProjectId: false,
|
||||
urlKeys: ['QWEN_BASE_URL', 'QWEN_OAUTH_BASE_URL']
|
||||
|
|
@ -324,13 +325,13 @@ export async function isValidOAuthCredentials(filePath) {
|
|||
* @returns {Object} 新的提供商配置对象
|
||||
*/
|
||||
export function createProviderConfig(options) {
|
||||
const { credPathKey, credPath, defaultCheckModel, needsProjectId, urlKeys } = options;
|
||||
const { credPathKey, credPath, defaultCheckModel, defaultCheckHealth, needsProjectId, urlKeys } = options;
|
||||
|
||||
const newProvider = {
|
||||
[credPathKey]: credPath,
|
||||
uuid: generateUUID(),
|
||||
checkModelName: defaultCheckModel,
|
||||
checkHealth: false,
|
||||
checkHealth: defaultCheckHealth ?? false,
|
||||
isHealthy: true,
|
||||
isDisabled: false,
|
||||
lastUsed: null,
|
||||
|
|
|
|||
|
|
@ -32,6 +32,12 @@ function updateConfigProviderConfigs(configs) {
|
|||
if (tlsSidecarProvidersEl) {
|
||||
renderProviderTags(tlsSidecarProvidersEl, configs, false);
|
||||
}
|
||||
|
||||
// 渲染定时健康检查的提供商选择
|
||||
const scheduledHealthCheckProvidersEl = document.getElementById('scheduledHealthCheckProviders');
|
||||
if (scheduledHealthCheckProvidersEl) {
|
||||
renderProviderTags(scheduledHealthCheckProvidersEl, configs, false);
|
||||
}
|
||||
|
||||
// 重新加载当前配置以恢复选中状态
|
||||
loadConfiguration();
|
||||
|
|
|
|||
|
|
@ -71,43 +71,6 @@ textarea.form-control {
|
|||
margin: 0 auto;
|
||||
}
|
||||
|
||||
.form-group {
|
||||
margin-bottom: 1.5rem;
|
||||
}
|
||||
|
||||
.form-row {
|
||||
display: grid;
|
||||
grid-template-columns: 1fr 1fr;
|
||||
gap: 1.5rem;
|
||||
}
|
||||
|
||||
.form-group label {
|
||||
display: block;
|
||||
margin-bottom: 0.5rem;
|
||||
font-weight: 600;
|
||||
color: var(--text-primary);
|
||||
font-size: 0.9rem;
|
||||
}
|
||||
|
||||
.optional-tag, .form-group label .optional-mark {
|
||||
font-size: 0.75rem;
|
||||
color: var(--text-tertiary);
|
||||
font-weight: 400;
|
||||
margin-left: 0.5rem;
|
||||
background: var(--bg-tertiary);
|
||||
padding: 0.125rem 0.375rem;
|
||||
border-radius: var(--radius-sm);
|
||||
}
|
||||
|
||||
.form-control::placeholder {
|
||||
color: var(--text-tertiary);
|
||||
}
|
||||
|
||||
textarea.form-control {
|
||||
resize: vertical;
|
||||
font-family: inherit;
|
||||
}
|
||||
|
||||
/* 密码输入框样式 */
|
||||
.password-input-group {
|
||||
position: relative;
|
||||
|
|
|
|||
|
|
@ -62,10 +62,7 @@
|
|||
<i class="fas fa-reply"></i>
|
||||
<span>OpenAI Responses</span>
|
||||
</button>
|
||||
<button type="button" class="provider-tag" data-value="openai-iflow">
|
||||
<i class="fas fa-stream"></i>
|
||||
<span data-i18n="dashboard.routing.nodeName.iflow">iFlow OAuth</span>
|
||||
</button>
|
||||
|
||||
<button type="button" class="provider-tag" data-value="openai-codex-oauth">
|
||||
<i class="fas fa-code"></i>
|
||||
<span data-i18n="dashboard.routing.nodeName.codex">OpenAI Codex OAuth</span>
|
||||
|
|
@ -118,10 +115,7 @@
|
|||
<i class="fas fa-reply"></i>
|
||||
<span>OpenAI Responses</span>
|
||||
</button>
|
||||
<button type="button" class="provider-tag" data-value="openai-iflow">
|
||||
<i class="fas fa-stream"></i>
|
||||
<span>iFlow OAuth</span>
|
||||
</button>
|
||||
|
||||
<button type="button" class="provider-tag" data-value="openai-codex-oauth">
|
||||
<i class="fas fa-code"></i>
|
||||
<span>OpenAI Codex OAuth</span>
|
||||
|
|
@ -233,14 +227,14 @@
|
|||
<h3 data-i18n="config.healthCheck.title"><i class="fas fa-heartbeat"></i> 定时健康检查</h3>
|
||||
<div class="config-row">
|
||||
<div class="form-group">
|
||||
<label data-i18n="config.healthCheck.enabled">启用定时检查</label>
|
||||
<span class="toggle-label" data-i18n="config.healthCheck.enabled">启用定时检查</span>
|
||||
<label class="toggle-switch">
|
||||
<input type="checkbox" id="scheduledHealthCheckEnabled">
|
||||
<span class="toggle-slider"></span>
|
||||
</label>
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label data-i18n="config.healthCheck.startupRun">启动时运行</label>
|
||||
<span class="toggle-label" data-i18n="config.healthCheck.startupRun">启动时运行</span>
|
||||
<label class="toggle-switch">
|
||||
<input type="checkbox" id="scheduledHealthCheckStartupRun">
|
||||
<span class="toggle-slider"></span>
|
||||
|
|
@ -290,10 +284,7 @@
|
|||
<i class="fas fa-reply"></i>
|
||||
<span>OpenAI Responses</span>
|
||||
</button>
|
||||
<button type="button" class="provider-tag" data-value="openai-iflow">
|
||||
<i class="fas fa-stream"></i>
|
||||
<span>iFlow OAuth</span>
|
||||
</button>
|
||||
|
||||
<button type="button" class="provider-tag" data-value="openai-codex-oauth">
|
||||
<i class="fas fa-code"></i>
|
||||
<span>OpenAI Codex OAuth</span>
|
||||
|
|
|
|||
Loading…
Reference in a new issue