Merge branch 'main' of https://github.com/justlovemaki/AIClient-2-API
This commit is contained in:
commit
d70352b26e
9 changed files with 328 additions and 39 deletions
2
VERSION
2
VERSION
|
|
@ -1 +1 @@
|
|||
2.6.6
|
||||
2.6.7
|
||||
|
|
|
|||
|
|
@ -75,6 +75,7 @@ export async function initializeConfig(args = process.argv.slice(2), configFileP
|
|||
PROMPT_LOG_MODE: "none",
|
||||
REQUEST_MAX_RETRIES: 3,
|
||||
REQUEST_BASE_DELAY: 1000,
|
||||
CREDENTIAL_SWITCH_MAX_RETRIES: 5, // 坏凭证切换最大重试次数(用于认证错误后切换凭证)
|
||||
CRON_NEAR_MINUTES: 15,
|
||||
CRON_REFRESH_TOKEN: false,
|
||||
PROVIDER_POOLS_FILE_PATH: null, // 新增号池配置文件路径
|
||||
|
|
|
|||
|
|
@ -9,7 +9,8 @@ import * as https from 'https';
|
|||
import { getProviderModels } from '../provider-models.js';
|
||||
import { countTokens } from '@anthropic-ai/tokenizer';
|
||||
import { configureAxiosProxy } from '../../utils/proxy-utils.js';
|
||||
import { isRetryableNetworkError } from '../../utils/common.js';
|
||||
import { isRetryableNetworkError, MODEL_PROVIDER } from '../../utils/common.js';
|
||||
import { getProviderPoolManager } from '../../services/service-manager.js';
|
||||
|
||||
const KIRO_THINKING = {
|
||||
MAX_BUDGET_TOKENS: 24576,
|
||||
|
|
@ -1212,16 +1213,27 @@ async initializeAuth(forceRefresh = false) {
|
|||
// 检查是否为可重试的网络错误
|
||||
const isNetworkError = isRetryableNetworkError(error);
|
||||
|
||||
if (status === 403 && !isRetry) {
|
||||
console.log('[Kiro] Received 403. Attempting token refresh and retrying...');
|
||||
// Handle 401 (Unauthorized) - try to refresh token first
|
||||
if (status === 401 && !isRetry) {
|
||||
console.log('[Kiro] Received 401. Attempting token refresh...');
|
||||
try {
|
||||
await this.initializeAuth(true); // Force refresh token
|
||||
console.log('[Kiro] Token refresh successful after 401, retrying request...');
|
||||
return this.callApi(method, model, body, true, retryCount);
|
||||
} catch (refreshError) {
|
||||
console.error('[Kiro] Token refresh failed during 403 retry:', refreshError.message);
|
||||
console.error('[Kiro] Token refresh failed during 401 retry:', refreshError.message);
|
||||
// Mark credential as unhealthy immediately and attach marker to error
|
||||
this._markCredentialUnhealthy('401 Unauthorized - Token refresh failed', refreshError);
|
||||
throw refreshError;
|
||||
}
|
||||
}
|
||||
|
||||
// Handle 403 (Forbidden) - mark as unhealthy immediately, no retry
|
||||
if (status === 403) {
|
||||
console.log('[Kiro] Received 403. Marking credential as unhealthy...');
|
||||
this._markCredentialUnhealthy('403 Forbidden', error);
|
||||
throw error;
|
||||
}
|
||||
|
||||
// Handle 429 (Too Many Requests) with exponential backoff
|
||||
if (status === 429 && retryCount < maxRetries) {
|
||||
|
|
@ -1253,6 +1265,31 @@ async initializeAuth(forceRefresh = false) {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper method to mark the current credential as unhealthy
|
||||
* @param {string} reason - The reason for marking unhealthy
|
||||
* @param {Error} [error] - Optional error object to attach the marker to
|
||||
* @returns {boolean} - Whether the credential was successfully marked as unhealthy
|
||||
* @private
|
||||
*/
|
||||
_markCredentialUnhealthy(reason, error = null) {
|
||||
const poolManager = getProviderPoolManager();
|
||||
if (poolManager && this.uuid) {
|
||||
console.log(`[Kiro] Marking credential ${this.uuid} as unhealthy. Reason: ${reason}`);
|
||||
poolManager.markProviderUnhealthyImmediately(MODEL_PROVIDER.KIRO_API, {
|
||||
uuid: this.uuid
|
||||
}, reason);
|
||||
// Attach marker to error object to prevent duplicate marking in upper layers
|
||||
if (error) {
|
||||
error.credentialMarkedUnhealthy = true;
|
||||
}
|
||||
return true;
|
||||
} else {
|
||||
console.warn(`[Kiro] Cannot mark credential as unhealthy: poolManager=${!!poolManager}, uuid=${this.uuid}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
_processApiResponse(response) {
|
||||
const rawResponseText = Buffer.isBuffer(response.data) ? response.data.toString('utf8') : String(response.data);
|
||||
//console.log(`[Kiro] Raw response length: ${rawResponseText.length}`);
|
||||
|
|
@ -1537,11 +1574,27 @@ async initializeAuth(forceRefresh = false) {
|
|||
// 检查是否为可重试的网络错误
|
||||
const isNetworkError = isRetryableNetworkError(error);
|
||||
|
||||
if (status === 403 && !isRetry) {
|
||||
console.log('[Kiro] Received 403 in stream. Attempting token refresh and retrying...');
|
||||
await this.initializeAuth(true);
|
||||
yield* this.streamApiReal(method, model, body, true, retryCount);
|
||||
return;
|
||||
// Handle 401 (Unauthorized) - try to refresh token first
|
||||
if (status === 401 && !isRetry) {
|
||||
console.log('[Kiro] Received 401 in stream. Attempting token refresh...');
|
||||
try {
|
||||
await this.initializeAuth(true); // Force refresh token
|
||||
console.log('[Kiro] Token refresh successful after 401, retrying stream...');
|
||||
yield* this.streamApiReal(method, model, body, true, retryCount);
|
||||
return;
|
||||
} catch (refreshError) {
|
||||
console.error('[Kiro] Token refresh failed during 401 retry:', refreshError.message);
|
||||
// Mark credential as unhealthy immediately and attach marker to error
|
||||
this._markCredentialUnhealthy('401 Unauthorized - Token refresh failed', refreshError);
|
||||
throw refreshError;
|
||||
}
|
||||
}
|
||||
|
||||
// Handle 403 (Forbidden) - mark as unhealthy immediately, no retry
|
||||
if (status === 403) {
|
||||
console.log('[Kiro] Received 403 in stream. Marking credential as unhealthy...');
|
||||
this._markCredentialUnhealthy('403 Forbidden', error);
|
||||
throw error;
|
||||
}
|
||||
|
||||
if (status === 429 && retryCount < maxRetries) {
|
||||
|
|
@ -2355,24 +2408,42 @@ async initializeAuth(forceRefresh = false) {
|
|||
console.log('[Kiro] Usage limits fetched successfully');
|
||||
return response.data;
|
||||
} catch (error) {
|
||||
// 如果是 403 错误,尝试刷新 token 后重试
|
||||
if (error.response?.status === 403) {
|
||||
console.log('[Kiro] Received 403 on getUsageLimits. Attempting token refresh and retrying...');
|
||||
try {
|
||||
await this.initializeAuth(true);
|
||||
// 更新 Authorization header
|
||||
headers['Authorization'] = `Bearer ${this.accessToken}`;
|
||||
headers['amz-sdk-invocation-id'] = uuidv4();
|
||||
const retryResponse = await this.axiosInstance.get(fullUrl, { headers });
|
||||
console.log('[Kiro] Usage limits fetched successfully after token refresh');
|
||||
return retryResponse.data;
|
||||
} catch (refreshError) {
|
||||
console.error('[Kiro] Token refresh failed during getUsageLimits retry:', refreshError.message);
|
||||
throw refreshError;
|
||||
const status = error.response?.status;
|
||||
|
||||
// 从响应体中提取错误信息
|
||||
let errorMessage = error.message;
|
||||
if (error.response?.data) {
|
||||
// 尝试从响应体中获取错误描述
|
||||
const responseData = error.response.data;
|
||||
if (typeof responseData === 'string') {
|
||||
errorMessage = responseData;
|
||||
} else if (responseData.message) {
|
||||
errorMessage = responseData.message;
|
||||
} else if (responseData.error) {
|
||||
errorMessage = typeof responseData.error === 'string' ? responseData.error : responseData.error.message || JSON.stringify(responseData.error);
|
||||
}
|
||||
}
|
||||
console.error('[Kiro] Failed to fetch usage limits:', error.message, error);
|
||||
throw error;
|
||||
|
||||
// 构建包含状态码和错误描述的错误信息
|
||||
const formattedError = status
|
||||
? new Error(`API call failed: ${status} - ${errorMessage}`)
|
||||
: new Error(`API call failed: ${errorMessage}`);
|
||||
|
||||
// 对于用量查询,401/403 错误直接标记凭证为不健康,不重试
|
||||
if (status === 401) {
|
||||
console.log('[Kiro] Received 401 on getUsageLimits. Marking credential as unhealthy (no retry)...');
|
||||
this._markCredentialUnhealthy('401 Unauthorized on usage query', formattedError);
|
||||
throw formattedError;
|
||||
}
|
||||
|
||||
if (status === 403) {
|
||||
console.log('[Kiro] Received 403 on getUsageLimits. Marking credential as unhealthy (no retry)...');
|
||||
this._markCredentialUnhealthy('403 Forbidden on usage query', formattedError);
|
||||
throw formattedError;
|
||||
}
|
||||
|
||||
console.error('[Kiro] Failed to fetch usage limits:', formattedError.message, error);
|
||||
throw formattedError;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -429,6 +429,35 @@ export class ProviderPoolManager {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Marks a provider as unhealthy immediately (without accumulating error count).
|
||||
* Used for definitive authentication errors like 401/403.
|
||||
* @param {string} providerType - The type of the provider.
|
||||
* @param {object} providerConfig - The configuration of the provider to mark.
|
||||
* @param {string} [errorMessage] - Optional error message to store.
|
||||
*/
|
||||
markProviderUnhealthyImmediately(providerType, providerConfig, errorMessage = null) {
|
||||
if (!providerConfig?.uuid) {
|
||||
this._log('error', 'Invalid providerConfig in markProviderUnhealthyImmediately');
|
||||
return;
|
||||
}
|
||||
|
||||
const provider = this._findProvider(providerType, providerConfig.uuid);
|
||||
if (provider) {
|
||||
provider.config.isHealthy = false;
|
||||
provider.config.errorCount = this.maxErrorCount; // Set to max to indicate definitive failure
|
||||
provider.config.lastErrorTime = new Date().toISOString();
|
||||
provider.config.lastUsed = new Date().toISOString();
|
||||
|
||||
if (errorMessage) {
|
||||
provider.config.lastErrorMessage = errorMessage;
|
||||
}
|
||||
|
||||
this._log('warn', `Immediately marked provider as unhealthy: ${providerConfig.uuid} for type ${providerType}. Reason: ${errorMessage || 'Authentication error'}`);
|
||||
this._debouncedSave(providerType);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Marks a provider as healthy.
|
||||
* @param {string} providerType - The type of the provider.
|
||||
|
|
|
|||
|
|
@ -83,6 +83,7 @@ export async function handleUpdateConfig(req, res, currentConfig) {
|
|||
if (newConfig.PROMPT_LOG_MODE !== undefined) currentConfig.PROMPT_LOG_MODE = newConfig.PROMPT_LOG_MODE;
|
||||
if (newConfig.REQUEST_MAX_RETRIES !== undefined) currentConfig.REQUEST_MAX_RETRIES = newConfig.REQUEST_MAX_RETRIES;
|
||||
if (newConfig.REQUEST_BASE_DELAY !== undefined) currentConfig.REQUEST_BASE_DELAY = newConfig.REQUEST_BASE_DELAY;
|
||||
if (newConfig.CREDENTIAL_SWITCH_MAX_RETRIES !== undefined) currentConfig.CREDENTIAL_SWITCH_MAX_RETRIES = newConfig.CREDENTIAL_SWITCH_MAX_RETRIES;
|
||||
if (newConfig.CRON_NEAR_MINUTES !== undefined) currentConfig.CRON_NEAR_MINUTES = newConfig.CRON_NEAR_MINUTES;
|
||||
if (newConfig.CRON_REFRESH_TOKEN !== undefined) currentConfig.CRON_REFRESH_TOKEN = newConfig.CRON_REFRESH_TOKEN;
|
||||
if (newConfig.PROVIDER_POOLS_FILE_PATH !== undefined) currentConfig.PROVIDER_POOLS_FILE_PATH = newConfig.PROVIDER_POOLS_FILE_PATH;
|
||||
|
|
@ -131,6 +132,7 @@ export async function handleUpdateConfig(req, res, currentConfig) {
|
|||
PROMPT_LOG_MODE: currentConfig.PROMPT_LOG_MODE,
|
||||
REQUEST_MAX_RETRIES: currentConfig.REQUEST_MAX_RETRIES,
|
||||
REQUEST_BASE_DELAY: currentConfig.REQUEST_BASE_DELAY,
|
||||
CREDENTIAL_SWITCH_MAX_RETRIES: currentConfig.CREDENTIAL_SWITCH_MAX_RETRIES,
|
||||
CRON_NEAR_MINUTES: currentConfig.CRON_NEAR_MINUTES,
|
||||
CRON_REFRESH_TOKEN: currentConfig.CRON_REFRESH_TOKEN,
|
||||
PROVIDER_POOLS_FILE_PATH: currentConfig.PROVIDER_POOLS_FILE_PATH,
|
||||
|
|
|
|||
|
|
@ -715,7 +715,18 @@ export async function handleHealthCheck(req, res, currentConfig, providerPoolMan
|
|||
message: 'Healthy'
|
||||
});
|
||||
} else {
|
||||
providerPoolManager.markProviderUnhealthy(providerType, providerConfig, healthResult.errorMessage);
|
||||
// 检查是否为认证错误(401/403),如果是则立即标记为不健康
|
||||
const errorMessage = healthResult.errorMessage || 'Check failed';
|
||||
const isAuthError = /\b(401|403)\b/.test(errorMessage) ||
|
||||
/\b(Unauthorized|Forbidden|AccessDenied|InvalidToken|ExpiredToken)\b/i.test(errorMessage);
|
||||
|
||||
if (isAuthError) {
|
||||
providerPoolManager.markProviderUnhealthyImmediately(providerType, providerConfig, errorMessage);
|
||||
console.log(`[UI API] Auth error detected for ${providerConfig.uuid}, immediately marked as unhealthy`);
|
||||
} else {
|
||||
providerPoolManager.markProviderUnhealthy(providerType, providerConfig, errorMessage);
|
||||
}
|
||||
|
||||
providerStatus.config.lastHealthCheckTime = new Date().toISOString();
|
||||
if (healthResult.modelName) {
|
||||
providerStatus.config.lastHealthCheckModel = healthResult.modelName;
|
||||
|
|
@ -724,15 +735,28 @@ export async function handleHealthCheck(req, res, currentConfig, providerPoolMan
|
|||
uuid: providerConfig.uuid,
|
||||
success: false,
|
||||
modelName: healthResult.modelName,
|
||||
message: healthResult.errorMessage || 'Check failed'
|
||||
message: errorMessage,
|
||||
isAuthError: isAuthError
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
providerPoolManager.markProviderUnhealthy(providerType, providerConfig, error.message);
|
||||
const errorMessage = error.message || 'Unknown error';
|
||||
// 检查是否为认证错误(401/403),如果是则立即标记为不健康
|
||||
const isAuthError = /\b(401|403)\b/.test(errorMessage) ||
|
||||
/\b(Unauthorized|Forbidden|AccessDenied|InvalidToken|ExpiredToken)\b/i.test(errorMessage);
|
||||
|
||||
if (isAuthError) {
|
||||
providerPoolManager.markProviderUnhealthyImmediately(providerType, providerConfig, errorMessage);
|
||||
console.log(`[UI API] Auth error detected for ${providerConfig.uuid}, immediately marked as unhealthy`);
|
||||
} else {
|
||||
providerPoolManager.markProviderUnhealthy(providerType, providerConfig, errorMessage);
|
||||
}
|
||||
|
||||
results.push({
|
||||
uuid: providerConfig.uuid,
|
||||
success: false,
|
||||
message: error.message
|
||||
message: errorMessage,
|
||||
isAuthError: isAuthError
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -217,13 +217,22 @@ export async function handleUnifiedResponse(res, responsePayload, isStream) {
|
|||
}
|
||||
}
|
||||
|
||||
export async function handleStreamRequest(res, service, model, requestBody, fromProvider, toProvider, PROMPT_LOG_MODE, PROMPT_LOG_FILENAME, providerPoolManager, pooluuid, customName) {
|
||||
export async function handleStreamRequest(res, service, model, requestBody, fromProvider, toProvider, PROMPT_LOG_MODE, PROMPT_LOG_FILENAME, providerPoolManager, pooluuid, customName, retryContext = null) {
|
||||
let fullResponseText = '';
|
||||
let fullResponseJson = '';
|
||||
let fullOldResponseJson = '';
|
||||
let responseClosed = false;
|
||||
|
||||
// 重试上下文:包含 CONFIG 和重试计数
|
||||
const maxRetries = retryContext?.maxRetries ?? 2;
|
||||
const currentRetry = retryContext?.currentRetry ?? 0;
|
||||
const CONFIG = retryContext?.CONFIG;
|
||||
const isRetry = currentRetry > 0;
|
||||
|
||||
await handleUnifiedResponse(res, '', true);
|
||||
// 只在首次请求时发送响应头,重试时跳过(响应头已发送)
|
||||
if (!isRetry) {
|
||||
await handleUnifiedResponse(res, '', true);
|
||||
}
|
||||
|
||||
// fs.writeFile('request'+Date.now()+'.json', JSON.stringify(requestBody));
|
||||
// The service returns a stream in its native format (toProvider).
|
||||
|
|
@ -283,12 +292,80 @@ export async function handleStreamRequest(res, service, model, requestBody, from
|
|||
|
||||
} catch (error) {
|
||||
console.error('\n[Server] Error during stream processing:', error.stack);
|
||||
if (providerPoolManager && pooluuid) {
|
||||
console.log(`[Provider Pool] Marking ${toProvider} as unhealthy due to stream error`);
|
||||
|
||||
// 如果已经发送了内容,不进行重试(避免响应数据损坏)
|
||||
if (fullResponseText.length > 0) {
|
||||
console.log(`[Stream Retry] Cannot retry: ${fullResponseText.length} bytes already sent to client`);
|
||||
// 直接发送错误并结束
|
||||
const errorPayload = createStreamErrorResponse(error, fromProvider);
|
||||
res.write(errorPayload);
|
||||
res.end();
|
||||
responseClosed = true;
|
||||
return;
|
||||
}
|
||||
|
||||
// 获取状态码(用于日志记录,不再用于判断是否重试)
|
||||
const status = error.response?.status;
|
||||
|
||||
// 检查凭证是否已在底层被标记为不健康(避免重复标记)
|
||||
let credentialMarkedUnhealthy = error.credentialMarkedUnhealthy === true;
|
||||
|
||||
// 如果底层未标记,则在此处标记
|
||||
if (!credentialMarkedUnhealthy && providerPoolManager && pooluuid) {
|
||||
console.log(`[Provider Pool] Marking ${toProvider} as unhealthy due to stream error (status: ${status || 'unknown'})`);
|
||||
// 如果是号池模式,并且请求处理失败,则标记当前使用的提供者为不健康
|
||||
providerPoolManager.markProviderUnhealthy(toProvider, {
|
||||
uuid: pooluuid
|
||||
});
|
||||
credentialMarkedUnhealthy = true;
|
||||
} else if (credentialMarkedUnhealthy) {
|
||||
console.log(`[Provider Pool] Credential ${pooluuid} already marked as unhealthy by lower layer, skipping duplicate marking`);
|
||||
}
|
||||
|
||||
// 凭证已被标记为不健康后,尝试切换到新凭证重试
|
||||
// 不再依赖状态码判断,只要凭证被标记不健康且可以重试,就尝试切换
|
||||
if (credentialMarkedUnhealthy && currentRetry < maxRetries && providerPoolManager && CONFIG) {
|
||||
console.log(`[Stream Retry] Credential marked unhealthy. Attempting retry ${currentRetry + 1}/${maxRetries} with different credential...`);
|
||||
|
||||
try {
|
||||
// 动态导入以避免循环依赖
|
||||
const { getApiServiceWithFallback } = await import('../services/service-manager.js');
|
||||
const result = await getApiServiceWithFallback(CONFIG, model);
|
||||
|
||||
if (result && result.service && result.uuid !== pooluuid) {
|
||||
console.log(`[Stream Retry] Switched to new credential: ${result.uuid} (provider: ${result.actualProviderType})`);
|
||||
|
||||
// 使用新服务重试
|
||||
const newRetryContext = {
|
||||
...retryContext,
|
||||
CONFIG,
|
||||
currentRetry: currentRetry + 1,
|
||||
maxRetries
|
||||
};
|
||||
|
||||
// 递归调用,使用新的服务
|
||||
return await handleStreamRequest(
|
||||
res,
|
||||
result.service,
|
||||
result.actualModel || model,
|
||||
requestBody,
|
||||
fromProvider,
|
||||
result.actualProviderType || toProvider,
|
||||
PROMPT_LOG_MODE,
|
||||
PROMPT_LOG_FILENAME,
|
||||
providerPoolManager,
|
||||
result.uuid,
|
||||
result.serviceConfig?.customName || customName,
|
||||
newRetryContext
|
||||
);
|
||||
} else if (result && result.uuid === pooluuid) {
|
||||
console.log(`[Stream Retry] No different healthy credential available. Same credential selected.`);
|
||||
} else {
|
||||
console.log(`[Stream Retry] No healthy credential available for retry.`);
|
||||
}
|
||||
} catch (retryError) {
|
||||
console.error(`[Stream Retry] Failed to get alternative service:`, retryError.message);
|
||||
}
|
||||
}
|
||||
|
||||
// 使用新方法创建符合 fromProvider 格式的流式错误响应
|
||||
|
|
@ -307,7 +384,12 @@ export async function handleStreamRequest(res, service, model, requestBody, from
|
|||
}
|
||||
|
||||
|
||||
export async function handleUnaryRequest(res, service, model, requestBody, fromProvider, toProvider, PROMPT_LOG_MODE, PROMPT_LOG_FILENAME, providerPoolManager, pooluuid, customName) {
|
||||
export async function handleUnaryRequest(res, service, model, requestBody, fromProvider, toProvider, PROMPT_LOG_MODE, PROMPT_LOG_FILENAME, providerPoolManager, pooluuid, customName, retryContext = null) {
|
||||
// 重试上下文:包含 CONFIG 和重试计数
|
||||
const maxRetries = retryContext?.maxRetries ?? 2;
|
||||
const currentRetry = retryContext?.currentRetry ?? 0;
|
||||
const CONFIG = retryContext?.CONFIG;
|
||||
|
||||
try{
|
||||
// The service returns the response in its native format (toProvider).
|
||||
const needsConversion = getProtocolPrefix(fromProvider) !== getProtocolPrefix(toProvider);
|
||||
|
|
@ -338,12 +420,69 @@ export async function handleUnaryRequest(res, service, model, requestBody, fromP
|
|||
}
|
||||
} catch (error) {
|
||||
console.error('\n[Server] Error during unary processing:', error.stack);
|
||||
if (providerPoolManager && pooluuid) {
|
||||
console.log(`[Provider Pool] Marking ${toProvider} as unhealthy due to stream error`);
|
||||
|
||||
// 获取状态码(用于日志记录,不再用于判断是否重试)
|
||||
const status = error.response?.status;
|
||||
|
||||
// 检查凭证是否已在底层被标记为不健康(避免重复标记)
|
||||
let credentialMarkedUnhealthy = error.credentialMarkedUnhealthy === true;
|
||||
|
||||
// 如果底层未标记,则在此处标记
|
||||
if (!credentialMarkedUnhealthy && providerPoolManager && pooluuid) {
|
||||
console.log(`[Provider Pool] Marking ${toProvider} as unhealthy due to unary error (status: ${status || 'unknown'})`);
|
||||
// 如果是号池模式,并且请求处理失败,则标记当前使用的提供者为不健康
|
||||
providerPoolManager.markProviderUnhealthy(toProvider, {
|
||||
uuid: pooluuid
|
||||
});
|
||||
credentialMarkedUnhealthy = true;
|
||||
} else if (credentialMarkedUnhealthy) {
|
||||
console.log(`[Provider Pool] Credential ${pooluuid} already marked as unhealthy by lower layer, skipping duplicate marking`);
|
||||
}
|
||||
|
||||
// 凭证已被标记为不健康后,尝试切换到新凭证重试
|
||||
// 不再依赖状态码判断,只要凭证被标记不健康且可以重试,就尝试切换
|
||||
if (credentialMarkedUnhealthy && currentRetry < maxRetries && providerPoolManager && CONFIG) {
|
||||
console.log(`[Unary Retry] Credential marked unhealthy. Attempting retry ${currentRetry + 1}/${maxRetries} with different credential...`);
|
||||
|
||||
try {
|
||||
// 动态导入以避免循环依赖
|
||||
const { getApiServiceWithFallback } = await import('../services/service-manager.js');
|
||||
const result = await getApiServiceWithFallback(CONFIG, model);
|
||||
|
||||
if (result && result.service && result.uuid !== pooluuid) {
|
||||
console.log(`[Unary Retry] Switched to new credential: ${result.uuid} (provider: ${result.actualProviderType})`);
|
||||
|
||||
// 使用新服务重试
|
||||
const newRetryContext = {
|
||||
...retryContext,
|
||||
CONFIG,
|
||||
currentRetry: currentRetry + 1,
|
||||
maxRetries
|
||||
};
|
||||
|
||||
// 递归调用,使用新的服务
|
||||
return await handleUnaryRequest(
|
||||
res,
|
||||
result.service,
|
||||
result.actualModel || model,
|
||||
requestBody,
|
||||
fromProvider,
|
||||
result.actualProviderType || toProvider,
|
||||
PROMPT_LOG_MODE,
|
||||
PROMPT_LOG_FILENAME,
|
||||
providerPoolManager,
|
||||
result.uuid,
|
||||
result.serviceConfig?.customName || customName,
|
||||
newRetryContext
|
||||
);
|
||||
} else if (result && result.uuid === pooluuid) {
|
||||
console.log(`[Unary Retry] No different healthy credential available. Same credential selected.`);
|
||||
} else {
|
||||
console.log(`[Unary Retry] No healthy credential available for retry.`);
|
||||
}
|
||||
} catch (retryError) {
|
||||
console.error(`[Unary Retry] Failed to get alternative service:`, retryError.message);
|
||||
}
|
||||
}
|
||||
|
||||
// 使用新方法创建符合 fromProvider 格式的错误响应
|
||||
|
|
@ -487,10 +626,19 @@ export async function handleContentGenerationRequest(req, res, service, endpoint
|
|||
await logConversation('input', promptText, CONFIG.PROMPT_LOG_MODE, PROMPT_LOG_FILENAME);
|
||||
|
||||
// 5. Call the appropriate stream or unary handler, passing the provider info.
|
||||
// 创建重试上下文,包含 CONFIG 以便在认证错误时切换凭证重试
|
||||
// 凭证切换重试次数(默认 5),可在配置中自定义更大的值
|
||||
// 注意:这与底层的 429/5xx 重试(REQUEST_MAX_RETRIES)是不同层次的重试机制
|
||||
// - 底层重试:同一凭证遇到 429/5xx 时的重试
|
||||
// - 凭证切换重试:凭证被标记不健康后切换到其他凭证
|
||||
// 当没有不同的健康凭证可用时,重试会自动停止
|
||||
const credentialSwitchMaxRetries = CONFIG.CREDENTIAL_SWITCH_MAX_RETRIES || 5;
|
||||
const retryContext = providerPoolManager ? { CONFIG, currentRetry: 0, maxRetries: credentialSwitchMaxRetries } : null;
|
||||
|
||||
if (isStream) {
|
||||
await handleStreamRequest(res, service, model, processedRequestBody, fromProvider, toProvider, CONFIG.PROMPT_LOG_MODE, PROMPT_LOG_FILENAME, providerPoolManager, actualUuid, actualCustomName);
|
||||
await handleStreamRequest(res, service, model, processedRequestBody, fromProvider, toProvider, CONFIG.PROMPT_LOG_MODE, PROMPT_LOG_FILENAME, providerPoolManager, actualUuid, actualCustomName, retryContext);
|
||||
} else {
|
||||
await handleUnaryRequest(res, service, model, processedRequestBody, fromProvider, toProvider, CONFIG.PROMPT_LOG_MODE, PROMPT_LOG_FILENAME, providerPoolManager, actualUuid, actualCustomName);
|
||||
await handleUnaryRequest(res, service, model, processedRequestBody, fromProvider, toProvider, CONFIG.PROMPT_LOG_MODE, PROMPT_LOG_FILENAME, providerPoolManager, actualUuid, actualCustomName, retryContext);
|
||||
}
|
||||
|
||||
// 执行插件钩子:内容生成后
|
||||
|
|
|
|||
|
|
@ -90,6 +90,11 @@ async function loadConfiguration() {
|
|||
if (promptLogModeEl) promptLogModeEl.value = data.PROMPT_LOG_MODE || 'none';
|
||||
if (requestMaxRetriesEl) requestMaxRetriesEl.value = data.REQUEST_MAX_RETRIES || 3;
|
||||
if (requestBaseDelayEl) requestBaseDelayEl.value = data.REQUEST_BASE_DELAY || 1000;
|
||||
|
||||
// 坏凭证切换最大重试次数
|
||||
const credentialSwitchMaxRetriesEl = document.getElementById('credentialSwitchMaxRetries');
|
||||
if (credentialSwitchMaxRetriesEl) credentialSwitchMaxRetriesEl.value = data.CREDENTIAL_SWITCH_MAX_RETRIES || 5;
|
||||
|
||||
if (cronNearMinutesEl) cronNearMinutesEl.value = data.CRON_NEAR_MINUTES || 1;
|
||||
if (cronRefreshTokenEl) cronRefreshTokenEl.checked = data.CRON_REFRESH_TOKEN || false;
|
||||
if (providerPoolsFilePathEl) providerPoolsFilePathEl.value = data.PROVIDER_POOLS_FILE_PATH;
|
||||
|
|
@ -187,6 +192,7 @@ async function saveConfiguration() {
|
|||
config.PROMPT_LOG_MODE = document.getElementById('promptLogMode')?.value || '';
|
||||
config.REQUEST_MAX_RETRIES = parseInt(document.getElementById('requestMaxRetries')?.value || 3);
|
||||
config.REQUEST_BASE_DELAY = parseInt(document.getElementById('requestBaseDelay')?.value || 1000);
|
||||
config.CREDENTIAL_SWITCH_MAX_RETRIES = parseInt(document.getElementById('credentialSwitchMaxRetries')?.value || 5);
|
||||
config.CRON_NEAR_MINUTES = parseInt(document.getElementById('cronNearMinutes')?.value || 1);
|
||||
config.CRON_REFRESH_TOKEN = document.getElementById('cronRefreshToken')?.checked || false;
|
||||
config.PROVIDER_POOLS_FILE_PATH = document.getElementById('providerPoolsFilePath')?.value || '';
|
||||
|
|
|
|||
|
|
@ -162,6 +162,14 @@
|
|||
</div>
|
||||
</div>
|
||||
|
||||
<div class="config-row">
|
||||
<div class="form-group">
|
||||
<label for="credentialSwitchMaxRetries" data-i18n="config.advanced.credentialSwitchMaxRetries">坏凭证切换最大重试次数</label>
|
||||
<input type="number" id="credentialSwitchMaxRetries" class="form-control" min="1" max="50" value="5">
|
||||
<small class="form-text" data-i18n="config.advanced.credentialSwitchMaxRetriesNote">认证错误(401/403)后切换凭证的最大重试次数,默认 5 次</small>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="config-row">
|
||||
<div class="form-group">
|
||||
<label for="cronNearMinutes" data-i18n="config.advanced.cronInterval">OAuth令牌刷新间隔(分钟)</label>
|
||||
|
|
|
|||
Loading…
Reference in a new issue