fix(attachments): enforce serialized payload budget
This commit is contained in:
parent
16161a2642
commit
960beaad44
3 changed files with 97 additions and 1 deletions
|
|
@ -1,4 +1,9 @@
|
||||||
import { allocateImageBudgets, planResizeDimensions, sortAttachmentsForDelivery } from './budgets';
|
import {
|
||||||
|
allocateImageBudgets,
|
||||||
|
estimateAgentAttachmentSerializedPayloadBytes,
|
||||||
|
planResizeDimensions,
|
||||||
|
sortAttachmentsForDelivery,
|
||||||
|
} from './budgets';
|
||||||
|
|
||||||
describe('agent attachment budgets', () => {
|
describe('agent attachment budgets', () => {
|
||||||
it('does not upscale small images', () => {
|
it('does not upscale small images', () => {
|
||||||
|
|
@ -36,4 +41,19 @@ describe('agent attachment budgets', () => {
|
||||||
]);
|
]);
|
||||||
expect(sorted.map((item) => item.id)).toEqual(['a', 'b']);
|
expect(sorted.map((item) => item.id)).toEqual(['a', 'b']);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('estimates serialized provider payload bytes including base64 overhead', () => {
|
||||||
|
const bytes = estimateAgentAttachmentSerializedPayloadBytes({
|
||||||
|
text: 'hello',
|
||||||
|
attachments: [
|
||||||
|
{
|
||||||
|
filename: 'red.png',
|
||||||
|
mimeType: 'image/png',
|
||||||
|
data: 'a'.repeat(128),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(bytes).toBeGreaterThan(128);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -9,6 +9,47 @@ export const DEFAULT_AGENT_IMAGE_OPTIMIZATION_BUDGET: ImageOptimizationBudget =
|
||||||
jpegQualityAttempts: [0.86, 0.82, 0.78, 0.74, 0.72],
|
jpegQualityAttempts: [0.86, 0.82, 0.78, 0.74, 0.72],
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export const MAX_AGENT_ATTACHMENT_SERIALIZED_PAYLOAD_BYTES = 7_500_000;
|
||||||
|
|
||||||
|
const utf8Encoder = new TextEncoder();
|
||||||
|
|
||||||
|
export function estimateAgentAttachmentSerializedPayloadBytes(input: {
|
||||||
|
text?: string;
|
||||||
|
attachments: Array<{
|
||||||
|
mimeType: string;
|
||||||
|
data: string;
|
||||||
|
filename?: string;
|
||||||
|
}>;
|
||||||
|
}): number {
|
||||||
|
const contentBlocks: unknown[] = [{ type: 'text', text: input.text ?? '' }];
|
||||||
|
for (const attachment of input.attachments) {
|
||||||
|
const isImage = attachment.mimeType.startsWith('image/');
|
||||||
|
contentBlocks.push({
|
||||||
|
type: isImage ? 'image' : 'document',
|
||||||
|
...(isImage
|
||||||
|
? {}
|
||||||
|
: {
|
||||||
|
title: attachment.filename ?? 'attachment',
|
||||||
|
}),
|
||||||
|
source: {
|
||||||
|
type: 'base64',
|
||||||
|
media_type: attachment.mimeType,
|
||||||
|
data: attachment.data,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return utf8Encoder.encode(
|
||||||
|
JSON.stringify({
|
||||||
|
type: 'user',
|
||||||
|
message: {
|
||||||
|
role: 'user',
|
||||||
|
content: contentBlocks,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
).byteLength;
|
||||||
|
}
|
||||||
|
|
||||||
export function calculatePixelCount(dimensions: ImageDimensions): number {
|
export function calculatePixelCount(dimensions: ImageDimensions): number {
|
||||||
return dimensions.width * dimensions.height;
|
return dimensions.width * dimensions.height;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -5,6 +5,10 @@ import { getAppIconPath } from '@main/utils/appIcon';
|
||||||
import { getAppDataPath, getTeamsBasePath } from '@main/utils/pathDecoder';
|
import { getAppDataPath, getTeamsBasePath } from '@main/utils/pathDecoder';
|
||||||
import { safeSendToRenderer } from '@main/utils/safeWebContentsSend';
|
import { safeSendToRenderer } from '@main/utils/safeWebContentsSend';
|
||||||
import { stripMarkdown } from '@main/utils/textFormatting';
|
import { stripMarkdown } from '@main/utils/textFormatting';
|
||||||
|
import {
|
||||||
|
estimateAgentAttachmentSerializedPayloadBytes,
|
||||||
|
MAX_AGENT_ATTACHMENT_SERIALIZED_PAYLOAD_BYTES,
|
||||||
|
} from '@features/agent-attachments/core/domain';
|
||||||
import {
|
import {
|
||||||
TEAM_ADD_MEMBER,
|
TEAM_ADD_MEMBER,
|
||||||
TEAM_ADD_TASK_COMMENT,
|
TEAM_ADD_TASK_COMMENT,
|
||||||
|
|
@ -2492,6 +2496,30 @@ function validateAttachments(
|
||||||
return { valid: true, value: result };
|
return { valid: true, value: result };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function formatAttachmentBytes(bytes: number): string {
|
||||||
|
if (bytes < 1024) return `${bytes} B`;
|
||||||
|
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
|
||||||
|
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function validateAttachmentSerializedPayload(input: {
|
||||||
|
text: string;
|
||||||
|
attachments: AttachmentPayload[];
|
||||||
|
}): { valid: true } | { valid: false; error: string } {
|
||||||
|
const estimatedBytes = estimateAgentAttachmentSerializedPayloadBytes(input);
|
||||||
|
if (estimatedBytes <= MAX_AGENT_ATTACHMENT_SERIALIZED_PAYLOAD_BYTES) {
|
||||||
|
return { valid: true };
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
valid: false,
|
||||||
|
error: `Attachment payload is too large after optimization: ${formatAttachmentBytes(
|
||||||
|
estimatedBytes
|
||||||
|
)} serialized. Limit is ${formatAttachmentBytes(
|
||||||
|
MAX_AGENT_ATTACHMENT_SERIALIZED_PAYLOAD_BYTES
|
||||||
|
)}. Remove an image or use a smaller screenshot.`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
function buildMessageDeliveryText(
|
function buildMessageDeliveryText(
|
||||||
baseText: string,
|
baseText: string,
|
||||||
opts: {
|
opts: {
|
||||||
|
|
@ -2725,6 +2753,13 @@ async function handleSendMessage(
|
||||||
return { success: false, error: attResult.error };
|
return { success: false, error: attResult.error };
|
||||||
}
|
}
|
||||||
validatedAttachments = attResult.value;
|
validatedAttachments = attResult.value;
|
||||||
|
const serializedResult = validateAttachmentSerializedPayload({
|
||||||
|
text: payload.text!,
|
||||||
|
attachments: validatedAttachments,
|
||||||
|
});
|
||||||
|
if (!serializedResult.valid) {
|
||||||
|
return { success: false, error: serializedResult.error };
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const tn = validatedTeamName.value!;
|
const tn = validatedTeamName.value!;
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue