fix(attachments): support claude gif delivery

This commit is contained in:
777genius 2026-05-25 23:43:29 +03:00
parent a67c74e343
commit b5d7da1ea8
12 changed files with 327 additions and 39 deletions

View file

@ -1,14 +1,29 @@
import type { AgentAttachmentCapability, AgentAttachmentCapabilityTarget } from './types';
import type {
AgentAttachmentCapability,
AgentAttachmentCapabilityTarget,
ProviderImageMimeType,
} from './types';
const DEFAULT_IMAGE_BYTES_PER_PROVIDER = 4 * 1024 * 1024;
const DEFAULT_IMAGE_BYTES_TOTAL = 8 * 1024 * 1024;
const DEFAULT_FILE_BYTES_PER_PROVIDER = 4 * 1024 * 1024;
function supportedImagesOnly(displayText: string): AgentAttachmentCapability {
export const NATIVE_IMAGE_MIME_TYPES = ['image/png', 'image/jpeg', 'image/webp'] as const;
export const CLAUDE_IMAGE_MIME_TYPES = [
'image/png',
'image/jpeg',
'image/gif',
'image/webp',
] as const;
function supportedImagesOnly(
displayText: string,
supportedImageMimeTypes: readonly ProviderImageMimeType[] = NATIVE_IMAGE_MIME_TYPES
): AgentAttachmentCapability {
return {
supportsImages: true,
supportsFiles: false,
supportedImageMimeTypes: ['image/png', 'image/jpeg'],
supportedImageMimeTypes: [...supportedImageMimeTypes],
supportedFileMimeTypes: [],
maxImages: 5,
maxFiles: 0,
@ -24,7 +39,7 @@ function supportedImagesOnly(displayText: string): AgentAttachmentCapability {
function supportedClaude(displayText: string): AgentAttachmentCapability {
return {
...supportedImagesOnly(displayText),
...supportedImagesOnly(displayText, CLAUDE_IMAGE_MIME_TYPES),
supportsFiles: true,
supportedFileMimeTypes: ['application/pdf', 'text/*'],
maxFiles: 5,

View file

@ -2,8 +2,8 @@ export const AGENT_ATTACHMENT_SCHEMA_VERSION = 1 as const;
export type AgentAttachmentKind = 'image' | 'file' | 'unsupported';
export type AgentImageMimeType = 'image/png' | 'image/jpeg' | 'image/webp';
export type ProviderImageMimeType = 'image/png' | 'image/jpeg';
export type AgentImageMimeType = 'image/png' | 'image/jpeg' | 'image/gif' | 'image/webp';
export type ProviderImageMimeType = AgentImageMimeType;
export type ProviderFileMimeType = 'application/pdf' | 'text/*';
export type AttachmentDeliveryFailureCode =

View file

@ -90,6 +90,44 @@ describe('agent attachment validation', () => {
expect(result).toEqual({ ok: true, warnings: [] });
});
it('allows Claude GIF image delivery without requiring optimization support', () => {
const capability = resolveAgentAttachmentCapability({
providerId: 'anthropic',
model: 'claude-haiku-4-5',
});
const result = validateAttachmentForCapability({
attachment: fakeImageAttachment({
id: 'att_gif',
originalName: 'clip.gif',
mimeType: 'image/gif',
}),
capability,
});
expect(result).toEqual({ ok: true, warnings: [] });
});
it('blocks GIF images for Codex native delivery', () => {
const capability = resolveAgentAttachmentCapability({
providerId: 'codex',
model: 'gpt-5.4-mini',
});
const result = validateAttachmentForCapability({
attachment: fakeImageAttachment({
id: 'att_gif',
originalName: 'clip.gif',
mimeType: 'image/gif',
}),
capability,
});
expect(result.ok).toBe(false);
if (!result.ok) {
expect(result.code).toBe('attachment_type_unsupported');
expect(result.message).toContain('image type');
}
});
it('blocks non-image files for Codex native delivery', () => {
const capability = resolveAgentAttachmentCapability({
providerId: 'codex',

View file

@ -13,10 +13,22 @@ import type {
const AGENT_IMAGE_MIME_TYPES = new Set<AgentImageMimeType>([
'image/png',
'image/jpeg',
'image/gif',
'image/webp',
]);
const PROVIDER_IMAGE_MIME_TYPES = new Set<ProviderImageMimeType>(['image/png', 'image/jpeg']);
const OPTIMIZABLE_AGENT_IMAGE_MIME_TYPES = new Set<Exclude<AgentImageMimeType, 'image/gif'>>([
'image/png',
'image/jpeg',
'image/webp',
]);
const PROVIDER_IMAGE_MIME_TYPES = new Set<ProviderImageMimeType>([
'image/png',
'image/jpeg',
'image/gif',
'image/webp',
]);
export function isAgentImageMimeType(mimeType: string): mimeType is AgentImageMimeType {
return AGENT_IMAGE_MIME_TYPES.has(mimeType as AgentImageMimeType);
@ -26,12 +38,27 @@ export function isProviderImageMimeType(mimeType: string): mimeType is ProviderI
return PROVIDER_IMAGE_MIME_TYPES.has(mimeType as ProviderImageMimeType);
}
function isOptimizableAgentImageMimeType(
mimeType: string
): mimeType is Exclude<AgentImageMimeType, 'image/gif'> {
return OPTIMIZABLE_AGENT_IMAGE_MIME_TYPES.has(
mimeType as Exclude<AgentImageMimeType, 'image/gif'>
);
}
function isProviderFileMimeType(mimeType: string, supported: readonly string[]): boolean {
return supported.some((candidate) =>
candidate.endsWith('/*') ? mimeType.startsWith(candidate.slice(0, -1)) : candidate === mimeType
);
}
function isCapabilityImageMimeType(
mimeType: string,
supported: readonly ProviderImageMimeType[]
): boolean {
return supported.includes(mimeType as ProviderImageMimeType);
}
export function classifyAttachmentMime(mimeType: string): AgentAttachmentKind {
if (isAgentImageMimeType(mimeType)) return 'image';
if (mimeType === 'application/pdf' || mimeType === 'text/plain' || mimeType.startsWith('text/')) {
@ -48,11 +75,11 @@ export function validateImageOptimizationInput(input: {
budget?: ImageOptimizationBudget;
}): AttachmentValidationResult {
const budget = input.budget ?? DEFAULT_AGENT_IMAGE_OPTIMIZATION_BUDGET;
if (!isAgentImageMimeType(input.mimeType)) {
if (!isOptimizableAgentImageMimeType(input.mimeType)) {
return {
ok: false,
code: 'attachment_type_unsupported',
message: 'This file type is not supported for agent image delivery.',
message: 'This image type is not supported for optimization.',
warnings: [],
};
}
@ -139,7 +166,7 @@ export function validateAttachmentForCapability(input: {
};
}
if (!isProviderImageMimeType(attachment.mimeType)) {
if (!isCapabilityImageMimeType(attachment.mimeType, capability.supportedImageMimeTypes)) {
return {
ok: false,
code: 'attachment_type_unsupported',

View file

@ -31,11 +31,30 @@ describe('Claude attachment adapter', () => {
});
expect(result.kind).toBe('structured_blocks');
expect(result.blocks[0]).toEqual({ type: 'text', text: 'What color?' });
expect(result.blocks[1]).toMatchObject({
expect(result.blocks[0]).toMatchObject({
type: 'image',
source: { type: 'base64', media_type: 'image/png' },
});
expect(result.blocks[1]).toEqual({ type: 'text', text: 'What color?' });
});
it.each([
['image/jpeg', 'photo.jpg'],
['image/gif', 'animation.gif'],
['image/webp', 'screenshot.webp'],
])('serializes %s images as structured image blocks', (mimeType, filename) => {
const result = buildClaudeAttachmentDeliveryParts({
text: 'What color?',
attachments: [attachment({ filename, mimeType })],
});
expect(result.blocks).toMatchObject([
{
type: 'image',
source: { type: 'base64', media_type: mimeType },
},
{ type: 'text', text: 'What color?' },
]);
});
it('serializes UTF-8 text files as text document blocks', () => {
@ -50,11 +69,12 @@ describe('Claude attachment adapter', () => {
],
});
expect(result.blocks[1]).toEqual({
expect(result.blocks[0]).toEqual({
type: 'document',
source: { type: 'text', media_type: 'text/plain', data: 'hello' },
title: 'note.txt',
});
expect(result.blocks[1]).toEqual({ type: 'text', text: 'Read this' });
});
it('serializes text subtypes as text document blocks', () => {
@ -69,11 +89,12 @@ describe('Claude attachment adapter', () => {
],
});
expect(result.blocks[1]).toEqual({
expect(result.blocks[0]).toEqual({
type: 'document',
source: { type: 'text', media_type: 'text/plain', data: '# hello' },
title: 'notes.md',
});
expect(result.blocks[1]).toEqual({ type: 'text', text: 'Read this' });
});
it('rejects unsupported non-image files before provider send', () => {
@ -85,11 +106,11 @@ describe('Claude attachment adapter', () => {
).toThrow(/Claude attachment MIME unsupported/);
});
it('rejects unsupported image mime types before provider send', () => {
it('rejects image mime types outside Claude vision support before provider send', () => {
expect(() =>
buildClaudeAttachmentDeliveryParts({
text: 'see gif',
attachments: [attachment({ mimeType: 'image/gif' })],
text: 'see avif',
attachments: [attachment({ mimeType: 'image/avif' })],
})
).toThrow(/Claude attachment MIME unsupported/);
});

View file

@ -1,4 +1,7 @@
import { AgentAttachmentError } from '@features/agent-attachments/core/domain';
import {
AgentAttachmentError,
CLAUDE_IMAGE_MIME_TYPES,
} from '@features/agent-attachments/core/domain';
import type { AttachmentPayload } from '@shared/types';
@ -27,20 +30,25 @@ function decodeBase64Text(data: string): { ok: true; text: string } | { ok: fals
return { ok: true, text: decoded };
}
const CLAUDE_IMAGE_MIME_TYPE_SET = new Set<string>(CLAUDE_IMAGE_MIME_TYPES);
export function buildClaudeAttachmentDeliveryParts(input: {
text: string;
attachments?: AttachmentPayload[];
}): ClaudeAttachmentDeliveryParts {
const contentBlocks: ClaudeInputBlock[] = [{ type: 'text', text: input.text }];
const textBlock: ClaudeInputBlock = { type: 'text', text: input.text };
const attachments = input.attachments ?? [];
if (attachments.length === 0) {
return { kind: 'legacy_text', blocks: contentBlocks };
return { kind: 'legacy_text', blocks: [textBlock] };
}
const imageBlocks: ClaudeInputBlock[] = [];
const documentBlocks: ClaudeInputBlock[] = [];
for (const attachment of attachments) {
if (attachment.mimeType === 'application/pdf') {
contentBlocks.push({
documentBlocks.push({
type: 'document',
source: {
type: 'base64',
@ -54,7 +62,7 @@ export function buildClaudeAttachmentDeliveryParts(input: {
if (attachment.mimeType === 'text/plain' || attachment.mimeType.startsWith('text/')) {
const decoded = decodeBase64Text(attachment.data);
contentBlocks.push(
documentBlocks.push(
decoded.ok
? {
type: 'document',
@ -78,8 +86,8 @@ export function buildClaudeAttachmentDeliveryParts(input: {
continue;
}
if (attachment.mimeType === 'image/png' || attachment.mimeType === 'image/jpeg') {
contentBlocks.push({
if (CLAUDE_IMAGE_MIME_TYPE_SET.has(attachment.mimeType)) {
imageBlocks.push({
type: 'image',
source: {
// Claude expects image bytes inside the structured image block as base64.
@ -99,7 +107,7 @@ export function buildClaudeAttachmentDeliveryParts(input: {
);
}
return { kind: 'structured_blocks', blocks: contentBlocks };
return { kind: 'structured_blocks', blocks: [...imageBlocks, ...documentBlocks, textBlock] };
}
export function redactClaudeBlocksForDiagnostics(blocks: ClaudeInputBlock[]): ClaudeInputBlock[] {

View file

@ -2572,6 +2572,17 @@ function validateAttachmentSerializedPayload(input: {
};
}
function formatAttachmentDeliveryFailure(error: unknown, teamStillAlive: boolean): string {
if (!teamStillAlive) {
return 'Failed to deliver message with attachments: team process became unavailable';
}
const message = getErrorMessage(error);
if (message.startsWith('Failed to deliver message with attachments:')) {
return message;
}
return `Failed to deliver message with attachments: ${message}`;
}
function buildMessageDeliveryText(
baseText: string,
opts: {
@ -2920,11 +2931,11 @@ async function handleSendMessage(
);
stdinSent = true;
} catch (stdinError: unknown) {
// Stdin failed (process died between check and write)
// If attachments were requested, fail rather than silently dropping them
// If attachments were requested, fail rather than silently dropping them.
// Only report offline when liveness confirms the process is unavailable.
if (validatedAttachments?.length) {
throw new Error(
'Failed to deliver message with attachments: team process became unavailable'
formatAttachmentDeliveryFailure(stdinError, provisioning.isTeamAlive(tn))
);
}
const errMsg = stdinError instanceof Error ? stdinError.message : 'unknown error';

View file

@ -31,6 +31,10 @@ function isSupportedFileMime(mimeType: string, supported: readonly string[]): bo
);
}
function isSupportedImageMime(mimeType: string, supported: readonly string[]): boolean {
return supported.includes(mimeType);
}
function canReceiveAnyAttachment(capability: AgentAttachmentCapability): boolean {
return capability.supportsImages || capability.supportsFiles;
}
@ -68,7 +72,7 @@ export function getAttachmentInputAcceptForMember(
}
const { capability } = resolveMemberAttachmentCapability(member);
if (capability.supportsImages && !capability.supportsFiles) {
return 'image/png,image/jpeg,image/webp';
return capability.supportedImageMimeTypes.join(',');
}
return '*/*';
}
@ -99,6 +103,10 @@ export function validateAttachmentFilesForMember(input: {
if (!capability.supportsImages) {
return capability.displayText;
}
const mimeType = getEffectiveMimeType(file);
if (!isSupportedImageMime(mimeType, capability.supportedImageMimeTypes)) {
return 'This image type is not supported by the selected model.';
}
continue;
}
if (!capability.supportsFiles) {
@ -136,6 +144,9 @@ export function validateAttachmentPayloadsForMember(input: {
if (!capability.supportsImages) {
return capability.displayText;
}
if (!isSupportedImageMime(attachment.mimeType, capability.supportedImageMimeTypes)) {
return 'This image type is not supported by the selected model.';
}
if (attachment.size > capability.maxBytesPerImage) {
return 'Image is too large for the selected model.';
}

View file

@ -85,7 +85,9 @@ vi.mock('@shared/utils/logger', () => ({
// Mock pathDecoder
vi.mock('@main/utils/pathDecoder', () => ({
getAppDataPath: () => path.join(os.homedir(), '.agent-teams-ai', 'data'),
getClaudeBasePath: () => path.join(os.homedir(), '.claude'),
getHomeDir: () => os.homedir(),
}));
import * as fs from 'fs/promises';

View file

@ -1192,6 +1192,65 @@ describe('ipc teams handlers', () => {
}
});
it('preserves attachment delivery errors when the lead process is still alive', async () => {
const sendHandler = handlers.get(TEAM_SEND_MESSAGE);
expect(sendHandler).toBeDefined();
provisioningService.isTeamAlive.mockReturnValue(true);
provisioningService.sendMessageToTeam.mockRejectedValueOnce(
new Error('Claude attachment MIME unsupported: image/avif')
);
const result = (await sendHandler!({} as never, 'my-team', {
member: 'team-lead',
text: 'see this',
attachments: [
{
id: 'att-1',
filename: 'screenshot.png',
mimeType: 'image/png',
size: 4,
data: Buffer.from('test').toString('base64'),
},
],
})) as { success: boolean; error?: string };
expect(result.success).toBe(false);
expect(result.error).toBe(
'Failed to deliver message with attachments: Claude attachment MIME unsupported: image/avif'
);
expect(result.error).not.toContain('team process became unavailable');
expect(service.sendDirectToLead).not.toHaveBeenCalled();
vi.mocked(console.error).mockClear();
});
it('reports attachment delivery as unavailable only when liveness confirms it', async () => {
const sendHandler = handlers.get(TEAM_SEND_MESSAGE);
expect(sendHandler).toBeDefined();
provisioningService.isTeamAlive.mockReturnValueOnce(true).mockReturnValueOnce(false);
provisioningService.sendMessageToTeam.mockRejectedValueOnce(new Error('write EPIPE'));
const result = (await sendHandler!({} as never, 'my-team', {
member: 'team-lead',
text: 'see this',
attachments: [
{
id: 'att-1',
filename: 'screenshot.png',
mimeType: 'image/png',
size: 4,
data: Buffer.from('test').toString('base64'),
},
],
})) as { success: boolean; error?: string };
expect(result.success).toBe(false);
expect(result.error).toBe(
'Failed to deliver message with attachments: team process became unavailable'
);
expect(service.sendDirectToLead).not.toHaveBeenCalled();
vi.mocked(console.error).mockClear();
});
it('rejects delegate mode when recipient is not the team lead', async () => {
const sendHandler = handlers.get(TEAM_SEND_MESSAGE);
expect(sendHandler).toBeDefined();

View file

@ -10117,7 +10117,10 @@ describe('Team agent launch matrix safe e2e', () => {
message: { content: Array<Record<string, unknown>> };
};
expect(payload.message.content).toMatchObject([
{ type: 'text', text: 'review the attached files' },
{
type: 'image',
source: { type: 'base64', media_type: 'image/png', data: 'iVBORw0KGgo=' },
},
{
type: 'document',
source: { type: 'text', media_type: 'text/plain', data: 'line one\nline two' },
@ -10128,15 +10131,48 @@ describe('Team agent launch matrix safe e2e', () => {
source: { type: 'base64', media_type: 'application/pdf', data: 'JVBERi0xLjQ=' },
title: 'brief.pdf',
},
{
type: 'image',
source: { type: 'base64', media_type: 'image/png', data: 'iVBORw0KGgo=' },
},
{ type: 'text', text: 'review the attached files' },
]);
expect(svc.isTeamAlive(firstTeamName)).toBe(true);
expect(svc.isTeamAlive(secondTeamName)).toBe(true);
});
it('serializes Claude GIF and WebP attachments without marking the team offline', async () => {
const teamName = 'pure-anthropic-extended-image-mimes-safe-e2e';
await writePureAnthropicTeamConfig({ teamName, projectPath });
await writePureAnthropicTeamMeta(teamName, projectPath);
await writePureAnthropicMembersMeta(teamName);
const svc = new TeamProvisioningService();
const run = createPureAnthropicLiveRun({ teamName, projectPath });
const writes: string[] = [];
run.child = { stdin: createWritableStdin(writes) };
trackLiveRun(svc, run);
await svc.sendMessageToTeam(teamName, 'review these browser images', [
{
filename: 'clip.gif',
mimeType: 'image/gif',
data: 'R0lGODlhAQABAAAAACw=',
},
{
filename: 'clip.webp',
mimeType: 'image/webp',
data: 'UklGRiIAAABXRUJQ',
},
]);
expect(writes).toHaveLength(1);
const payload = JSON.parse(writes[0].trim()) as {
message: { content: Array<Record<string, unknown>> };
};
expect(payload.message.content).toMatchObject([
{ type: 'image', source: { type: 'base64', media_type: 'image/gif' } },
{ type: 'image', source: { type: 'base64', media_type: 'image/webp' } },
{ type: 'text', text: 'review these browser images' },
]);
expect(svc.isTeamAlive(teamName)).toBe(true);
});
it('routes messages to the current pure Anthropic run after same-team relaunch', async () => {
const teamName = 'pure-anthropic-message-current-run-safe-e2e';
await writePureAnthropicTeamConfig({ teamName, projectPath });

View file

@ -46,7 +46,9 @@ describe('attachmentRecipientCapabilities', () => {
expect(getMemberAttachmentUnavailableReason(bob)).toBe(
'This OpenCode model is not verified for image attachments. Choose a vision-capable model or remove the image.'
);
expect(validateAttachmentFilesForMember({ member: bob, files: [file('diagram.png', 'image/png')] })).toBe(
expect(
validateAttachmentFilesForMember({ member: bob, files: [file('diagram.png', 'image/png')] })
).toBe(
'This OpenCode model is not verified for image attachments. Choose a vision-capable model or remove the image.'
);
expect(validateAttachmentPayloadsForMember({ member: bob, attachments: [payload({})] })).toBe(
@ -62,8 +64,56 @@ describe('attachmentRecipientCapabilities', () => {
expect(getMemberAttachmentUnavailableReason(bob)).toBeNull();
expect(getAttachmentInputAcceptForMember(bob)).toBe('image/png,image/jpeg,image/webp');
expect(validateAttachmentFilesForMember({ member: bob, files: [file('diagram.png', 'image/png')] })).toBeNull();
expect(validateAttachmentPayloadsForMember({ member: bob, attachments: [payload({})] })).toBeNull();
expect(
validateAttachmentFilesForMember({ member: bob, files: [file('diagram.png', 'image/png')] })
).toBeNull();
expect(
validateAttachmentPayloadsForMember({ member: bob, attachments: [payload({})] })
).toBeNull();
});
it('blocks image MIME types not supported by an otherwise image-capable provider', () => {
const codexLead = member({
name: 'lead',
agentType: 'team-lead',
providerId: 'codex',
model: 'gpt-5.5',
});
expect(
validateAttachmentFilesForMember({
member: codexLead,
files: [file('animation.gif', 'image/gif')],
})
).toBe('This image type is not supported by the selected model.');
expect(
validateAttachmentPayloadsForMember({
member: codexLead,
attachments: [payload({ filename: 'animation.gif', mimeType: 'image/gif' })],
})
).toBe('This image type is not supported by the selected model.');
});
it('allows Claude GIF and WebP image payloads', () => {
const anthropicLead = member({
name: 'lead',
agentType: 'team-lead',
providerId: 'anthropic',
model: 'claude-opus-4-6',
});
expect(
validateAttachmentFilesForMember({
member: anthropicLead,
files: [file('clip.gif', 'image/gif')],
})
).toBeNull();
expect(
validateAttachmentPayloadsForMember({
member: anthropicLead,
attachments: [payload({ filename: 'clip.webp', mimeType: 'image/webp' })],
})
).toBeNull();
});
it('blocks non-image files for image-only providers', () => {
@ -74,7 +124,12 @@ describe('attachmentRecipientCapabilities', () => {
model: 'gpt-5.5',
});
expect(validateAttachmentFilesForMember({ member: codexLead, files: [file('notes.md', 'text/markdown')] })).toBe(
expect(
validateAttachmentFilesForMember({
member: codexLead,
files: [file('notes.md', 'text/markdown')],
})
).toBe(
'This provider path currently supports image attachments only. Non-image files are blocked before provider delivery.'
);
expect(
@ -95,7 +150,12 @@ describe('attachmentRecipientCapabilities', () => {
model: 'claude-opus-4-6',
});
expect(validateAttachmentFilesForMember({ member: anthropicLead, files: [file('brief.pdf', 'application/pdf')] })).toBeNull();
expect(
validateAttachmentFilesForMember({
member: anthropicLead,
files: [file('brief.pdf', 'application/pdf')],
})
).toBeNull();
expect(
validateAttachmentPayloadsForMember({
member: anthropicLead,