fix(attachments): support claude gif delivery
This commit is contained in:
parent
a67c74e343
commit
b5d7da1ea8
12 changed files with 327 additions and 39 deletions
|
|
@ -1,14 +1,29 @@
|
||||||
import type { AgentAttachmentCapability, AgentAttachmentCapabilityTarget } from './types';
|
import type {
|
||||||
|
AgentAttachmentCapability,
|
||||||
|
AgentAttachmentCapabilityTarget,
|
||||||
|
ProviderImageMimeType,
|
||||||
|
} from './types';
|
||||||
|
|
||||||
const DEFAULT_IMAGE_BYTES_PER_PROVIDER = 4 * 1024 * 1024;
|
const DEFAULT_IMAGE_BYTES_PER_PROVIDER = 4 * 1024 * 1024;
|
||||||
const DEFAULT_IMAGE_BYTES_TOTAL = 8 * 1024 * 1024;
|
const DEFAULT_IMAGE_BYTES_TOTAL = 8 * 1024 * 1024;
|
||||||
const DEFAULT_FILE_BYTES_PER_PROVIDER = 4 * 1024 * 1024;
|
const DEFAULT_FILE_BYTES_PER_PROVIDER = 4 * 1024 * 1024;
|
||||||
|
|
||||||
function supportedImagesOnly(displayText: string): AgentAttachmentCapability {
|
export const NATIVE_IMAGE_MIME_TYPES = ['image/png', 'image/jpeg', 'image/webp'] as const;
|
||||||
|
export const CLAUDE_IMAGE_MIME_TYPES = [
|
||||||
|
'image/png',
|
||||||
|
'image/jpeg',
|
||||||
|
'image/gif',
|
||||||
|
'image/webp',
|
||||||
|
] as const;
|
||||||
|
|
||||||
|
function supportedImagesOnly(
|
||||||
|
displayText: string,
|
||||||
|
supportedImageMimeTypes: readonly ProviderImageMimeType[] = NATIVE_IMAGE_MIME_TYPES
|
||||||
|
): AgentAttachmentCapability {
|
||||||
return {
|
return {
|
||||||
supportsImages: true,
|
supportsImages: true,
|
||||||
supportsFiles: false,
|
supportsFiles: false,
|
||||||
supportedImageMimeTypes: ['image/png', 'image/jpeg'],
|
supportedImageMimeTypes: [...supportedImageMimeTypes],
|
||||||
supportedFileMimeTypes: [],
|
supportedFileMimeTypes: [],
|
||||||
maxImages: 5,
|
maxImages: 5,
|
||||||
maxFiles: 0,
|
maxFiles: 0,
|
||||||
|
|
@ -24,7 +39,7 @@ function supportedImagesOnly(displayText: string): AgentAttachmentCapability {
|
||||||
|
|
||||||
function supportedClaude(displayText: string): AgentAttachmentCapability {
|
function supportedClaude(displayText: string): AgentAttachmentCapability {
|
||||||
return {
|
return {
|
||||||
...supportedImagesOnly(displayText),
|
...supportedImagesOnly(displayText, CLAUDE_IMAGE_MIME_TYPES),
|
||||||
supportsFiles: true,
|
supportsFiles: true,
|
||||||
supportedFileMimeTypes: ['application/pdf', 'text/*'],
|
supportedFileMimeTypes: ['application/pdf', 'text/*'],
|
||||||
maxFiles: 5,
|
maxFiles: 5,
|
||||||
|
|
|
||||||
|
|
@ -2,8 +2,8 @@ export const AGENT_ATTACHMENT_SCHEMA_VERSION = 1 as const;
|
||||||
|
|
||||||
export type AgentAttachmentKind = 'image' | 'file' | 'unsupported';
|
export type AgentAttachmentKind = 'image' | 'file' | 'unsupported';
|
||||||
|
|
||||||
export type AgentImageMimeType = 'image/png' | 'image/jpeg' | 'image/webp';
|
export type AgentImageMimeType = 'image/png' | 'image/jpeg' | 'image/gif' | 'image/webp';
|
||||||
export type ProviderImageMimeType = 'image/png' | 'image/jpeg';
|
export type ProviderImageMimeType = AgentImageMimeType;
|
||||||
export type ProviderFileMimeType = 'application/pdf' | 'text/*';
|
export type ProviderFileMimeType = 'application/pdf' | 'text/*';
|
||||||
|
|
||||||
export type AttachmentDeliveryFailureCode =
|
export type AttachmentDeliveryFailureCode =
|
||||||
|
|
|
||||||
|
|
@ -90,6 +90,44 @@ describe('agent attachment validation', () => {
|
||||||
expect(result).toEqual({ ok: true, warnings: [] });
|
expect(result).toEqual({ ok: true, warnings: [] });
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('allows Claude GIF image delivery without requiring optimization support', () => {
|
||||||
|
const capability = resolveAgentAttachmentCapability({
|
||||||
|
providerId: 'anthropic',
|
||||||
|
model: 'claude-haiku-4-5',
|
||||||
|
});
|
||||||
|
const result = validateAttachmentForCapability({
|
||||||
|
attachment: fakeImageAttachment({
|
||||||
|
id: 'att_gif',
|
||||||
|
originalName: 'clip.gif',
|
||||||
|
mimeType: 'image/gif',
|
||||||
|
}),
|
||||||
|
capability,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toEqual({ ok: true, warnings: [] });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('blocks GIF images for Codex native delivery', () => {
|
||||||
|
const capability = resolveAgentAttachmentCapability({
|
||||||
|
providerId: 'codex',
|
||||||
|
model: 'gpt-5.4-mini',
|
||||||
|
});
|
||||||
|
const result = validateAttachmentForCapability({
|
||||||
|
attachment: fakeImageAttachment({
|
||||||
|
id: 'att_gif',
|
||||||
|
originalName: 'clip.gif',
|
||||||
|
mimeType: 'image/gif',
|
||||||
|
}),
|
||||||
|
capability,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.ok).toBe(false);
|
||||||
|
if (!result.ok) {
|
||||||
|
expect(result.code).toBe('attachment_type_unsupported');
|
||||||
|
expect(result.message).toContain('image type');
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
it('blocks non-image files for Codex native delivery', () => {
|
it('blocks non-image files for Codex native delivery', () => {
|
||||||
const capability = resolveAgentAttachmentCapability({
|
const capability = resolveAgentAttachmentCapability({
|
||||||
providerId: 'codex',
|
providerId: 'codex',
|
||||||
|
|
|
||||||
|
|
@ -13,10 +13,22 @@ import type {
|
||||||
const AGENT_IMAGE_MIME_TYPES = new Set<AgentImageMimeType>([
|
const AGENT_IMAGE_MIME_TYPES = new Set<AgentImageMimeType>([
|
||||||
'image/png',
|
'image/png',
|
||||||
'image/jpeg',
|
'image/jpeg',
|
||||||
|
'image/gif',
|
||||||
'image/webp',
|
'image/webp',
|
||||||
]);
|
]);
|
||||||
|
|
||||||
const PROVIDER_IMAGE_MIME_TYPES = new Set<ProviderImageMimeType>(['image/png', 'image/jpeg']);
|
const OPTIMIZABLE_AGENT_IMAGE_MIME_TYPES = new Set<Exclude<AgentImageMimeType, 'image/gif'>>([
|
||||||
|
'image/png',
|
||||||
|
'image/jpeg',
|
||||||
|
'image/webp',
|
||||||
|
]);
|
||||||
|
|
||||||
|
const PROVIDER_IMAGE_MIME_TYPES = new Set<ProviderImageMimeType>([
|
||||||
|
'image/png',
|
||||||
|
'image/jpeg',
|
||||||
|
'image/gif',
|
||||||
|
'image/webp',
|
||||||
|
]);
|
||||||
|
|
||||||
export function isAgentImageMimeType(mimeType: string): mimeType is AgentImageMimeType {
|
export function isAgentImageMimeType(mimeType: string): mimeType is AgentImageMimeType {
|
||||||
return AGENT_IMAGE_MIME_TYPES.has(mimeType as AgentImageMimeType);
|
return AGENT_IMAGE_MIME_TYPES.has(mimeType as AgentImageMimeType);
|
||||||
|
|
@ -26,12 +38,27 @@ export function isProviderImageMimeType(mimeType: string): mimeType is ProviderI
|
||||||
return PROVIDER_IMAGE_MIME_TYPES.has(mimeType as ProviderImageMimeType);
|
return PROVIDER_IMAGE_MIME_TYPES.has(mimeType as ProviderImageMimeType);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function isOptimizableAgentImageMimeType(
|
||||||
|
mimeType: string
|
||||||
|
): mimeType is Exclude<AgentImageMimeType, 'image/gif'> {
|
||||||
|
return OPTIMIZABLE_AGENT_IMAGE_MIME_TYPES.has(
|
||||||
|
mimeType as Exclude<AgentImageMimeType, 'image/gif'>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
function isProviderFileMimeType(mimeType: string, supported: readonly string[]): boolean {
|
function isProviderFileMimeType(mimeType: string, supported: readonly string[]): boolean {
|
||||||
return supported.some((candidate) =>
|
return supported.some((candidate) =>
|
||||||
candidate.endsWith('/*') ? mimeType.startsWith(candidate.slice(0, -1)) : candidate === mimeType
|
candidate.endsWith('/*') ? mimeType.startsWith(candidate.slice(0, -1)) : candidate === mimeType
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function isCapabilityImageMimeType(
|
||||||
|
mimeType: string,
|
||||||
|
supported: readonly ProviderImageMimeType[]
|
||||||
|
): boolean {
|
||||||
|
return supported.includes(mimeType as ProviderImageMimeType);
|
||||||
|
}
|
||||||
|
|
||||||
export function classifyAttachmentMime(mimeType: string): AgentAttachmentKind {
|
export function classifyAttachmentMime(mimeType: string): AgentAttachmentKind {
|
||||||
if (isAgentImageMimeType(mimeType)) return 'image';
|
if (isAgentImageMimeType(mimeType)) return 'image';
|
||||||
if (mimeType === 'application/pdf' || mimeType === 'text/plain' || mimeType.startsWith('text/')) {
|
if (mimeType === 'application/pdf' || mimeType === 'text/plain' || mimeType.startsWith('text/')) {
|
||||||
|
|
@ -48,11 +75,11 @@ export function validateImageOptimizationInput(input: {
|
||||||
budget?: ImageOptimizationBudget;
|
budget?: ImageOptimizationBudget;
|
||||||
}): AttachmentValidationResult {
|
}): AttachmentValidationResult {
|
||||||
const budget = input.budget ?? DEFAULT_AGENT_IMAGE_OPTIMIZATION_BUDGET;
|
const budget = input.budget ?? DEFAULT_AGENT_IMAGE_OPTIMIZATION_BUDGET;
|
||||||
if (!isAgentImageMimeType(input.mimeType)) {
|
if (!isOptimizableAgentImageMimeType(input.mimeType)) {
|
||||||
return {
|
return {
|
||||||
ok: false,
|
ok: false,
|
||||||
code: 'attachment_type_unsupported',
|
code: 'attachment_type_unsupported',
|
||||||
message: 'This file type is not supported for agent image delivery.',
|
message: 'This image type is not supported for optimization.',
|
||||||
warnings: [],
|
warnings: [],
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
@ -139,7 +166,7 @@ export function validateAttachmentForCapability(input: {
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!isProviderImageMimeType(attachment.mimeType)) {
|
if (!isCapabilityImageMimeType(attachment.mimeType, capability.supportedImageMimeTypes)) {
|
||||||
return {
|
return {
|
||||||
ok: false,
|
ok: false,
|
||||||
code: 'attachment_type_unsupported',
|
code: 'attachment_type_unsupported',
|
||||||
|
|
|
||||||
|
|
@ -31,11 +31,30 @@ describe('Claude attachment adapter', () => {
|
||||||
});
|
});
|
||||||
|
|
||||||
expect(result.kind).toBe('structured_blocks');
|
expect(result.kind).toBe('structured_blocks');
|
||||||
expect(result.blocks[0]).toEqual({ type: 'text', text: 'What color?' });
|
expect(result.blocks[0]).toMatchObject({
|
||||||
expect(result.blocks[1]).toMatchObject({
|
|
||||||
type: 'image',
|
type: 'image',
|
||||||
source: { type: 'base64', media_type: 'image/png' },
|
source: { type: 'base64', media_type: 'image/png' },
|
||||||
});
|
});
|
||||||
|
expect(result.blocks[1]).toEqual({ type: 'text', text: 'What color?' });
|
||||||
|
});
|
||||||
|
|
||||||
|
it.each([
|
||||||
|
['image/jpeg', 'photo.jpg'],
|
||||||
|
['image/gif', 'animation.gif'],
|
||||||
|
['image/webp', 'screenshot.webp'],
|
||||||
|
])('serializes %s images as structured image blocks', (mimeType, filename) => {
|
||||||
|
const result = buildClaudeAttachmentDeliveryParts({
|
||||||
|
text: 'What color?',
|
||||||
|
attachments: [attachment({ filename, mimeType })],
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.blocks).toMatchObject([
|
||||||
|
{
|
||||||
|
type: 'image',
|
||||||
|
source: { type: 'base64', media_type: mimeType },
|
||||||
|
},
|
||||||
|
{ type: 'text', text: 'What color?' },
|
||||||
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('serializes UTF-8 text files as text document blocks', () => {
|
it('serializes UTF-8 text files as text document blocks', () => {
|
||||||
|
|
@ -50,11 +69,12 @@ describe('Claude attachment adapter', () => {
|
||||||
],
|
],
|
||||||
});
|
});
|
||||||
|
|
||||||
expect(result.blocks[1]).toEqual({
|
expect(result.blocks[0]).toEqual({
|
||||||
type: 'document',
|
type: 'document',
|
||||||
source: { type: 'text', media_type: 'text/plain', data: 'hello' },
|
source: { type: 'text', media_type: 'text/plain', data: 'hello' },
|
||||||
title: 'note.txt',
|
title: 'note.txt',
|
||||||
});
|
});
|
||||||
|
expect(result.blocks[1]).toEqual({ type: 'text', text: 'Read this' });
|
||||||
});
|
});
|
||||||
|
|
||||||
it('serializes text subtypes as text document blocks', () => {
|
it('serializes text subtypes as text document blocks', () => {
|
||||||
|
|
@ -69,11 +89,12 @@ describe('Claude attachment adapter', () => {
|
||||||
],
|
],
|
||||||
});
|
});
|
||||||
|
|
||||||
expect(result.blocks[1]).toEqual({
|
expect(result.blocks[0]).toEqual({
|
||||||
type: 'document',
|
type: 'document',
|
||||||
source: { type: 'text', media_type: 'text/plain', data: '# hello' },
|
source: { type: 'text', media_type: 'text/plain', data: '# hello' },
|
||||||
title: 'notes.md',
|
title: 'notes.md',
|
||||||
});
|
});
|
||||||
|
expect(result.blocks[1]).toEqual({ type: 'text', text: 'Read this' });
|
||||||
});
|
});
|
||||||
|
|
||||||
it('rejects unsupported non-image files before provider send', () => {
|
it('rejects unsupported non-image files before provider send', () => {
|
||||||
|
|
@ -85,11 +106,11 @@ describe('Claude attachment adapter', () => {
|
||||||
).toThrow(/Claude attachment MIME unsupported/);
|
).toThrow(/Claude attachment MIME unsupported/);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('rejects unsupported image mime types before provider send', () => {
|
it('rejects image mime types outside Claude vision support before provider send', () => {
|
||||||
expect(() =>
|
expect(() =>
|
||||||
buildClaudeAttachmentDeliveryParts({
|
buildClaudeAttachmentDeliveryParts({
|
||||||
text: 'see gif',
|
text: 'see avif',
|
||||||
attachments: [attachment({ mimeType: 'image/gif' })],
|
attachments: [attachment({ mimeType: 'image/avif' })],
|
||||||
})
|
})
|
||||||
).toThrow(/Claude attachment MIME unsupported/);
|
).toThrow(/Claude attachment MIME unsupported/);
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,7 @@
|
||||||
import { AgentAttachmentError } from '@features/agent-attachments/core/domain';
|
import {
|
||||||
|
AgentAttachmentError,
|
||||||
|
CLAUDE_IMAGE_MIME_TYPES,
|
||||||
|
} from '@features/agent-attachments/core/domain';
|
||||||
|
|
||||||
import type { AttachmentPayload } from '@shared/types';
|
import type { AttachmentPayload } from '@shared/types';
|
||||||
|
|
||||||
|
|
@ -27,20 +30,25 @@ function decodeBase64Text(data: string): { ok: true; text: string } | { ok: fals
|
||||||
return { ok: true, text: decoded };
|
return { ok: true, text: decoded };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const CLAUDE_IMAGE_MIME_TYPE_SET = new Set<string>(CLAUDE_IMAGE_MIME_TYPES);
|
||||||
|
|
||||||
export function buildClaudeAttachmentDeliveryParts(input: {
|
export function buildClaudeAttachmentDeliveryParts(input: {
|
||||||
text: string;
|
text: string;
|
||||||
attachments?: AttachmentPayload[];
|
attachments?: AttachmentPayload[];
|
||||||
}): ClaudeAttachmentDeliveryParts {
|
}): ClaudeAttachmentDeliveryParts {
|
||||||
const contentBlocks: ClaudeInputBlock[] = [{ type: 'text', text: input.text }];
|
const textBlock: ClaudeInputBlock = { type: 'text', text: input.text };
|
||||||
const attachments = input.attachments ?? [];
|
const attachments = input.attachments ?? [];
|
||||||
|
|
||||||
if (attachments.length === 0) {
|
if (attachments.length === 0) {
|
||||||
return { kind: 'legacy_text', blocks: contentBlocks };
|
return { kind: 'legacy_text', blocks: [textBlock] };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const imageBlocks: ClaudeInputBlock[] = [];
|
||||||
|
const documentBlocks: ClaudeInputBlock[] = [];
|
||||||
|
|
||||||
for (const attachment of attachments) {
|
for (const attachment of attachments) {
|
||||||
if (attachment.mimeType === 'application/pdf') {
|
if (attachment.mimeType === 'application/pdf') {
|
||||||
contentBlocks.push({
|
documentBlocks.push({
|
||||||
type: 'document',
|
type: 'document',
|
||||||
source: {
|
source: {
|
||||||
type: 'base64',
|
type: 'base64',
|
||||||
|
|
@ -54,7 +62,7 @@ export function buildClaudeAttachmentDeliveryParts(input: {
|
||||||
|
|
||||||
if (attachment.mimeType === 'text/plain' || attachment.mimeType.startsWith('text/')) {
|
if (attachment.mimeType === 'text/plain' || attachment.mimeType.startsWith('text/')) {
|
||||||
const decoded = decodeBase64Text(attachment.data);
|
const decoded = decodeBase64Text(attachment.data);
|
||||||
contentBlocks.push(
|
documentBlocks.push(
|
||||||
decoded.ok
|
decoded.ok
|
||||||
? {
|
? {
|
||||||
type: 'document',
|
type: 'document',
|
||||||
|
|
@ -78,8 +86,8 @@ export function buildClaudeAttachmentDeliveryParts(input: {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (attachment.mimeType === 'image/png' || attachment.mimeType === 'image/jpeg') {
|
if (CLAUDE_IMAGE_MIME_TYPE_SET.has(attachment.mimeType)) {
|
||||||
contentBlocks.push({
|
imageBlocks.push({
|
||||||
type: 'image',
|
type: 'image',
|
||||||
source: {
|
source: {
|
||||||
// Claude expects image bytes inside the structured image block as base64.
|
// Claude expects image bytes inside the structured image block as base64.
|
||||||
|
|
@ -99,7 +107,7 @@ export function buildClaudeAttachmentDeliveryParts(input: {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
return { kind: 'structured_blocks', blocks: contentBlocks };
|
return { kind: 'structured_blocks', blocks: [...imageBlocks, ...documentBlocks, textBlock] };
|
||||||
}
|
}
|
||||||
|
|
||||||
export function redactClaudeBlocksForDiagnostics(blocks: ClaudeInputBlock[]): ClaudeInputBlock[] {
|
export function redactClaudeBlocksForDiagnostics(blocks: ClaudeInputBlock[]): ClaudeInputBlock[] {
|
||||||
|
|
|
||||||
|
|
@ -2572,6 +2572,17 @@ function validateAttachmentSerializedPayload(input: {
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function formatAttachmentDeliveryFailure(error: unknown, teamStillAlive: boolean): string {
|
||||||
|
if (!teamStillAlive) {
|
||||||
|
return 'Failed to deliver message with attachments: team process became unavailable';
|
||||||
|
}
|
||||||
|
const message = getErrorMessage(error);
|
||||||
|
if (message.startsWith('Failed to deliver message with attachments:')) {
|
||||||
|
return message;
|
||||||
|
}
|
||||||
|
return `Failed to deliver message with attachments: ${message}`;
|
||||||
|
}
|
||||||
|
|
||||||
function buildMessageDeliveryText(
|
function buildMessageDeliveryText(
|
||||||
baseText: string,
|
baseText: string,
|
||||||
opts: {
|
opts: {
|
||||||
|
|
@ -2920,11 +2931,11 @@ async function handleSendMessage(
|
||||||
);
|
);
|
||||||
stdinSent = true;
|
stdinSent = true;
|
||||||
} catch (stdinError: unknown) {
|
} catch (stdinError: unknown) {
|
||||||
// Stdin failed (process died between check and write)
|
// If attachments were requested, fail rather than silently dropping them.
|
||||||
// If attachments were requested, fail rather than silently dropping them
|
// Only report offline when liveness confirms the process is unavailable.
|
||||||
if (validatedAttachments?.length) {
|
if (validatedAttachments?.length) {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
'Failed to deliver message with attachments: team process became unavailable'
|
formatAttachmentDeliveryFailure(stdinError, provisioning.isTeamAlive(tn))
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
const errMsg = stdinError instanceof Error ? stdinError.message : 'unknown error';
|
const errMsg = stdinError instanceof Error ? stdinError.message : 'unknown error';
|
||||||
|
|
|
||||||
|
|
@ -31,6 +31,10 @@ function isSupportedFileMime(mimeType: string, supported: readonly string[]): bo
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function isSupportedImageMime(mimeType: string, supported: readonly string[]): boolean {
|
||||||
|
return supported.includes(mimeType);
|
||||||
|
}
|
||||||
|
|
||||||
function canReceiveAnyAttachment(capability: AgentAttachmentCapability): boolean {
|
function canReceiveAnyAttachment(capability: AgentAttachmentCapability): boolean {
|
||||||
return capability.supportsImages || capability.supportsFiles;
|
return capability.supportsImages || capability.supportsFiles;
|
||||||
}
|
}
|
||||||
|
|
@ -68,7 +72,7 @@ export function getAttachmentInputAcceptForMember(
|
||||||
}
|
}
|
||||||
const { capability } = resolveMemberAttachmentCapability(member);
|
const { capability } = resolveMemberAttachmentCapability(member);
|
||||||
if (capability.supportsImages && !capability.supportsFiles) {
|
if (capability.supportsImages && !capability.supportsFiles) {
|
||||||
return 'image/png,image/jpeg,image/webp';
|
return capability.supportedImageMimeTypes.join(',');
|
||||||
}
|
}
|
||||||
return '*/*';
|
return '*/*';
|
||||||
}
|
}
|
||||||
|
|
@ -99,6 +103,10 @@ export function validateAttachmentFilesForMember(input: {
|
||||||
if (!capability.supportsImages) {
|
if (!capability.supportsImages) {
|
||||||
return capability.displayText;
|
return capability.displayText;
|
||||||
}
|
}
|
||||||
|
const mimeType = getEffectiveMimeType(file);
|
||||||
|
if (!isSupportedImageMime(mimeType, capability.supportedImageMimeTypes)) {
|
||||||
|
return 'This image type is not supported by the selected model.';
|
||||||
|
}
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if (!capability.supportsFiles) {
|
if (!capability.supportsFiles) {
|
||||||
|
|
@ -136,6 +144,9 @@ export function validateAttachmentPayloadsForMember(input: {
|
||||||
if (!capability.supportsImages) {
|
if (!capability.supportsImages) {
|
||||||
return capability.displayText;
|
return capability.displayText;
|
||||||
}
|
}
|
||||||
|
if (!isSupportedImageMime(attachment.mimeType, capability.supportedImageMimeTypes)) {
|
||||||
|
return 'This image type is not supported by the selected model.';
|
||||||
|
}
|
||||||
if (attachment.size > capability.maxBytesPerImage) {
|
if (attachment.size > capability.maxBytesPerImage) {
|
||||||
return 'Image is too large for the selected model.';
|
return 'Image is too large for the selected model.';
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -85,7 +85,9 @@ vi.mock('@shared/utils/logger', () => ({
|
||||||
|
|
||||||
// Mock pathDecoder
|
// Mock pathDecoder
|
||||||
vi.mock('@main/utils/pathDecoder', () => ({
|
vi.mock('@main/utils/pathDecoder', () => ({
|
||||||
|
getAppDataPath: () => path.join(os.homedir(), '.agent-teams-ai', 'data'),
|
||||||
getClaudeBasePath: () => path.join(os.homedir(), '.claude'),
|
getClaudeBasePath: () => path.join(os.homedir(), '.claude'),
|
||||||
|
getHomeDir: () => os.homedir(),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
import * as fs from 'fs/promises';
|
import * as fs from 'fs/promises';
|
||||||
|
|
|
||||||
|
|
@ -1192,6 +1192,65 @@ describe('ipc teams handlers', () => {
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('preserves attachment delivery errors when the lead process is still alive', async () => {
|
||||||
|
const sendHandler = handlers.get(TEAM_SEND_MESSAGE);
|
||||||
|
expect(sendHandler).toBeDefined();
|
||||||
|
provisioningService.isTeamAlive.mockReturnValue(true);
|
||||||
|
provisioningService.sendMessageToTeam.mockRejectedValueOnce(
|
||||||
|
new Error('Claude attachment MIME unsupported: image/avif')
|
||||||
|
);
|
||||||
|
|
||||||
|
const result = (await sendHandler!({} as never, 'my-team', {
|
||||||
|
member: 'team-lead',
|
||||||
|
text: 'see this',
|
||||||
|
attachments: [
|
||||||
|
{
|
||||||
|
id: 'att-1',
|
||||||
|
filename: 'screenshot.png',
|
||||||
|
mimeType: 'image/png',
|
||||||
|
size: 4,
|
||||||
|
data: Buffer.from('test').toString('base64'),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
})) as { success: boolean; error?: string };
|
||||||
|
|
||||||
|
expect(result.success).toBe(false);
|
||||||
|
expect(result.error).toBe(
|
||||||
|
'Failed to deliver message with attachments: Claude attachment MIME unsupported: image/avif'
|
||||||
|
);
|
||||||
|
expect(result.error).not.toContain('team process became unavailable');
|
||||||
|
expect(service.sendDirectToLead).not.toHaveBeenCalled();
|
||||||
|
vi.mocked(console.error).mockClear();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('reports attachment delivery as unavailable only when liveness confirms it', async () => {
|
||||||
|
const sendHandler = handlers.get(TEAM_SEND_MESSAGE);
|
||||||
|
expect(sendHandler).toBeDefined();
|
||||||
|
provisioningService.isTeamAlive.mockReturnValueOnce(true).mockReturnValueOnce(false);
|
||||||
|
provisioningService.sendMessageToTeam.mockRejectedValueOnce(new Error('write EPIPE'));
|
||||||
|
|
||||||
|
const result = (await sendHandler!({} as never, 'my-team', {
|
||||||
|
member: 'team-lead',
|
||||||
|
text: 'see this',
|
||||||
|
attachments: [
|
||||||
|
{
|
||||||
|
id: 'att-1',
|
||||||
|
filename: 'screenshot.png',
|
||||||
|
mimeType: 'image/png',
|
||||||
|
size: 4,
|
||||||
|
data: Buffer.from('test').toString('base64'),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
})) as { success: boolean; error?: string };
|
||||||
|
|
||||||
|
expect(result.success).toBe(false);
|
||||||
|
expect(result.error).toBe(
|
||||||
|
'Failed to deliver message with attachments: team process became unavailable'
|
||||||
|
);
|
||||||
|
expect(service.sendDirectToLead).not.toHaveBeenCalled();
|
||||||
|
vi.mocked(console.error).mockClear();
|
||||||
|
});
|
||||||
|
|
||||||
it('rejects delegate mode when recipient is not the team lead', async () => {
|
it('rejects delegate mode when recipient is not the team lead', async () => {
|
||||||
const sendHandler = handlers.get(TEAM_SEND_MESSAGE);
|
const sendHandler = handlers.get(TEAM_SEND_MESSAGE);
|
||||||
expect(sendHandler).toBeDefined();
|
expect(sendHandler).toBeDefined();
|
||||||
|
|
|
||||||
|
|
@ -10117,7 +10117,10 @@ describe('Team agent launch matrix safe e2e', () => {
|
||||||
message: { content: Array<Record<string, unknown>> };
|
message: { content: Array<Record<string, unknown>> };
|
||||||
};
|
};
|
||||||
expect(payload.message.content).toMatchObject([
|
expect(payload.message.content).toMatchObject([
|
||||||
{ type: 'text', text: 'review the attached files' },
|
{
|
||||||
|
type: 'image',
|
||||||
|
source: { type: 'base64', media_type: 'image/png', data: 'iVBORw0KGgo=' },
|
||||||
|
},
|
||||||
{
|
{
|
||||||
type: 'document',
|
type: 'document',
|
||||||
source: { type: 'text', media_type: 'text/plain', data: 'line one\nline two' },
|
source: { type: 'text', media_type: 'text/plain', data: 'line one\nline two' },
|
||||||
|
|
@ -10128,15 +10131,48 @@ describe('Team agent launch matrix safe e2e', () => {
|
||||||
source: { type: 'base64', media_type: 'application/pdf', data: 'JVBERi0xLjQ=' },
|
source: { type: 'base64', media_type: 'application/pdf', data: 'JVBERi0xLjQ=' },
|
||||||
title: 'brief.pdf',
|
title: 'brief.pdf',
|
||||||
},
|
},
|
||||||
{
|
{ type: 'text', text: 'review the attached files' },
|
||||||
type: 'image',
|
|
||||||
source: { type: 'base64', media_type: 'image/png', data: 'iVBORw0KGgo=' },
|
|
||||||
},
|
|
||||||
]);
|
]);
|
||||||
expect(svc.isTeamAlive(firstTeamName)).toBe(true);
|
expect(svc.isTeamAlive(firstTeamName)).toBe(true);
|
||||||
expect(svc.isTeamAlive(secondTeamName)).toBe(true);
|
expect(svc.isTeamAlive(secondTeamName)).toBe(true);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('serializes Claude GIF and WebP attachments without marking the team offline', async () => {
|
||||||
|
const teamName = 'pure-anthropic-extended-image-mimes-safe-e2e';
|
||||||
|
await writePureAnthropicTeamConfig({ teamName, projectPath });
|
||||||
|
await writePureAnthropicTeamMeta(teamName, projectPath);
|
||||||
|
await writePureAnthropicMembersMeta(teamName);
|
||||||
|
const svc = new TeamProvisioningService();
|
||||||
|
const run = createPureAnthropicLiveRun({ teamName, projectPath });
|
||||||
|
const writes: string[] = [];
|
||||||
|
run.child = { stdin: createWritableStdin(writes) };
|
||||||
|
trackLiveRun(svc, run);
|
||||||
|
|
||||||
|
await svc.sendMessageToTeam(teamName, 'review these browser images', [
|
||||||
|
{
|
||||||
|
filename: 'clip.gif',
|
||||||
|
mimeType: 'image/gif',
|
||||||
|
data: 'R0lGODlhAQABAAAAACw=',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
filename: 'clip.webp',
|
||||||
|
mimeType: 'image/webp',
|
||||||
|
data: 'UklGRiIAAABXRUJQ',
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
expect(writes).toHaveLength(1);
|
||||||
|
const payload = JSON.parse(writes[0].trim()) as {
|
||||||
|
message: { content: Array<Record<string, unknown>> };
|
||||||
|
};
|
||||||
|
expect(payload.message.content).toMatchObject([
|
||||||
|
{ type: 'image', source: { type: 'base64', media_type: 'image/gif' } },
|
||||||
|
{ type: 'image', source: { type: 'base64', media_type: 'image/webp' } },
|
||||||
|
{ type: 'text', text: 'review these browser images' },
|
||||||
|
]);
|
||||||
|
expect(svc.isTeamAlive(teamName)).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
it('routes messages to the current pure Anthropic run after same-team relaunch', async () => {
|
it('routes messages to the current pure Anthropic run after same-team relaunch', async () => {
|
||||||
const teamName = 'pure-anthropic-message-current-run-safe-e2e';
|
const teamName = 'pure-anthropic-message-current-run-safe-e2e';
|
||||||
await writePureAnthropicTeamConfig({ teamName, projectPath });
|
await writePureAnthropicTeamConfig({ teamName, projectPath });
|
||||||
|
|
|
||||||
|
|
@ -46,7 +46,9 @@ describe('attachmentRecipientCapabilities', () => {
|
||||||
expect(getMemberAttachmentUnavailableReason(bob)).toBe(
|
expect(getMemberAttachmentUnavailableReason(bob)).toBe(
|
||||||
'This OpenCode model is not verified for image attachments. Choose a vision-capable model or remove the image.'
|
'This OpenCode model is not verified for image attachments. Choose a vision-capable model or remove the image.'
|
||||||
);
|
);
|
||||||
expect(validateAttachmentFilesForMember({ member: bob, files: [file('diagram.png', 'image/png')] })).toBe(
|
expect(
|
||||||
|
validateAttachmentFilesForMember({ member: bob, files: [file('diagram.png', 'image/png')] })
|
||||||
|
).toBe(
|
||||||
'This OpenCode model is not verified for image attachments. Choose a vision-capable model or remove the image.'
|
'This OpenCode model is not verified for image attachments. Choose a vision-capable model or remove the image.'
|
||||||
);
|
);
|
||||||
expect(validateAttachmentPayloadsForMember({ member: bob, attachments: [payload({})] })).toBe(
|
expect(validateAttachmentPayloadsForMember({ member: bob, attachments: [payload({})] })).toBe(
|
||||||
|
|
@ -62,8 +64,56 @@ describe('attachmentRecipientCapabilities', () => {
|
||||||
|
|
||||||
expect(getMemberAttachmentUnavailableReason(bob)).toBeNull();
|
expect(getMemberAttachmentUnavailableReason(bob)).toBeNull();
|
||||||
expect(getAttachmentInputAcceptForMember(bob)).toBe('image/png,image/jpeg,image/webp');
|
expect(getAttachmentInputAcceptForMember(bob)).toBe('image/png,image/jpeg,image/webp');
|
||||||
expect(validateAttachmentFilesForMember({ member: bob, files: [file('diagram.png', 'image/png')] })).toBeNull();
|
expect(
|
||||||
expect(validateAttachmentPayloadsForMember({ member: bob, attachments: [payload({})] })).toBeNull();
|
validateAttachmentFilesForMember({ member: bob, files: [file('diagram.png', 'image/png')] })
|
||||||
|
).toBeNull();
|
||||||
|
expect(
|
||||||
|
validateAttachmentPayloadsForMember({ member: bob, attachments: [payload({})] })
|
||||||
|
).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('blocks image MIME types not supported by an otherwise image-capable provider', () => {
|
||||||
|
const codexLead = member({
|
||||||
|
name: 'lead',
|
||||||
|
agentType: 'team-lead',
|
||||||
|
providerId: 'codex',
|
||||||
|
model: 'gpt-5.5',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(
|
||||||
|
validateAttachmentFilesForMember({
|
||||||
|
member: codexLead,
|
||||||
|
files: [file('animation.gif', 'image/gif')],
|
||||||
|
})
|
||||||
|
).toBe('This image type is not supported by the selected model.');
|
||||||
|
expect(
|
||||||
|
validateAttachmentPayloadsForMember({
|
||||||
|
member: codexLead,
|
||||||
|
attachments: [payload({ filename: 'animation.gif', mimeType: 'image/gif' })],
|
||||||
|
})
|
||||||
|
).toBe('This image type is not supported by the selected model.');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('allows Claude GIF and WebP image payloads', () => {
|
||||||
|
const anthropicLead = member({
|
||||||
|
name: 'lead',
|
||||||
|
agentType: 'team-lead',
|
||||||
|
providerId: 'anthropic',
|
||||||
|
model: 'claude-opus-4-6',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(
|
||||||
|
validateAttachmentFilesForMember({
|
||||||
|
member: anthropicLead,
|
||||||
|
files: [file('clip.gif', 'image/gif')],
|
||||||
|
})
|
||||||
|
).toBeNull();
|
||||||
|
expect(
|
||||||
|
validateAttachmentPayloadsForMember({
|
||||||
|
member: anthropicLead,
|
||||||
|
attachments: [payload({ filename: 'clip.webp', mimeType: 'image/webp' })],
|
||||||
|
})
|
||||||
|
).toBeNull();
|
||||||
});
|
});
|
||||||
|
|
||||||
it('blocks non-image files for image-only providers', () => {
|
it('blocks non-image files for image-only providers', () => {
|
||||||
|
|
@ -74,7 +124,12 @@ describe('attachmentRecipientCapabilities', () => {
|
||||||
model: 'gpt-5.5',
|
model: 'gpt-5.5',
|
||||||
});
|
});
|
||||||
|
|
||||||
expect(validateAttachmentFilesForMember({ member: codexLead, files: [file('notes.md', 'text/markdown')] })).toBe(
|
expect(
|
||||||
|
validateAttachmentFilesForMember({
|
||||||
|
member: codexLead,
|
||||||
|
files: [file('notes.md', 'text/markdown')],
|
||||||
|
})
|
||||||
|
).toBe(
|
||||||
'This provider path currently supports image attachments only. Non-image files are blocked before provider delivery.'
|
'This provider path currently supports image attachments only. Non-image files are blocked before provider delivery.'
|
||||||
);
|
);
|
||||||
expect(
|
expect(
|
||||||
|
|
@ -95,7 +150,12 @@ describe('attachmentRecipientCapabilities', () => {
|
||||||
model: 'claude-opus-4-6',
|
model: 'claude-opus-4-6',
|
||||||
});
|
});
|
||||||
|
|
||||||
expect(validateAttachmentFilesForMember({ member: anthropicLead, files: [file('brief.pdf', 'application/pdf')] })).toBeNull();
|
expect(
|
||||||
|
validateAttachmentFilesForMember({
|
||||||
|
member: anthropicLead,
|
||||||
|
files: [file('brief.pdf', 'application/pdf')],
|
||||||
|
})
|
||||||
|
).toBeNull();
|
||||||
expect(
|
expect(
|
||||||
validateAttachmentPayloadsForMember({
|
validateAttachmentPayloadsForMember({
|
||||||
member: anthropicLead,
|
member: anthropicLead,
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue