#!/usr/bin/env node import { spawn, spawnSync } from 'node:child_process'; import { existsSync } from 'node:fs'; import { mkdir, mkdtemp, readFile, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import path from 'node:path'; import { deflateSync } from 'node:zlib'; const PROMPT = 'Look at the attached image. Reply with exactly one word: red, green, or blue.'; const MULTI_IMAGE_PROMPT = 'Look at every attached image. Each image has a dominant background color. Reply with exactly one word: red, green, or blue. Use red if the dominant background color is red in all images.'; const TIMEOUT_MS = 90_000; const CASES = [ { id: 'claude-subscription-streaming', runtime: 'claude', model: process.env.CLAUDE_ATTACHMENTS_SMOKE_CLAUDE_MODEL || 'claude-haiku-4-5', command: async (imagePath, cwd, testCase, imagePaths) => ({ bin: 'claude', args: [ '-p', '--input-format', 'stream-json', '--output-format', 'stream-json', '--verbose', '--no-session-persistence', '--model', testCase.model, ], cwd, stdin: await buildClaudeStreamJsonPrompt(imagePaths, PROMPT), }), expected: /red/i, }, { id: 'claude-subscription-streaming-multi-image', runtime: 'claude', model: process.env.CLAUDE_ATTACHMENTS_SMOKE_CLAUDE_MODEL || 'claude-haiku-4-5', imageCount: 3, imageWidth: 1600, imageHeight: 1100, command: async (imagePath, cwd, testCase, imagePaths) => ({ bin: 'claude', args: [ '-p', '--input-format', 'stream-json', '--output-format', 'stream-json', '--verbose', '--no-session-persistence', '--model', testCase.model, ], cwd, stdin: await buildClaudeStreamJsonPrompt(imagePaths, MULTI_IMAGE_PROMPT), }), expected: /red/i, }, { id: 'codex-native-gpt-5-4-mini', runtime: 'codex', model: 'gpt-5.4-mini', command: (imagePath, cwd) => ({ bin: 'codex', args: [ 'exec', '--json', '--skip-git-repo-check', '-C', cwd, '--model', 'gpt-5.4-mini', '--image', imagePath, '-', ], stdin: PROMPT, }), expected: /red/i, }, { id: 'codex-native-gpt-5-4-mini-multi-image', runtime: 'codex', model: 'gpt-5.4-mini', imageCount: 3, imageWidth: 1600, imageHeight: 1100, command: (imagePath, cwd, testCase, imagePaths) => ({ bin: 'codex', args: [ 'exec', '--json', '--skip-git-repo-check', '-C', cwd, '--model', 'gpt-5.4-mini', ...imagePaths.flatMap((candidate) => ['--image', candidate]), '-', ], stdin: MULTI_IMAGE_PROMPT, }), expected: /red/i, }, { id: 'opencode-openai-gpt-5-4-mini', runtime: 'opencode', model: 'openai/gpt-5.4-mini', command: (imagePath, cwd) => ({ bin: 'opencode', args: [ 'run', '--pure', '--format', 'json', '--dir', cwd, '--model', 'openai/gpt-5.4-mini', PROMPT, '-f', imagePath, ], }), expected: /red/i, }, { id: 'opencode-openrouter-kimi-k2-6-multi-image', runtime: 'opencode', model: 'openrouter/moonshotai/kimi-k2.6', envRequired: ['OPENROUTER_API_KEY'], imageCount: 3, imageWidth: 1600, imageHeight: 1100, command: (imagePath, cwd, testCase, imagePaths) => ({ bin: 'opencode', args: [ 'run', '--pure', '--format', 'json', '--dir', cwd, '--model', 'openrouter/moonshotai/kimi-k2.6', MULTI_IMAGE_PROMPT, ...imagePaths.flatMap((candidate) => ['-f', candidate]), ], }), expected: /red/i, }, { id: 'opencode-openrouter-kimi-k2-6', runtime: 'opencode', model: 'openrouter/moonshotai/kimi-k2.6', envRequired: ['OPENROUTER_API_KEY'], command: (imagePath, cwd) => ({ bin: 'opencode', args: [ 'run', '--pure', '--format', 'json', '--dir', cwd, '--model', 'openrouter/moonshotai/kimi-k2.6', PROMPT, '-f', imagePath, ], }), expected: /red/i, }, { id: 'opencode-openrouter-glm-4-5v', runtime: 'opencode', model: 'openrouter/z-ai/glm-4.5v', envRequired: ['OPENROUTER_API_KEY'], command: (imagePath, cwd) => ({ bin: 'opencode', args: [ 'run', '--pure', '--format', 'json', '--dir', cwd, '--model', 'openrouter/z-ai/glm-4.5v', PROMPT, '-f', imagePath, ], }), expected: /red/i, }, { id: 'opencode-openrouter-glm-5-1-negative', runtime: 'opencode', model: 'openrouter/z-ai/glm-5.1', envRequired: ['OPENROUTER_API_KEY'], command: (imagePath, cwd) => ({ bin: 'opencode', args: [ 'run', '--pure', '--format', 'json', '--dir', cwd, '--model', 'openrouter/z-ai/glm-5.1', PROMPT, '-f', imagePath, ], }), expectedUnsupported: true, }, ]; function crc32(bytes) { let crc = 0xffffffff; for (const byte of bytes) { crc ^= byte; for (let bit = 0; bit < 8; bit += 1) { crc = (crc >>> 1) ^ (0xedb88320 & -(crc & 1)); } } return (crc ^ 0xffffffff) >>> 0; } function chunk(type, data) { const typeBytes = Buffer.from(type, 'ascii'); const length = Buffer.alloc(4); length.writeUInt32BE(data.length, 0); const crc = Buffer.alloc(4); crc.writeUInt32BE(crc32(Buffer.concat([typeBytes, data])), 0); return Buffer.concat([length, typeBytes, data, crc]); } function createRedCardPng(width = 320, height = 240) { const raw = Buffer.alloc((width * 4 + 1) * height); for (let y = 0; y < height; y += 1) { const row = y * (width * 4 + 1); raw[row] = 0; for (let x = 0; x < width; x += 1) { const offset = row + 1 + x * 4; const marker = x > 135 && x < 185 && y > 95 && y < 145; raw[offset] = 230; raw[offset + 1] = marker ? 245 : 20; raw[offset + 2] = marker ? 245 : 20; raw[offset + 3] = 255; } } const ihdr = Buffer.alloc(13); ihdr.writeUInt32BE(width, 0); ihdr.writeUInt32BE(height, 4); ihdr[8] = 8; ihdr[9] = 6; ihdr[10] = 0; ihdr[11] = 0; ihdr[12] = 0; return Buffer.concat([ Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), chunk('IHDR', ihdr), chunk('IDAT', deflateSync(raw)), chunk('IEND', Buffer.alloc(0)), ]); } async function buildClaudeStreamJsonPrompt(imagePaths, prompt) { const imageBlocks = []; for (const imagePath of imagePaths) { const data = await readFile(imagePath, 'base64'); imageBlocks.push({ type: 'image', source: { // Claude stream-json expects image bytes inside a structured image block. // Do not replace this with base64-in-text fallback because that tests a different path. type: 'base64', media_type: 'image/png', data, }, }); } return `${JSON.stringify({ type: 'user', message: { role: 'user', content: [ { type: 'text', text: prompt }, ...imageBlocks, ], }, })}\n`; } function parseArgs(argv) { const selected = []; let all = false; let list = false; let jsonPath = null; for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; if (arg === '--case' && argv[index + 1]) { selected.push(argv[index + 1]); index += 1; } else if (arg === '--list') { list = true; } else if (arg === '--all') { all = true; } else if (arg === '--json' && argv[index + 1]) { jsonPath = argv[index + 1]; index += 1; } else { throw new Error(`Unknown or incomplete argument: ${arg}`); } } if (all && selected.length) { throw new Error('Use either --all or one or more --case arguments, not both'); } return { all, jsonPath, list, selected }; } function hasPathSeparator(value) { return value.includes('/') || value.includes('\\'); } function resolveWindowsSpawnBinary(binary) { if (process.platform !== 'win32') { return binary; } if (hasPathSeparator(binary)) { if (!path.extname(binary) && existsSync(`${binary}.cmd`)) { return `${binary}.cmd`; } return binary; } const whereResult = spawnSync('where.exe', [binary], { encoding: 'utf8', windowsHide: true, }); if (whereResult.status !== 0 || !whereResult.stdout) { return binary; } const candidates = whereResult.stdout .split(/\r?\n/) .map((candidate) => candidate.trim()) .filter(Boolean); const extensionlessShim = candidates.find( (candidate) => !path.extname(candidate) && existsSync(`${candidate}.cmd`) ); if (extensionlessShim) { return `${extensionlessShim}.cmd`; } return ( candidates.find((candidate) => /\.exe$/i.test(candidate)) ?? candidates.find((candidate) => /\.(?:cmd|bat)$/i.test(candidate)) ?? candidates[0] ?? binary ); } function quoteWindowsCmdArg(value) { const text = String(value); if (text.length === 0) { return '""'; } if (!/[ \t\r\n"&|<>^()%!]/.test(text)) { return text; } return `"${text.replace(/%/g, '%%').replace(/(["^&|<>])/g, '^$1')}"`; } function buildSpawnInvocation(command) { if (process.platform !== 'win32') { return { bin: command.bin, args: command.args, options: { windowsHide: true }, }; } const resolvedBin = resolveWindowsSpawnBinary(command.bin); if (/\.(?:cmd|bat)$/i.test(resolvedBin)) { const commandLine = [resolvedBin, ...command.args].map(quoteWindowsCmdArg).join(' '); return { bin: process.env.ComSpec || 'cmd.exe', args: ['/d', '/s', '/c', commandLine], options: { windowsHide: true, windowsVerbatimArguments: true }, }; } return { bin: resolvedBin, args: command.args, options: { windowsHide: true }, }; } function runCommand(command) { return new Promise((resolve) => { const spawnInvocation = buildSpawnInvocation(command); const child = spawn(spawnInvocation.bin, spawnInvocation.args, { stdio: ['pipe', 'pipe', 'pipe'], env: process.env, cwd: command.cwd, ...spawnInvocation.options, }); let stdout = ''; let stderr = ''; const timer = setTimeout(() => { child.kill('SIGTERM'); resolve({ ok: false, timedOut: true, exitCode: null, stdout, stderr }); }, TIMEOUT_MS); child.stdout.on('data', (chunk) => { stdout += String(chunk); }); child.stderr.on('data', (chunk) => { stderr += String(chunk); }); child.on('error', (error) => { clearTimeout(timer); resolve({ ok: false, timedOut: false, exitCode: null, stdout, stderr: error.message }); }); child.on('close', (code) => { clearTimeout(timer); resolve({ ok: code === 0, timedOut: false, exitCode: code, stdout, stderr }); }); if (command.stdin) { child.stdin.end(command.stdin); } else { child.stdin.end(); } }); } function redactSmokeText(value) { let redacted = value .replace(/(data:image\/[a-z0-9.+-]+;base64,)[a-z0-9+/=]+/gi, '$1[redacted]') .replace(/("[Dd]ata"\s*:\s*")[a-z0-9+/=]{80,}(")/g, '$1[redacted]$2') .replace(/("[Ss]ignature"\s*:\s*")[^"]{80,}(")/g, '$1[redacted]$2') .replace(/\b[A-Za-z0-9+/]{400,}={0,2}\b/g, '[redacted long encoded payload]') .replace(/(Authorization\s*[:=]\s*Bearer\s+)[^\s"']+/gi, '$1[redacted]') .replace(/(Bearer\s+)[A-Za-z0-9._~+/=-]{20,}/g, '$1[redacted]') .replace(/\b(sk-(?:ant|or|proj|live|test|codex|openai)[A-Za-z0-9._~+/=-]{12,})\b/g, '[redacted api key]'); for (const [name, secret] of Object.entries(process.env)) { if (!secret || secret.length < 8) continue; if (!/(API[_-]?KEY|TOKEN|SECRET|AUTH|PASSWORD|OPENROUTER|ANTHROPIC|OPENAI|CODEX)/i.test(name)) { continue; } redacted = redacted.split(secret).join(`[redacted ${name}]`); } return redacted; } function extractAssistantTextFromJsonLine(line) { let parsed; try { parsed = JSON.parse(line); } catch { return null; } if (typeof parsed.result === 'string') { return parsed.result; } if (parsed.type === 'item.completed' && typeof parsed.item?.text === 'string') { return parsed.item.text; } if (parsed.type === 'text' && typeof parsed.part?.text === 'string') { return parsed.part.text; } const content = parsed.message?.content; if (Array.isArray(content)) { return content .map((part) => (part?.type === 'text' && typeof part.text === 'string' ? part.text : '')) .join('\n') .trim(); } return null; } function extractAssistantText(output) { const texts = []; for (const line of output.split(/\r?\n/)) { const trimmed = line.trim(); if (!trimmed.startsWith('{')) continue; const text = extractAssistantTextFromJsonLine(trimmed); if (text?.trim()) { texts.push(text.trim()); } } return texts.join('\n').trim(); } async function main() { const args = parseArgs(process.argv.slice(2)); if (args.list) { console.log(CASES.map((testCase) => testCase.id).join('\n')); return; } const selected = args.all || !args.selected.length ? CASES : CASES.filter((testCase) => args.selected.includes(testCase.id)); const missing = args.selected.filter((id) => !CASES.some((testCase) => testCase.id === id)); if (missing.length) { throw new Error(`Unknown smoke case: ${missing.join(', ')}`); } const cwd = await mkdtemp(path.join(tmpdir(), 'agent-attachments-smoke-')); await mkdir(cwd, { recursive: true }); const results = []; const imagePathsByCase = {}; for (const testCase of selected) { const missingEnv = (testCase.envRequired ?? []).filter((name) => !process.env[name]); if (missingEnv.length) { results.push({ id: testCase.id, runtime: testCase.runtime, model: testCase.model, status: 'skipped', reason: `missing env: ${missingEnv.join(', ')}`, }); continue; } const imagePaths = await prepareSmokeImages(cwd, testCase); imagePathsByCase[testCase.id] = imagePaths; const command = await testCase.command(imagePaths[0], cwd, testCase, imagePaths); const result = await runCommand(command); const output = `${result.stdout}\n${result.stderr}`; const assistantText = extractAssistantText(output); const matched = testCase.expected ? result.ok && assistantText.length > 0 && testCase.expected.test(assistantText) : false; const unsupportedMatched = testCase.expectedUnsupported ? result.ok && assistantText.length > 0 && /cannot|unable|unsupported|text-only|vision|image|не могу|не поддерживает|изображен/i.test( assistantText ) : false; results.push({ id: testCase.id, runtime: testCase.runtime, model: testCase.model, status: (testCase.expectedUnsupported ? unsupportedMatched : result.ok && matched) ? 'passed' : 'failed', exitCode: result.exitCode, timedOut: result.timedOut, assistantText: redactSmokeText(assistantText.slice(-1000)), stdoutTail: redactSmokeText(result.stdout.slice(-4000)), stderrTail: redactSmokeText(result.stderr.slice(-4000)), }); } const firstImagePath = Object.values(imagePathsByCase)[0]?.[0] ?? null; const report = { imagePath: firstImagePath, imagePathsByCase, results }; if (args.jsonPath) { await writeFile(path.resolve(args.jsonPath), `${JSON.stringify(report, null, 2)}\n`); } console.log(JSON.stringify(report, null, 2)); if (results.some((result) => result.status === 'failed')) { process.exitCode = 1; } } async function prepareSmokeImages(cwd, testCase) { const imageCount = testCase.imageCount ?? 1; const width = testCase.imageWidth ?? 320; const height = testCase.imageHeight ?? 240; const paths = []; for (let index = 0; index < imageCount; index += 1) { const imagePath = path.join(cwd, `red-card-${testCase.id}-${index + 1}.png`); await writeFile(imagePath, createRedCardPng(width, height)); paths.push(imagePath); } return paths; } main().catch((error) => { console.error(error instanceof Error ? error.message : String(error)); process.exitCode = 1; });