docs(attachments): add smoke harness
This commit is contained in:
parent
b95033b9b7
commit
cb54ac0880
2 changed files with 381 additions and 0 deletions
91
docs/team-management/agent-attachments.md
Normal file
91
docs/team-management/agent-attachments.md
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
# Agent attachments
|
||||
|
||||
This document describes the v1 attachment path for Agent Teams.
|
||||
|
||||
## Supported runtime paths
|
||||
|
||||
- Claude lead/runtime: structured stream-json content blocks.
|
||||
- Codex native: optimized app-owned image files passed with repeatable `--image <file>`.
|
||||
- OpenCode: model-gated `file` parts sent through the OpenCode session API.
|
||||
|
||||
Do not append base64 to prompt text. Base64 is only valid inside provider-native structured payloads.
|
||||
|
||||
## Current image model policy
|
||||
|
||||
- Claude: image attachments are allowed through structured image blocks.
|
||||
- Codex native: image attachments are allowed through native image args.
|
||||
- OpenCode `openai/gpt-5.4-mini`: allowed.
|
||||
- OpenCode `openrouter/moonshotai/kimi-k2.6`: allowed.
|
||||
- OpenCode `openrouter/z-ai/glm-4.5v`: allowed.
|
||||
- OpenCode `openrouter/z-ai/glm-5.1`: blocked for images.
|
||||
- Unknown OpenCode models: blocked for images until verified.
|
||||
|
||||
Text-only messages continue to work for unsupported image models.
|
||||
|
||||
## Size and optimization rules
|
||||
|
||||
The renderer optimizes images before send. The backend still validates and owns final delivery decisions.
|
||||
|
||||
- Original attachments are immutable.
|
||||
- Optimized variants are derived artifacts.
|
||||
- If optimized images exceed the runtime budget, sending must fail before provider delivery.
|
||||
- Multiple images must be delivered together or blocked together. No partial image delivery.
|
||||
|
||||
## Diagnostics rules
|
||||
|
||||
Diagnostics may include:
|
||||
|
||||
- attachment count;
|
||||
- optimized bytes;
|
||||
- target runtime and model;
|
||||
- capability decision;
|
||||
- provider/runtime error text.
|
||||
|
||||
Diagnostics must not include:
|
||||
|
||||
- base64 payloads;
|
||||
- data URLs;
|
||||
- API keys;
|
||||
- bearer tokens.
|
||||
|
||||
## Smoke tests
|
||||
|
||||
The smoke harness generates a deterministic red PNG and checks real CLI transports.
|
||||
|
||||
List cases:
|
||||
|
||||
```bash
|
||||
node scripts/smoke/agent-attachments-smoke.mjs --list
|
||||
```
|
||||
|
||||
Run Codex native:
|
||||
|
||||
```bash
|
||||
node scripts/smoke/agent-attachments-smoke.mjs --case codex-native-gpt-5-4-mini
|
||||
```
|
||||
|
||||
Run OpenCode OpenAI:
|
||||
|
||||
```bash
|
||||
node scripts/smoke/agent-attachments-smoke.mjs --case opencode-openai-gpt-5-4-mini
|
||||
```
|
||||
|
||||
Run OpenRouter cases:
|
||||
|
||||
```bash
|
||||
OPENROUTER_API_KEY=... node scripts/smoke/agent-attachments-smoke.mjs --case opencode-openrouter-kimi-k2-6
|
||||
OPENROUTER_API_KEY=... node scripts/smoke/agent-attachments-smoke.mjs --case opencode-openrouter-glm-4-5v
|
||||
OPENROUTER_API_KEY=... node scripts/smoke/agent-attachments-smoke.mjs --case opencode-openrouter-glm-5-1-negative
|
||||
```
|
||||
|
||||
The script redacts nothing from stdout/stderr tails except that it never prints generated image bytes or configured secrets.
|
||||
|
||||
## Release checklist
|
||||
|
||||
- Text-only messages still work for Claude, Codex, and OpenCode.
|
||||
- Oversized images fail before provider delivery.
|
||||
- Claude image send uses structured image blocks.
|
||||
- Codex image send uses `--image`, not prompt base64.
|
||||
- OpenCode image send is blocked for unknown/non-vision models.
|
||||
- Attachment retry reuses the same artifacts or fails loudly.
|
||||
- Copied diagnostics do not include base64 or data URLs.
|
||||
290
scripts/smoke/agent-attachments-smoke.mjs
Normal file
290
scripts/smoke/agent-attachments-smoke.mjs
Normal file
|
|
@ -0,0 +1,290 @@
|
|||
#!/usr/bin/env node
|
||||
import { spawn } from 'node:child_process';
|
||||
import { mkdir, mkdtemp, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { deflateSync } from 'node:zlib';
|
||||
|
||||
const PROMPT = 'Look at the attached image. Reply with exactly one word: red, green, or blue.';
|
||||
const TIMEOUT_MS = 90_000;
|
||||
|
||||
const CASES = [
|
||||
{
|
||||
id: 'codex-native-gpt-5-4-mini',
|
||||
runtime: 'codex',
|
||||
model: 'gpt-5.4-mini',
|
||||
command: (imagePath, cwd) => ({
|
||||
bin: 'codex',
|
||||
args: [
|
||||
'exec',
|
||||
'--json',
|
||||
'--skip-git-repo-check',
|
||||
'-C',
|
||||
cwd,
|
||||
'--model',
|
||||
'gpt-5.4-mini',
|
||||
'--image',
|
||||
imagePath,
|
||||
'-',
|
||||
],
|
||||
stdin: PROMPT,
|
||||
}),
|
||||
expected: /red/i,
|
||||
},
|
||||
{
|
||||
id: 'opencode-openai-gpt-5-4-mini',
|
||||
runtime: 'opencode',
|
||||
model: 'openai/gpt-5.4-mini',
|
||||
command: (imagePath, cwd) => ({
|
||||
bin: 'opencode',
|
||||
args: [
|
||||
'run',
|
||||
'--pure',
|
||||
'--format',
|
||||
'json',
|
||||
'--dir',
|
||||
cwd,
|
||||
'--model',
|
||||
'openai/gpt-5.4-mini',
|
||||
PROMPT,
|
||||
'-f',
|
||||
imagePath,
|
||||
],
|
||||
}),
|
||||
expected: /red/i,
|
||||
},
|
||||
{
|
||||
id: 'opencode-openrouter-kimi-k2-6',
|
||||
runtime: 'opencode',
|
||||
model: 'openrouter/moonshotai/kimi-k2.6',
|
||||
envRequired: ['OPENROUTER_API_KEY'],
|
||||
command: (imagePath, cwd) => ({
|
||||
bin: 'opencode',
|
||||
args: [
|
||||
'run',
|
||||
'--pure',
|
||||
'--format',
|
||||
'json',
|
||||
'--dir',
|
||||
cwd,
|
||||
'--model',
|
||||
'openrouter/moonshotai/kimi-k2.6',
|
||||
PROMPT,
|
||||
'-f',
|
||||
imagePath,
|
||||
],
|
||||
}),
|
||||
expected: /red/i,
|
||||
},
|
||||
{
|
||||
id: 'opencode-openrouter-glm-4-5v',
|
||||
runtime: 'opencode',
|
||||
model: 'openrouter/z-ai/glm-4.5v',
|
||||
envRequired: ['OPENROUTER_API_KEY'],
|
||||
command: (imagePath, cwd) => ({
|
||||
bin: 'opencode',
|
||||
args: [
|
||||
'run',
|
||||
'--pure',
|
||||
'--format',
|
||||
'json',
|
||||
'--dir',
|
||||
cwd,
|
||||
'--model',
|
||||
'openrouter/z-ai/glm-4.5v',
|
||||
PROMPT,
|
||||
'-f',
|
||||
imagePath,
|
||||
],
|
||||
}),
|
||||
expected: /red/i,
|
||||
},
|
||||
{
|
||||
id: 'opencode-openrouter-glm-5-1-negative',
|
||||
runtime: 'opencode',
|
||||
model: 'openrouter/z-ai/glm-5.1',
|
||||
envRequired: ['OPENROUTER_API_KEY'],
|
||||
command: (imagePath, cwd) => ({
|
||||
bin: 'opencode',
|
||||
args: [
|
||||
'run',
|
||||
'--pure',
|
||||
'--format',
|
||||
'json',
|
||||
'--dir',
|
||||
cwd,
|
||||
'--model',
|
||||
'openrouter/z-ai/glm-5.1',
|
||||
PROMPT,
|
||||
'-f',
|
||||
imagePath,
|
||||
],
|
||||
}),
|
||||
expectedUnsupported: true,
|
||||
},
|
||||
];
|
||||
|
||||
function crc32(bytes) {
|
||||
let crc = 0xffffffff;
|
||||
for (const byte of bytes) {
|
||||
crc ^= byte;
|
||||
for (let bit = 0; bit < 8; bit += 1) {
|
||||
crc = (crc >>> 1) ^ (0xedb88320 & -(crc & 1));
|
||||
}
|
||||
}
|
||||
return (crc ^ 0xffffffff) >>> 0;
|
||||
}
|
||||
|
||||
function chunk(type, data) {
|
||||
const typeBytes = Buffer.from(type, 'ascii');
|
||||
const length = Buffer.alloc(4);
|
||||
length.writeUInt32BE(data.length, 0);
|
||||
const crc = Buffer.alloc(4);
|
||||
crc.writeUInt32BE(crc32(Buffer.concat([typeBytes, data])), 0);
|
||||
return Buffer.concat([length, typeBytes, data, crc]);
|
||||
}
|
||||
|
||||
function createRedCardPng(width = 320, height = 240) {
|
||||
const raw = Buffer.alloc((width * 4 + 1) * height);
|
||||
for (let y = 0; y < height; y += 1) {
|
||||
const row = y * (width * 4 + 1);
|
||||
raw[row] = 0;
|
||||
for (let x = 0; x < width; x += 1) {
|
||||
const offset = row + 1 + x * 4;
|
||||
const marker = x > 135 && x < 185 && y > 95 && y < 145;
|
||||
raw[offset] = 230;
|
||||
raw[offset + 1] = marker ? 245 : 20;
|
||||
raw[offset + 2] = marker ? 245 : 20;
|
||||
raw[offset + 3] = 255;
|
||||
}
|
||||
}
|
||||
const ihdr = Buffer.alloc(13);
|
||||
ihdr.writeUInt32BE(width, 0);
|
||||
ihdr.writeUInt32BE(height, 4);
|
||||
ihdr[8] = 8;
|
||||
ihdr[9] = 6;
|
||||
ihdr[10] = 0;
|
||||
ihdr[11] = 0;
|
||||
ihdr[12] = 0;
|
||||
return Buffer.concat([
|
||||
Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]),
|
||||
chunk('IHDR', ihdr),
|
||||
chunk('IDAT', deflateSync(raw)),
|
||||
chunk('IEND', Buffer.alloc(0)),
|
||||
]);
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const selected = [];
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === '--case' && argv[index + 1]) {
|
||||
selected.push(argv[index + 1]);
|
||||
index += 1;
|
||||
} else if (arg === '--list') {
|
||||
return { list: true, selected };
|
||||
}
|
||||
}
|
||||
return { list: false, selected };
|
||||
}
|
||||
|
||||
function runCommand(command) {
|
||||
return new Promise((resolve) => {
|
||||
const child = spawn(command.bin, command.args, {
|
||||
stdio: ['pipe', 'pipe', 'pipe'],
|
||||
env: process.env,
|
||||
});
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
const timer = setTimeout(() => {
|
||||
child.kill('SIGTERM');
|
||||
resolve({ ok: false, timedOut: true, exitCode: null, stdout, stderr });
|
||||
}, TIMEOUT_MS);
|
||||
child.stdout.on('data', (chunk) => {
|
||||
stdout += String(chunk);
|
||||
});
|
||||
child.stderr.on('data', (chunk) => {
|
||||
stderr += String(chunk);
|
||||
});
|
||||
child.on('error', (error) => {
|
||||
clearTimeout(timer);
|
||||
resolve({ ok: false, timedOut: false, exitCode: null, stdout, stderr: error.message });
|
||||
});
|
||||
child.on('close', (code) => {
|
||||
clearTimeout(timer);
|
||||
resolve({ ok: code === 0, timedOut: false, exitCode: code, stdout, stderr });
|
||||
});
|
||||
if (command.stdin) {
|
||||
child.stdin.end(command.stdin);
|
||||
} else {
|
||||
child.stdin.end();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
if (args.list) {
|
||||
console.log(CASES.map((testCase) => testCase.id).join('\n'));
|
||||
return;
|
||||
}
|
||||
|
||||
const selected = args.selected.length
|
||||
? CASES.filter((testCase) => args.selected.includes(testCase.id))
|
||||
: CASES;
|
||||
const missing = args.selected.filter((id) => !CASES.some((testCase) => testCase.id === id));
|
||||
if (missing.length) {
|
||||
throw new Error(`Unknown smoke case: ${missing.join(', ')}`);
|
||||
}
|
||||
|
||||
const cwd = await mkdtemp(path.join(tmpdir(), 'agent-attachments-smoke-'));
|
||||
await mkdir(cwd, { recursive: true });
|
||||
const imagePath = path.join(cwd, 'red-card.png');
|
||||
await writeFile(imagePath, createRedCardPng());
|
||||
|
||||
const results = [];
|
||||
for (const testCase of selected) {
|
||||
const missingEnv = (testCase.envRequired ?? []).filter((name) => !process.env[name]);
|
||||
if (missingEnv.length) {
|
||||
results.push({
|
||||
id: testCase.id,
|
||||
runtime: testCase.runtime,
|
||||
model: testCase.model,
|
||||
status: 'skipped',
|
||||
reason: `missing env: ${missingEnv.join(', ')}`,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const command = testCase.command(imagePath, cwd);
|
||||
const result = await runCommand(command);
|
||||
const output = `${result.stdout}\n${result.stderr}`;
|
||||
const matched = testCase.expected ? testCase.expected.test(output) : false;
|
||||
const unsupportedMatched = testCase.expectedUnsupported
|
||||
? /cannot|unable|unsupported|text-only|vision|image/i.test(output)
|
||||
: false;
|
||||
results.push({
|
||||
id: testCase.id,
|
||||
runtime: testCase.runtime,
|
||||
model: testCase.model,
|
||||
status:
|
||||
(testCase.expectedUnsupported ? unsupportedMatched : result.ok && matched)
|
||||
? 'passed'
|
||||
: 'failed',
|
||||
exitCode: result.exitCode,
|
||||
timedOut: result.timedOut,
|
||||
stdoutTail: result.stdout.slice(-4000),
|
||||
stderrTail: result.stderr.slice(-4000),
|
||||
});
|
||||
}
|
||||
|
||||
console.log(JSON.stringify({ imagePath, results }, null, 2));
|
||||
if (results.some((result) => result.status === 'failed')) {
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(error instanceof Error ? error.message : String(error));
|
||||
process.exitCode = 1;
|
||||
});
|
||||
Loading…
Reference in a new issue