fix(runtime-provider-management): keep filtered provider selection explicit
This commit is contained in:
777genius
parent
41ca9fc0cb
commit
951c52a5d2
8 changed files with 1564 additions and 341 deletions
|
|
@ -1142,11 +1142,6 @@ export function RuntimeProviderManagementPanelView({
|
|||
.includes(providerQuery)
|
||||
)
|
||||
: state.providers;
|
||||
const selectedProviderId = filteredProviders.some(
|
||||
(provider) => provider.providerId === state.selectedProviderId
|
||||
)
|
||||
? state.selectedProviderId
|
||||
: (filteredProviders[0]?.providerId ?? state.selectedProviderId ?? null);
|
||||
const canSearchDirectory =
|
||||
state.directorySupported && providerQuery.length >= 2 && filteredProviders.length === 0;
|
||||
|
||||
|
|
@ -1256,7 +1251,7 @@ export function RuntimeProviderManagementPanelView({
|
|||
key={provider.providerId}
|
||||
provider={provider}
|
||||
state={state}
|
||||
active={provider.providerId === selectedProviderId}
|
||||
active={provider.providerId === state.selectedProviderId}
|
||||
formOpen={state.activeFormProviderId === provider.providerId}
|
||||
apiKeyValue={state.apiKeyValue}
|
||||
busy={state.savingProviderId === provider.providerId}
|
||||
|
|
|
|||
113
test/fixtures/team/opencode/semantic-realistic-scenario.json
vendored
Normal file
113
test/fixtures/team/opencode/semantic-realistic-scenario.json
vendored
Normal file
|
|
@ -0,0 +1,113 @@
|
|||
{
|
||||
"teamNamePrefix": "opencode-semantic-realistic",
|
||||
"displayName": "OpenCode Semantic Realistic E2E",
|
||||
"description": "Realistic Agent Teams OpenCode scenario for prompt contract and live model checks.",
|
||||
"teamPromptLines": [
|
||||
"You are an Agent Teams crew working on a production desktop app that coordinates multiple AI teammates.",
|
||||
"The project has an Electron renderer, main-process services, OpenCode runtime bridges, MCP tools, task boards, and persistent message inboxes.",
|
||||
"Your launch goal is to inspect a regression where OpenCode teammates receive app-delivered messages, but replies must be visible through the Agent Teams MCP message_send tool.",
|
||||
"Treat the following constraints as production requirements: do not invent local team files, do not use SendMessage in OpenCode, do not answer only as plain assistant text when app MCP tools are available, and preserve taskRefs metadata when messages are tied to tasks.",
|
||||
"The team must be careful with long context. You should preserve identity, task ownership, reply routing, and exact user-visible reply tokens even when the inbound prompt contains project context, prior notes, and task descriptions.",
|
||||
"Known risk areas include stale runtime sessions, OpenCode provider catalogs, app-managed OpenRouter credentials, delayed runtime delivery, idle acknowledgements, fake task labels such as #00000000, and peer relay between OpenCode members.",
|
||||
"When responding to the app Messages UI, use concise natural language in the text field and structured metadata in the MCP tool fields.",
|
||||
"If you need to coordinate with another teammate, send a real team message to that teammate rather than pretending to have delegated work.",
|
||||
"If a task ref is present, keep the visible #displayId text human-readable and include matching taskRefs metadata in the tool call.",
|
||||
"Do not send readiness-only or idle acknowledgement messages like understood, ready, or no tasks unless the app explicitly asks for that visible status.",
|
||||
"Primary files likely involved: TeamProvisioningService, OpenCodeTeamRuntimeAdapter, OpenCodeBridgeCommandHandler, runtime provider management, and Messages UI components.",
|
||||
"Acceptance criteria: OpenCode members bootstrap silently, receive delivery prompts, reply through message_send, relay peer messages, and never synthesize fake task ids.",
|
||||
"This scenario intentionally includes enough context to stress instruction retention without relying on a synthetic wall of unrelated text.",
|
||||
"The test should remain deterministic in dry mode and opt-in for live model execution."
|
||||
],
|
||||
"members": [
|
||||
{
|
||||
"name": "bob",
|
||||
"role": "Developer",
|
||||
"workflowLines": [
|
||||
"Own OpenCode runtime delivery and direct app message replies.",
|
||||
"When assigned a task-linked message, preserve taskRefs and summarize concrete findings.",
|
||||
"Do not answer with plain assistant text when agent-teams_message_send is available."
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "jack",
|
||||
"role": "Reviewer",
|
||||
"workflowLines": [
|
||||
"Review peer relay behavior and verify that teammate-to-teammate messages are delivered through app inboxes.",
|
||||
"Reply to the app user only after receiving an explicit relayed instruction.",
|
||||
"Never prefix messages with fake task labels."
|
||||
]
|
||||
}
|
||||
],
|
||||
"projectFiles": [
|
||||
{
|
||||
"path": "AGENTS.md",
|
||||
"contentLines": [
|
||||
"# Test Project Instructions",
|
||||
"Use Agent Teams MCP tools for team messages.",
|
||||
"OpenCode teammates must call agent-teams_message_send for visible replies.",
|
||||
"Preserve taskRefs on task-linked updates."
|
||||
]
|
||||
},
|
||||
{
|
||||
"path": "src/runtimeDelivery.ts",
|
||||
"contentLines": [
|
||||
"export function normalizeDelivery(input: { text: string; taskRefs?: unknown[] }) {",
|
||||
" return { text: input.text.trim(), taskRefs: input.taskRefs ?? [] };",
|
||||
"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"path": "src/providerCatalog.ts",
|
||||
"contentLines": [
|
||||
"export const providerCatalog = ['openrouter/qwen/qwen3-coder', 'openrouter/minimax/minimax-m2.5'];"
|
||||
]
|
||||
}
|
||||
],
|
||||
"tasks": [
|
||||
{
|
||||
"taskId": "task-59560c95-runtime-delivery",
|
||||
"displayId": "59560c95",
|
||||
"subject": "Verify OpenCode app runtime delivery preserves taskRefs",
|
||||
"owner": "bob",
|
||||
"comment": "Check that a direct app-delivered message is acknowledged through message_send with matching taskRefs metadata."
|
||||
},
|
||||
{
|
||||
"taskId": "task-3375c939-peer-relay",
|
||||
"displayId": "3375c939",
|
||||
"subject": "Verify OpenCode peer relay reaches recipient runtime",
|
||||
"owner": "jack",
|
||||
"comment": "Check that a message from bob to jack is relayed and jack replies to the app user through message_send."
|
||||
}
|
||||
],
|
||||
"directDelivery": {
|
||||
"memberName": "bob",
|
||||
"replyRecipient": "user",
|
||||
"actionMode": "ask",
|
||||
"taskIndex": 0,
|
||||
"expectedReplyToken": "OPENCODE_DIRECT_REALISTIC_OK",
|
||||
"textLines": [
|
||||
"Investigate the runtime delivery contract for task #59560c95.",
|
||||
"Reply to the app user with exactly this token in your message text: OPENCODE_DIRECT_REALISTIC_OK.",
|
||||
"Use agent-teams_message_send with to=\"user\" and from=\"bob\".",
|
||||
"Include taskRefs metadata for task #59560c95 if the tool exposes taskRefs.",
|
||||
"Do not answer only as plain assistant text.",
|
||||
"Do not use SendMessage.",
|
||||
"Do not prefix the reply with #00000000."
|
||||
]
|
||||
},
|
||||
"peerDelivery": {
|
||||
"senderName": "bob",
|
||||
"recipientName": "jack",
|
||||
"replyRecipient": "jack",
|
||||
"actionMode": "delegate",
|
||||
"taskIndex": 1,
|
||||
"peerToken": "OPENCODE_PEER_RELAY_INBOX_OK",
|
||||
"expectedReplyToken": "OPENCODE_PEER_RELAY_USER_OK",
|
||||
"textLines": [
|
||||
"Send one team message to jack about task #3375c939.",
|
||||
"The exact teammate message must include OPENCODE_PEER_RELAY_INBOX_OK and ask jack to reply to the app user with OPENCODE_PEER_RELAY_USER_OK.",
|
||||
"Use agent-teams_message_send with to=\"jack\", from=\"bob\", and summary=\"peer relay\".",
|
||||
"Do not reply to user instead of sending the team message."
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,157 @@
|
|||
import { promises as fs } from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
|
||||
import { TeamProvisioningService } from '../../../../src/main/services/team/TeamProvisioningService';
|
||||
import { TeamRuntimeAdapterRegistry } from '../../../../src/main/services/team/runtime';
|
||||
import { setClaudeBasePathOverride } from '../../../../src/main/utils/pathDecoder';
|
||||
import {
|
||||
buildOpenCodeScenarioTeamRequest,
|
||||
buildScenarioRuntimeMessageInput,
|
||||
CapturingOpenCodeRuntimeAdapter,
|
||||
createCapturingOpenCodeBridge,
|
||||
createOpenCodeRuntimeAdapterFromCapture,
|
||||
dumpOpenCodePromptArtifacts,
|
||||
loadOpenCodeSemanticScenario,
|
||||
materializeOpenCodeScenarioProject,
|
||||
materializeOpenCodeScenarioTasks,
|
||||
} from './openCodeSemanticScenarioHarness';
|
||||
|
||||
describe('OpenCode production prompt artifacts safe e2e', () => {
|
||||
let tempDir: string;
|
||||
let tempClaudeRoot: string;
|
||||
let projectPath: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'opencode-production-prompts-safe-e2e-'));
|
||||
tempClaudeRoot = path.join(tempDir, '.claude');
|
||||
projectPath = path.join(tempDir, 'project');
|
||||
await fs.mkdir(tempClaudeRoot, { recursive: true });
|
||||
await fs.mkdir(projectPath, { recursive: true });
|
||||
setClaudeBasePathOverride(tempClaudeRoot);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
setClaudeBasePathOverride(null);
|
||||
await fs.rm(tempDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('builds realistic OpenCode launch and runtime delivery prompts through production paths', async () => {
|
||||
const scenario = await loadOpenCodeSemanticScenario();
|
||||
await materializeOpenCodeScenarioProject(scenario, projectPath);
|
||||
|
||||
const selectedModel = 'openrouter/qwen/qwen3-coder';
|
||||
const teamName = `${scenario.teamNamePrefix}-dry`;
|
||||
const captureAdapter = new CapturingOpenCodeRuntimeAdapter();
|
||||
const service = new TeamProvisioningService();
|
||||
service.setRuntimeAdapterRegistry(new TeamRuntimeAdapterRegistry([captureAdapter]));
|
||||
|
||||
await service.createTeam(
|
||||
buildOpenCodeScenarioTeamRequest({
|
||||
scenario,
|
||||
teamName,
|
||||
projectPath,
|
||||
model: selectedModel,
|
||||
}),
|
||||
() => undefined
|
||||
);
|
||||
await materializeOpenCodeScenarioTasks({ scenario, teamName, projectPath });
|
||||
|
||||
expect(captureAdapter.launchInputs).toHaveLength(1);
|
||||
const launchInput = captureAdapter.launchInputs[0];
|
||||
expect(launchInput).toBeDefined();
|
||||
expect(launchInput?.prompt ?? '').toContain('production desktop app');
|
||||
expect(launchInput?.expectedMembers.map((member) => member.name)).toEqual(['bob', 'jack']);
|
||||
expect(launchInput?.prompt?.length ?? 0).toBeGreaterThan(1_500);
|
||||
|
||||
const bridgeCapture = createCapturingOpenCodeBridge(selectedModel);
|
||||
const realAdapter = createOpenCodeRuntimeAdapterFromCapture(bridgeCapture);
|
||||
await expect(realAdapter.launch(launchInput!)).resolves.toMatchObject({
|
||||
teamLaunchState: 'clean_success',
|
||||
});
|
||||
|
||||
expect(bridgeCapture.launchCommands).toHaveLength(1);
|
||||
const launchCommand = bridgeCapture.launchCommands[0];
|
||||
expect(launchCommand?.leadPrompt).toContain('Known risk areas include stale runtime sessions');
|
||||
expect(launchCommand?.leadPrompt).toContain('OpenCode members bootstrap silently');
|
||||
expect(launchCommand?.leadPrompt.length ?? 0).toBeGreaterThan(1_500);
|
||||
expect(launchCommand?.leadPrompt.length ?? 0).toBeLessThan(80_000);
|
||||
expect(launchCommand?.members.map((member) => member.name)).toEqual(['bob', 'jack']);
|
||||
|
||||
for (const member of launchCommand?.members ?? []) {
|
||||
expect(member.prompt).toContain(`You are ${member.name}`);
|
||||
expect(member.prompt).toContain('Team launch context:');
|
||||
expect(member.prompt).toContain('agent-teams_member_briefing');
|
||||
expect(member.prompt).toContain('"runtimeProvider": "opencode"');
|
||||
expect(member.prompt).toContain('agent-teams_message_send');
|
||||
expect(member.prompt).toContain('Launch bootstrap is a silent attach');
|
||||
expect(member.prompt).toContain('stay idle silently');
|
||||
expect(member.prompt).not.toContain('Call SendMessage');
|
||||
expect(member.prompt).not.toContain('Use SendMessage');
|
||||
expect(member.prompt.length).toBeGreaterThan(2_000);
|
||||
expect(member.prompt.length).toBeLessThan(100_000);
|
||||
}
|
||||
|
||||
await expect(
|
||||
realAdapter.sendMessageToMember(
|
||||
buildScenarioRuntimeMessageInput({
|
||||
scenario,
|
||||
teamName,
|
||||
projectPath,
|
||||
runId: launchInput?.runId,
|
||||
kind: 'direct',
|
||||
})
|
||||
)
|
||||
).resolves.toMatchObject({ ok: true });
|
||||
await expect(
|
||||
realAdapter.sendMessageToMember(
|
||||
buildScenarioRuntimeMessageInput({
|
||||
scenario,
|
||||
teamName,
|
||||
projectPath,
|
||||
runId: launchInput?.runId,
|
||||
kind: 'peer',
|
||||
})
|
||||
)
|
||||
).resolves.toMatchObject({ ok: true });
|
||||
|
||||
expect(bridgeCapture.messageCommands).toHaveLength(2);
|
||||
const [directCommand, peerCommand] = bridgeCapture.messageCommands;
|
||||
expect(directCommand?.text).toContain('Use teamName="opencode-semantic-realistic-dry"');
|
||||
expect(directCommand?.text).toContain('to="user"');
|
||||
expect(directCommand?.text).toContain('from="bob"');
|
||||
expect(directCommand?.text).toContain('Include source="runtime_delivery"');
|
||||
expect(directCommand?.text).toContain('Include relayOfMessageId="semantic-direct-');
|
||||
expect(directCommand?.text).toContain('Action mode for this message: ask.');
|
||||
expect(directCommand?.text).toContain('"displayId":"59560c95"');
|
||||
expect(directCommand?.text).toContain('Do not use SendMessage or runtime_deliver_message');
|
||||
expect(directCommand?.text).toContain('never use #00000000');
|
||||
expect(directCommand?.taskRefs).toEqual([
|
||||
{ taskId: 'task-59560c95-runtime-delivery', displayId: '59560c95', teamName },
|
||||
]);
|
||||
|
||||
expect(peerCommand?.text).toContain('to="jack"');
|
||||
expect(peerCommand?.text).toContain('from="bob"');
|
||||
expect(peerCommand?.text).toContain('Action mode for this message: delegate.');
|
||||
expect(peerCommand?.text).toContain('"displayId":"3375c939"');
|
||||
expect(peerCommand?.taskRefs).toEqual([
|
||||
{ taskId: 'task-3375c939-peer-relay', displayId: '3375c939', teamName },
|
||||
]);
|
||||
|
||||
if (process.env.OPENCODE_E2E_DUMP_PROMPTS === '1') {
|
||||
await dumpOpenCodePromptArtifacts({
|
||||
outputDir: path.join(
|
||||
process.cwd(),
|
||||
'test-results',
|
||||
'opencode-semantic-prompts',
|
||||
teamName
|
||||
),
|
||||
launchInput: launchInput!,
|
||||
launchCommand: launchCommand!,
|
||||
messageCommands: bridgeCapture.messageCommands,
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
@ -1,38 +1,21 @@
|
|||
import { constants as fsConstants, promises as fs } from 'fs';
|
||||
import { promises as fs } from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
|
||||
import Fastify from 'fastify';
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
|
||||
import { registerTeamRoutes } from '../../../../src/main/http/teams';
|
||||
import { OpenCodeBridgeCommandClient } from '../../../../src/main/services/team/opencode/bridge/OpenCodeBridgeCommandClient';
|
||||
import {
|
||||
createOpenCodeBridgeCommandLeaseStore,
|
||||
createOpenCodeBridgeCommandLedgerStore,
|
||||
} from '../../../../src/main/services/team/opencode/bridge/OpenCodeBridgeCommandLedgerStore';
|
||||
import {
|
||||
createOpenCodeBridgeClientIdentity,
|
||||
OpenCodeBridgeCommandHandshakePort,
|
||||
} from '../../../../src/main/services/team/opencode/bridge/OpenCodeBridgeHandshakeClient';
|
||||
import { OpenCodeReadinessBridge } from '../../../../src/main/services/team/opencode/bridge/OpenCodeReadinessBridge';
|
||||
import { OpenCodeStateChangingBridgeCommandService } from '../../../../src/main/services/team/opencode/bridge/OpenCodeStateChangingBridgeCommandService';
|
||||
import { readOpenCodeRuntimeLaneIndex } from '../../../../src/main/services/team/opencode/store/OpenCodeRuntimeManifestEvidenceReader';
|
||||
import { applyOpenCodeAutoUpdatePolicy } from '../../../../src/main/services/runtime/openCodeAutoUpdatePolicy';
|
||||
import { OpenCodeTeamRuntimeAdapter } from '../../../../src/main/services/team/runtime/OpenCodeTeamRuntimeAdapter';
|
||||
import { TeamRuntimeAdapterRegistry } from '../../../../src/main/services/team/runtime/TeamRuntimeAdapter';
|
||||
import { resolveAgentTeamsMcpLaunchSpec } from '../../../../src/main/services/team/TeamMcpConfigBuilder';
|
||||
import { TeamProvisioningService } from '../../../../src/main/services/team/TeamProvisioningService';
|
||||
import { getTeamsBasePath, setClaudeBasePathOverride } from '../../../../src/main/utils/pathDecoder';
|
||||
import {
|
||||
getClaudeBasePath,
|
||||
getTeamsBasePath,
|
||||
setClaudeBasePathOverride,
|
||||
} from '../../../../src/main/utils/pathDecoder';
|
||||
createOpenCodeLiveHarness,
|
||||
getRuntimeTranscript,
|
||||
type InboxMessage,
|
||||
waitForMemberInboxMessage,
|
||||
waitForOpenCodeLanesStopped,
|
||||
waitForOpenCodePeerRelay,
|
||||
waitForUserInboxReply,
|
||||
} from './openCodeLiveTestHarness';
|
||||
|
||||
import type { HttpServices } from '../../../../src/main/http';
|
||||
import type { OpenCodeBridgeCommandExecutor } from '../../../../src/main/services/team/opencode/bridge/OpenCodeStateChangingBridgeCommandService';
|
||||
import type { RuntimeStoreManifestEvidence } from '../../../../src/main/services/team/opencode/bridge/OpenCodeBridgeCommandContract';
|
||||
import type { RuntimeStoreManifestReader } from '../../../../src/main/services/team/opencode/bridge/OpenCodeStateChangingBridgeCommandService';
|
||||
import type { TeamProvisioningProgress } from '../../../../src/shared/types';
|
||||
|
||||
const liveDescribe =
|
||||
|
|
@ -41,17 +24,8 @@ const liveDescribe =
|
|||
: describe.skip;
|
||||
|
||||
const PROJECT_PATH = process.env.OPENCODE_E2E_PROJECT_PATH?.trim() || process.cwd();
|
||||
const DEFAULT_ORCHESTRATOR_CLI = '/Users/belief/dev/projects/claude/agent_teams_orchestrator/cli';
|
||||
const DEFAULT_MODEL = 'opencode/big-pickle';
|
||||
|
||||
interface InboxMessage {
|
||||
from?: string;
|
||||
to?: string;
|
||||
text?: string;
|
||||
messageId?: string;
|
||||
read?: boolean;
|
||||
}
|
||||
|
||||
liveDescribe('OpenCode semantic messaging live e2e', () => {
|
||||
let tempDir: string;
|
||||
let tempClaudeRoot: string;
|
||||
|
|
@ -71,7 +45,10 @@ liveDescribe('OpenCode semantic messaging live e2e', () => {
|
|||
it(
|
||||
'delivers a desktop message to an OpenCode member and records the reply through agent-teams_message_send',
|
||||
async () => {
|
||||
const { bridgeClient, selectedModel, svc, dispose } = await createOpenCodeLiveHarness(tempDir);
|
||||
const { bridgeClient, selectedModel, svc, dispose } = await createOpenCodeLiveHarness({
|
||||
tempDir,
|
||||
selectedModel: process.env.OPENCODE_E2E_MODEL?.trim() || DEFAULT_MODEL,
|
||||
});
|
||||
|
||||
const teamName = `opencode-semantic-message-${Date.now()}`;
|
||||
const memberName = 'bob';
|
||||
|
|
@ -152,7 +129,12 @@ liveDescribe('OpenCode semantic messaging live e2e', () => {
|
|||
try {
|
||||
reply = await waitForUserInboxReply(teamName, memberName, expectedReply, 90_000);
|
||||
} catch (error) {
|
||||
const transcript = await getRuntimeTranscript(bridgeClient, teamName, memberName);
|
||||
const transcript = await getRuntimeTranscript({
|
||||
bridgeClient,
|
||||
teamName,
|
||||
memberName,
|
||||
projectPath: PROJECT_PATH,
|
||||
});
|
||||
throw new Error(
|
||||
`${error instanceof Error ? error.message : String(error)}\nTranscript: ${JSON.stringify(
|
||||
transcript,
|
||||
|
|
@ -169,10 +151,7 @@ liveDescribe('OpenCode semantic messaging live e2e', () => {
|
|||
} finally {
|
||||
await svc.stopTeam(teamName).catch(() => undefined);
|
||||
await dispose();
|
||||
await waitUntil(async () => {
|
||||
const laneIndex = await readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName);
|
||||
return Object.keys(laneIndex.lanes).length === 0;
|
||||
}, 90_000).catch(() => undefined);
|
||||
await waitForOpenCodeLanesStopped(teamName);
|
||||
}
|
||||
},
|
||||
300_000
|
||||
|
|
@ -181,7 +160,10 @@ liveDescribe('OpenCode semantic messaging live e2e', () => {
|
|||
it(
|
||||
'relays an OpenCode teammate message into another OpenCode member runtime and records the reply',
|
||||
async () => {
|
||||
const { bridgeClient, selectedModel, svc, dispose } = await createOpenCodeLiveHarness(tempDir);
|
||||
const { bridgeClient, selectedModel, svc, dispose } = await createOpenCodeLiveHarness({
|
||||
tempDir,
|
||||
selectedModel: process.env.OPENCODE_E2E_MODEL?.trim() || DEFAULT_MODEL,
|
||||
});
|
||||
|
||||
const teamName = `opencode-peer-message-${Date.now()}`;
|
||||
const senderName = 'bob';
|
||||
|
|
@ -282,7 +264,12 @@ liveDescribe('OpenCode semantic messaging live e2e', () => {
|
|||
180_000
|
||||
);
|
||||
} catch (error) {
|
||||
const transcript = await getRuntimeTranscript(bridgeClient, teamName, senderName);
|
||||
const transcript = await getRuntimeTranscript({
|
||||
bridgeClient,
|
||||
teamName,
|
||||
memberName: senderName,
|
||||
projectPath: PROJECT_PATH,
|
||||
});
|
||||
throw new Error(
|
||||
`${error instanceof Error ? error.message : String(error)}\n${senderName} transcript: ${JSON.stringify(
|
||||
transcript,
|
||||
|
|
@ -305,8 +292,18 @@ liveDescribe('OpenCode semantic messaging live e2e', () => {
|
|||
reply = await waitForUserInboxReply(teamName, recipientName, replyToken, 120_000);
|
||||
} catch (error) {
|
||||
const [senderTranscript, recipientTranscript] = await Promise.all([
|
||||
getRuntimeTranscript(bridgeClient, teamName, senderName),
|
||||
getRuntimeTranscript(bridgeClient, teamName, recipientName),
|
||||
getRuntimeTranscript({
|
||||
bridgeClient,
|
||||
teamName,
|
||||
memberName: senderName,
|
||||
projectPath: PROJECT_PATH,
|
||||
}),
|
||||
getRuntimeTranscript({
|
||||
bridgeClient,
|
||||
teamName,
|
||||
memberName: recipientName,
|
||||
projectPath: PROJECT_PATH,
|
||||
}),
|
||||
]);
|
||||
throw new Error(
|
||||
`${error instanceof Error ? error.message : String(error)}\n${senderName} transcript: ${JSON.stringify(
|
||||
|
|
@ -324,297 +321,9 @@ liveDescribe('OpenCode semantic messaging live e2e', () => {
|
|||
} finally {
|
||||
await svc.stopTeam(teamName).catch(() => undefined);
|
||||
await dispose();
|
||||
await waitUntil(async () => {
|
||||
const laneIndex = await readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName);
|
||||
return Object.keys(laneIndex.lanes).length === 0;
|
||||
}, 90_000).catch(() => undefined);
|
||||
await waitForOpenCodeLanesStopped(teamName);
|
||||
}
|
||||
},
|
||||
360_000
|
||||
);
|
||||
});
|
||||
|
||||
async function waitForUserInboxReply(
|
||||
teamName: string,
|
||||
from: string,
|
||||
expectedText: string,
|
||||
timeoutMs: number
|
||||
): Promise<InboxMessage> {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
const inboxPath = path.join(getTeamsBasePath(), teamName, 'inboxes', 'user.json');
|
||||
let lastMessages: InboxMessage[] = [];
|
||||
|
||||
while (Date.now() < deadline) {
|
||||
lastMessages = await readInboxMessages(inboxPath);
|
||||
const match = lastMessages.find(
|
||||
(message) =>
|
||||
message.from === from &&
|
||||
message.to === 'user' &&
|
||||
typeof message.text === 'string' &&
|
||||
message.text.includes(expectedText)
|
||||
);
|
||||
if (match) {
|
||||
return match;
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 1_500));
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
`Timed out waiting for OpenCode reply in ${inboxPath}. Last messages: ${JSON.stringify(
|
||||
lastMessages,
|
||||
null,
|
||||
2
|
||||
)}`
|
||||
);
|
||||
}
|
||||
|
||||
async function waitForMemberInboxMessage(
|
||||
teamName: string,
|
||||
memberName: string,
|
||||
from: string,
|
||||
expectedText: string | string[],
|
||||
timeoutMs: number
|
||||
): Promise<InboxMessage & { messageId: string }> {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
const inboxPath = path.join(getTeamsBasePath(), teamName, 'inboxes', `${memberName}.json`);
|
||||
let lastMessages: InboxMessage[] = [];
|
||||
const expectedTexts = Array.isArray(expectedText) ? expectedText : [expectedText];
|
||||
|
||||
while (Date.now() < deadline) {
|
||||
lastMessages = await readInboxMessages(inboxPath);
|
||||
const match = lastMessages.find(
|
||||
(message): message is InboxMessage & { messageId: string; text: string } => {
|
||||
if (message.from !== from || message.to !== memberName) return false;
|
||||
if (typeof message.messageId !== 'string' || !message.messageId.trim()) return false;
|
||||
const text = message.text;
|
||||
if (typeof text !== 'string') return false;
|
||||
return expectedTexts.every((expected) => text.includes(expected));
|
||||
}
|
||||
);
|
||||
if (match) {
|
||||
return match;
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 1_500));
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
`Timed out waiting for OpenCode member message in ${inboxPath}. Last messages: ${JSON.stringify(
|
||||
lastMessages,
|
||||
null,
|
||||
2
|
||||
)}`
|
||||
);
|
||||
}
|
||||
|
||||
async function waitForOpenCodePeerRelay(
|
||||
svc: TeamProvisioningService,
|
||||
teamName: string,
|
||||
memberName: string,
|
||||
messageId: string,
|
||||
timeoutMs: number
|
||||
): Promise<void> {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
let lastRelay: Awaited<ReturnType<TeamProvisioningService['relayOpenCodeMemberInboxMessages']>> | null =
|
||||
null;
|
||||
|
||||
while (Date.now() < deadline) {
|
||||
lastRelay = await svc.relayOpenCodeMemberInboxMessages(teamName, memberName, {
|
||||
onlyMessageId: messageId,
|
||||
source: 'manual',
|
||||
deliveryMetadata: {
|
||||
replyRecipient: 'user',
|
||||
},
|
||||
});
|
||||
if (lastRelay.delivered >= 1) {
|
||||
return;
|
||||
}
|
||||
if (lastRelay.failed > 0 && lastRelay.lastDelivery?.responsePending !== true) {
|
||||
break;
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 3_000));
|
||||
}
|
||||
|
||||
throw new Error(`OpenCode peer relay failed: ${JSON.stringify(lastRelay, null, 2)}`);
|
||||
}
|
||||
|
||||
async function readInboxMessages(inboxPath: string): Promise<InboxMessage[]> {
|
||||
try {
|
||||
const parsed = JSON.parse(await fs.readFile(inboxPath, 'utf8'));
|
||||
return Array.isArray(parsed) ? (parsed as InboxMessage[]) : [];
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
|
||||
return [];
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function waitUntil(
|
||||
predicate: () => Promise<boolean>,
|
||||
timeoutMs: number,
|
||||
pollMs = 500
|
||||
): Promise<void> {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() < deadline) {
|
||||
if (await predicate()) {
|
||||
return;
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, pollMs));
|
||||
}
|
||||
throw new Error(`Timed out after ${timeoutMs}ms waiting for condition`);
|
||||
}
|
||||
|
||||
async function createOpenCodeLiveHarness(tempDir: string): Promise<{
|
||||
bridgeClient: OpenCodeBridgeCommandClient;
|
||||
selectedModel: string;
|
||||
svc: TeamProvisioningService;
|
||||
dispose: () => Promise<void>;
|
||||
}> {
|
||||
const selectedModel = process.env.OPENCODE_E2E_MODEL?.trim() || DEFAULT_MODEL;
|
||||
const orchestratorCli =
|
||||
process.env.CLAUDE_AGENT_TEAMS_ORCHESTRATOR_CLI_PATH?.trim() || DEFAULT_ORCHESTRATOR_CLI;
|
||||
await assertExecutable(orchestratorCli);
|
||||
|
||||
const svc = new TeamProvisioningService();
|
||||
const controlApi = await startLiveTeamControlApi(svc);
|
||||
svc.setControlApiBaseUrlResolver(async () => controlApi.baseUrl);
|
||||
|
||||
const mcpLaunchSpec = await resolveAgentTeamsMcpLaunchSpec();
|
||||
const bridgeEnv = {
|
||||
...createStableBridgeEnv(),
|
||||
PATH: withBunOnPath(process.env.PATH ?? ''),
|
||||
XDG_DATA_HOME: path.join(tempDir, 'xdg-data'),
|
||||
AGENT_TEAMS_MCP_CLAUDE_DIR: getClaudeBasePath(),
|
||||
CLAUDE_TEAM_CONTROL_URL: controlApi.baseUrl,
|
||||
CLAUDE_MULTIMODEL_AGENT_TEAMS_MCP_COMMAND: mcpLaunchSpec.command,
|
||||
CLAUDE_MULTIMODEL_AGENT_TEAMS_MCP_ENTRY: mcpLaunchSpec.args[0] ?? '',
|
||||
};
|
||||
const bridgeClient = new OpenCodeBridgeCommandClient({
|
||||
binaryPath: orchestratorCli,
|
||||
tempDirectory: path.join(tempDir, 'bridge-input'),
|
||||
env: bridgeEnv,
|
||||
});
|
||||
const stateChangingCommands = createStateChangingCommands({
|
||||
bridge: bridgeClient,
|
||||
controlDir: path.join(tempDir, 'control'),
|
||||
});
|
||||
const readinessBridge = new OpenCodeReadinessBridge(bridgeClient, {
|
||||
stateChangingCommands,
|
||||
timeoutMs: 180_000,
|
||||
launchTimeoutMs: 180_000,
|
||||
reconcileTimeoutMs: 90_000,
|
||||
stopTimeoutMs: 90_000,
|
||||
});
|
||||
const adapter = new OpenCodeTeamRuntimeAdapter(readinessBridge);
|
||||
svc.setRuntimeAdapterRegistry(new TeamRuntimeAdapterRegistry([adapter]));
|
||||
return {
|
||||
bridgeClient,
|
||||
selectedModel,
|
||||
svc,
|
||||
dispose: async () => {
|
||||
svc.setControlApiBaseUrlResolver(null);
|
||||
await controlApi.close();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function startLiveTeamControlApi(svc: TeamProvisioningService): Promise<{
|
||||
baseUrl: string;
|
||||
close: () => Promise<void>;
|
||||
}> {
|
||||
const app = Fastify({ logger: false });
|
||||
registerTeamRoutes(app, {
|
||||
teamProvisioningService: svc,
|
||||
} as HttpServices);
|
||||
await app.listen({ host: '127.0.0.1', port: 0 });
|
||||
const address = app.server.address();
|
||||
if (!address || typeof address === 'string') {
|
||||
await app.close();
|
||||
throw new Error('Failed to start live team control API');
|
||||
}
|
||||
|
||||
return {
|
||||
baseUrl: `http://127.0.0.1:${address.port}`,
|
||||
close: async () => {
|
||||
await app.close();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function getRuntimeTranscript(
|
||||
bridgeClient: OpenCodeBridgeCommandClient,
|
||||
teamName: string,
|
||||
memberName: string
|
||||
): Promise<unknown> {
|
||||
return bridgeClient
|
||||
.execute<
|
||||
{ teamId: string; teamName: string; laneId: string; memberName: string },
|
||||
{ logProjection?: { messages?: unknown[] }; messages?: unknown[] }
|
||||
>(
|
||||
'opencode.getRuntimeTranscript',
|
||||
{ teamId: teamName, teamName, laneId: 'primary', memberName },
|
||||
{ cwd: PROJECT_PATH, timeoutMs: 60_000 }
|
||||
)
|
||||
.catch((transcriptError) => ({
|
||||
ok: false as const,
|
||||
error: String(transcriptError),
|
||||
}));
|
||||
}
|
||||
|
||||
function createStateChangingCommands(input: {
|
||||
bridge: OpenCodeBridgeCommandExecutor;
|
||||
controlDir: string;
|
||||
}): OpenCodeStateChangingBridgeCommandService {
|
||||
const clientIdentity = createOpenCodeBridgeClientIdentity({
|
||||
appVersion: '1.3.0-e2e',
|
||||
gitSha: null,
|
||||
buildId: 'opencode-semantic-message-e2e',
|
||||
});
|
||||
|
||||
return new OpenCodeStateChangingBridgeCommandService({
|
||||
expectedClientIdentity: clientIdentity,
|
||||
handshakePort: new OpenCodeBridgeCommandHandshakePort({
|
||||
bridge: input.bridge,
|
||||
clientIdentity,
|
||||
}),
|
||||
leaseStore: createOpenCodeBridgeCommandLeaseStore({
|
||||
filePath: path.join(input.controlDir, 'leases.json'),
|
||||
}),
|
||||
ledger: createOpenCodeBridgeCommandLedgerStore({
|
||||
filePath: path.join(input.controlDir, 'ledger.json'),
|
||||
}),
|
||||
bridge: input.bridge,
|
||||
manifestReader: new StaticManifestReader(),
|
||||
});
|
||||
}
|
||||
|
||||
class StaticManifestReader implements RuntimeStoreManifestReader {
|
||||
async read(): Promise<RuntimeStoreManifestEvidence> {
|
||||
return {
|
||||
highWatermark: 0,
|
||||
activeRunId: null,
|
||||
capabilitySnapshotId: null,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function assertExecutable(filePath: string): Promise<void> {
|
||||
await fs.access(filePath, fsConstants.X_OK);
|
||||
}
|
||||
|
||||
function withBunOnPath(pathValue: string): string {
|
||||
const bunDir = '/Users/belief/.bun/bin';
|
||||
return pathValue.split(path.delimiter).includes(bunDir)
|
||||
? pathValue
|
||||
: `${bunDir}${path.delimiter}${pathValue}`;
|
||||
}
|
||||
|
||||
function createStableBridgeEnv(): NodeJS.ProcessEnv {
|
||||
const realHome = os.userInfo().homedir;
|
||||
const env = applyOpenCodeAutoUpdatePolicy({ ...process.env });
|
||||
return {
|
||||
...env,
|
||||
HOME: realHome,
|
||||
USERPROFILE: realHome,
|
||||
};
|
||||
}
|
||||
|
|
|
|||
373
test/main/services/team/OpenCodeSemanticModelMatrix.live.test.ts
Normal file
373
test/main/services/team/OpenCodeSemanticModelMatrix.live.test.ts
Normal file
|
|
@ -0,0 +1,373 @@
|
|||
import { promises as fs } from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import { setClaudeBasePathOverride } from '../../../../src/main/utils/pathDecoder';
|
||||
import {
|
||||
buildOpenCodeScenarioTeamRequest,
|
||||
loadOpenCodeSemanticScenario,
|
||||
materializeOpenCodeScenarioProject,
|
||||
materializeOpenCodeScenarioTasks,
|
||||
parseOpenCodeE2EModelList,
|
||||
taskRefForScenario,
|
||||
type OpenCodeSemanticScenario,
|
||||
} from './openCodeSemanticScenarioHarness';
|
||||
import {
|
||||
createOpenCodeLiveHarness,
|
||||
getRuntimeTranscript,
|
||||
type InboxMessage,
|
||||
waitForMemberInboxMessage,
|
||||
waitForOpenCodeLanesStopped,
|
||||
waitForOpenCodePeerRelay,
|
||||
waitForUserInboxReply,
|
||||
} from './openCodeLiveTestHarness';
|
||||
|
||||
import type { TaskRef, TeamProvisioningProgress } from '../../../../src/shared/types';
|
||||
|
||||
const liveDescribe =
|
||||
process.env.OPENCODE_E2E === '1' && process.env.OPENCODE_E2E_SEMANTIC_MODEL_MATRIX === '1'
|
||||
? describe
|
||||
: describe.skip;
|
||||
|
||||
interface ModelMatrixReport {
|
||||
generatedAt: string;
|
||||
models: ModelResult[];
|
||||
}
|
||||
|
||||
interface ModelResult {
|
||||
model: string;
|
||||
passed: boolean;
|
||||
score: number;
|
||||
durationMs: number;
|
||||
stages: {
|
||||
launchBootstrap: boolean;
|
||||
directReply: boolean;
|
||||
peerRelay: boolean;
|
||||
taskRefs: boolean;
|
||||
longPrompt: boolean;
|
||||
latencyStable: boolean;
|
||||
};
|
||||
diagnostics: string[];
|
||||
}
|
||||
|
||||
liveDescribe('OpenCode semantic model matrix live e2e', () => {
|
||||
it(
|
||||
'launches realistic OpenCode teams and scores model behavior sequentially',
|
||||
async () => {
|
||||
const scenario = await loadOpenCodeSemanticScenario();
|
||||
const models = parseOpenCodeE2EModelList();
|
||||
const results: ModelResult[] = [];
|
||||
|
||||
for (const model of models) {
|
||||
results.push(await runModelScenario({ scenario, model }));
|
||||
}
|
||||
|
||||
await writeModelMatrixReport({
|
||||
generatedAt: new Date().toISOString(),
|
||||
models: results,
|
||||
});
|
||||
|
||||
const failures = results.filter((result) => !result.passed);
|
||||
expect(failures, JSON.stringify(results, null, 2)).toEqual([]);
|
||||
},
|
||||
Math.max(420_000, parseOpenCodeE2EModelList().length * 420_000)
|
||||
);
|
||||
});
|
||||
|
||||
async function runModelScenario(input: {
|
||||
scenario: OpenCodeSemanticScenario;
|
||||
model: string;
|
||||
}): Promise<ModelResult> {
|
||||
const startedAt = Date.now();
|
||||
const stages: ModelResult['stages'] = {
|
||||
launchBootstrap: false,
|
||||
directReply: false,
|
||||
peerRelay: false,
|
||||
taskRefs: false,
|
||||
longPrompt: false,
|
||||
latencyStable: false,
|
||||
};
|
||||
const diagnostics: string[] = [];
|
||||
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'opencode-semantic-model-matrix-'));
|
||||
const tempClaudeRoot = path.join(tempDir, '.claude');
|
||||
const projectPath = path.join(tempDir, 'project');
|
||||
const teamName = `${input.scenario.teamNamePrefix}-${sanitizeModelForTeamName(input.model)}-${Date.now()}`;
|
||||
let harness: Awaited<ReturnType<typeof createOpenCodeLiveHarness>> | null = null;
|
||||
|
||||
try {
|
||||
await fs.mkdir(tempClaudeRoot, { recursive: true });
|
||||
await fs.mkdir(projectPath, { recursive: true });
|
||||
setClaudeBasePathOverride(tempClaudeRoot);
|
||||
await materializeOpenCodeScenarioProject(input.scenario, projectPath);
|
||||
harness = await createOpenCodeLiveHarness({
|
||||
tempDir,
|
||||
selectedModel: input.model,
|
||||
});
|
||||
|
||||
const progressEvents: TeamProvisioningProgress[] = [];
|
||||
const createStartedAt = Date.now();
|
||||
const { runId } = await harness.svc.createTeam(
|
||||
buildOpenCodeScenarioTeamRequest({
|
||||
scenario: input.scenario,
|
||||
teamName,
|
||||
projectPath,
|
||||
model: harness.selectedModel,
|
||||
}),
|
||||
(progress) => progressEvents.push(progress)
|
||||
);
|
||||
diagnostics.push(`runId=${runId}`);
|
||||
await materializeOpenCodeScenarioTasks({ scenario: input.scenario, teamName, projectPath });
|
||||
|
||||
const progressDump = formatProgressDump(progressEvents);
|
||||
if (!progressEvents.some((progress) => progress.message.includes('OpenCode team launch is ready'))) {
|
||||
throw new Error(`OpenCode launch did not reach ready state.\n${progressDump}`);
|
||||
}
|
||||
const runtimeSnapshot = await harness.svc.getTeamAgentRuntimeSnapshot(teamName);
|
||||
for (const member of input.scenario.members) {
|
||||
const snapshot = runtimeSnapshot.members[member.name];
|
||||
if (!snapshot?.alive) {
|
||||
throw new Error(
|
||||
`OpenCode member ${member.name} is not alive. Snapshot: ${JSON.stringify(
|
||||
runtimeSnapshot,
|
||||
null,
|
||||
2
|
||||
)}`
|
||||
);
|
||||
}
|
||||
if (snapshot.runtimeModel !== harness.selectedModel) {
|
||||
diagnostics.push(
|
||||
`${member.name} runtime model ${snapshot.runtimeModel ?? 'unknown'} differs from ${harness.selectedModel}`
|
||||
);
|
||||
}
|
||||
}
|
||||
stages.launchBootstrap = true;
|
||||
stages.longPrompt = input.scenario.teamPromptLines.join('\n').length > 1_500;
|
||||
stages.latencyStable = Date.now() - createStartedAt < 240_000;
|
||||
|
||||
const directTaskRef = taskRefForScenario(
|
||||
input.scenario,
|
||||
input.scenario.directDelivery.taskIndex,
|
||||
teamName
|
||||
);
|
||||
const directDelivery = await harness.svc.deliverOpenCodeMemberMessage(teamName, {
|
||||
memberName: input.scenario.directDelivery.memberName,
|
||||
messageId: `ui-direct-${Date.now()}`,
|
||||
replyRecipient: input.scenario.directDelivery.replyRecipient,
|
||||
actionMode: input.scenario.directDelivery.actionMode,
|
||||
taskRefs: [directTaskRef],
|
||||
source: 'manual',
|
||||
text: input.scenario.directDelivery.textLines.join('\n'),
|
||||
});
|
||||
if (!directDelivery.delivered) {
|
||||
throw new Error(`Direct OpenCode delivery failed: ${JSON.stringify(directDelivery, null, 2)}`);
|
||||
}
|
||||
const directReply = await waitForReplyWithTranscript({
|
||||
bridgeClient: harness.bridgeClient,
|
||||
teamName,
|
||||
memberName: input.scenario.directDelivery.memberName,
|
||||
projectPath,
|
||||
expectedToken: input.scenario.directDelivery.expectedReplyToken,
|
||||
timeoutMs: 180_000,
|
||||
});
|
||||
assertVisibleReplyContract(directReply, {
|
||||
expectedFrom: input.scenario.directDelivery.memberName,
|
||||
expectedTo: 'user',
|
||||
expectedTaskRef: directTaskRef,
|
||||
});
|
||||
stages.directReply = true;
|
||||
stages.taskRefs = hasTaskRef(directReply, directTaskRef);
|
||||
|
||||
const peerTaskRef = taskRefForScenario(
|
||||
input.scenario,
|
||||
input.scenario.peerDelivery.taskIndex,
|
||||
teamName
|
||||
);
|
||||
const peerDelivery = await harness.svc.deliverOpenCodeMemberMessage(teamName, {
|
||||
memberName: input.scenario.peerDelivery.senderName,
|
||||
messageId: `ui-peer-${Date.now()}`,
|
||||
replyRecipient: input.scenario.peerDelivery.replyRecipient,
|
||||
actionMode: input.scenario.peerDelivery.actionMode,
|
||||
taskRefs: [peerTaskRef],
|
||||
source: 'manual',
|
||||
text: input.scenario.peerDelivery.textLines.join('\n'),
|
||||
});
|
||||
if (!peerDelivery.delivered) {
|
||||
throw new Error(`Peer OpenCode delivery failed: ${JSON.stringify(peerDelivery, null, 2)}`);
|
||||
}
|
||||
|
||||
const peerMessage = await waitForMemberInboxMessage(
|
||||
teamName,
|
||||
input.scenario.peerDelivery.recipientName,
|
||||
input.scenario.peerDelivery.senderName,
|
||||
input.scenario.peerDelivery.peerToken,
|
||||
180_000
|
||||
);
|
||||
assertVisibleReplyContract(peerMessage, {
|
||||
expectedFrom: input.scenario.peerDelivery.senderName,
|
||||
expectedTo: input.scenario.peerDelivery.recipientName,
|
||||
expectedTaskRef: peerTaskRef,
|
||||
});
|
||||
await waitForOpenCodePeerRelay(
|
||||
harness.svc,
|
||||
teamName,
|
||||
input.scenario.peerDelivery.recipientName,
|
||||
peerMessage.messageId,
|
||||
180_000
|
||||
);
|
||||
const peerReply = await waitForReplyWithTranscript({
|
||||
bridgeClient: harness.bridgeClient,
|
||||
teamName,
|
||||
memberName: input.scenario.peerDelivery.recipientName,
|
||||
projectPath,
|
||||
expectedToken: input.scenario.peerDelivery.expectedReplyToken,
|
||||
timeoutMs: 180_000,
|
||||
});
|
||||
assertVisibleReplyContract(peerReply, {
|
||||
expectedFrom: input.scenario.peerDelivery.recipientName,
|
||||
expectedTo: 'user',
|
||||
});
|
||||
stages.peerRelay = true;
|
||||
|
||||
const score = scoreModel(stages);
|
||||
return {
|
||||
model: input.model,
|
||||
passed: score === 100,
|
||||
score,
|
||||
durationMs: Date.now() - startedAt,
|
||||
stages,
|
||||
diagnostics,
|
||||
};
|
||||
} catch (error) {
|
||||
diagnostics.push(error instanceof Error ? error.message : String(error));
|
||||
return {
|
||||
model: input.model,
|
||||
passed: false,
|
||||
score: scoreModel(stages),
|
||||
durationMs: Date.now() - startedAt,
|
||||
stages,
|
||||
diagnostics,
|
||||
};
|
||||
} finally {
|
||||
if (harness) {
|
||||
await harness.svc.stopTeam(teamName).catch(() => undefined);
|
||||
await harness.dispose().catch(() => undefined);
|
||||
await waitForOpenCodeLanesStopped(teamName).catch(() => undefined);
|
||||
}
|
||||
setClaudeBasePathOverride(null);
|
||||
await fs.rm(tempDir, { recursive: true, force: true }).catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
async function waitForReplyWithTranscript(input: {
|
||||
bridgeClient: Parameters<typeof getRuntimeTranscript>[0]['bridgeClient'];
|
||||
teamName: string;
|
||||
memberName: string;
|
||||
projectPath: string;
|
||||
expectedToken: string;
|
||||
timeoutMs: number;
|
||||
}): Promise<InboxMessage> {
|
||||
try {
|
||||
return await waitForUserInboxReply(
|
||||
input.teamName,
|
||||
input.memberName,
|
||||
input.expectedToken,
|
||||
input.timeoutMs
|
||||
);
|
||||
} catch (error) {
|
||||
const transcript = await getRuntimeTranscript({
|
||||
bridgeClient: input.bridgeClient,
|
||||
teamName: input.teamName,
|
||||
memberName: input.memberName,
|
||||
projectPath: input.projectPath,
|
||||
});
|
||||
throw new Error(
|
||||
`${error instanceof Error ? error.message : String(error)}\nTranscript: ${JSON.stringify(
|
||||
transcript,
|
||||
null,
|
||||
2
|
||||
)}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function assertVisibleReplyContract(
|
||||
message: InboxMessage,
|
||||
input: {
|
||||
expectedFrom: string;
|
||||
expectedTo: string;
|
||||
expectedTaskRef?: TaskRef;
|
||||
}
|
||||
): void {
|
||||
expect(message).toMatchObject({
|
||||
from: input.expectedFrom,
|
||||
to: input.expectedTo,
|
||||
});
|
||||
const text = message.text ?? '';
|
||||
expect(text).not.toContain('SendMessage');
|
||||
expect(text).not.toContain('runtime_deliver_message');
|
||||
expect(text).not.toContain('#00000000');
|
||||
expect(text.trim()).not.toBe('\u041f\u043e\u043d\u044f\u043b');
|
||||
if (input.expectedTaskRef) {
|
||||
expect(hasTaskRef(message, input.expectedTaskRef)).toBe(true);
|
||||
}
|
||||
}
|
||||
|
||||
function hasTaskRef(message: InboxMessage, expected: TaskRef): boolean {
|
||||
return Boolean(
|
||||
message.taskRefs?.some(
|
||||
(taskRef) =>
|
||||
taskRef.teamName === expected.teamName &&
|
||||
taskRef.taskId === expected.taskId &&
|
||||
taskRef.displayId === expected.displayId
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
function scoreModel(stages: ModelResult['stages']): number {
|
||||
return (
|
||||
(stages.launchBootstrap ? 25 : 0) +
|
||||
(stages.directReply ? 25 : 0) +
|
||||
(stages.peerRelay ? 20 : 0) +
|
||||
(stages.taskRefs ? 15 : 0) +
|
||||
(stages.longPrompt ? 10 : 0) +
|
||||
(stages.latencyStable ? 5 : 0)
|
||||
);
|
||||
}
|
||||
|
||||
async function writeModelMatrixReport(report: ModelMatrixReport): Promise<void> {
|
||||
const outputDir = process.env.OPENCODE_E2E_REPORT_DIR?.trim()
|
||||
? path.resolve(process.env.OPENCODE_E2E_REPORT_DIR.trim())
|
||||
: path.join(process.cwd(), 'test-results', 'opencode-semantic-model-matrix');
|
||||
await fs.mkdir(outputDir, { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(outputDir, `report-${Date.now()}.json`),
|
||||
`${JSON.stringify(report, null, 2)}\n`,
|
||||
'utf8'
|
||||
);
|
||||
}
|
||||
|
||||
function formatProgressDump(progressEvents: TeamProvisioningProgress[]): string {
|
||||
return progressEvents
|
||||
.map((progress) =>
|
||||
[
|
||||
progress.state,
|
||||
progress.message,
|
||||
progress.messageSeverity,
|
||||
progress.error,
|
||||
progress.cliLogsTail,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(' | ')
|
||||
)
|
||||
.join('\n');
|
||||
}
|
||||
|
||||
function sanitizeModelForTeamName(model: string): string {
|
||||
return model
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, '-')
|
||||
.replace(/^-+|-+$/g, '')
|
||||
.slice(0, 48);
|
||||
}
|
||||
350
test/main/services/team/openCodeLiveTestHarness.ts
Normal file
350
test/main/services/team/openCodeLiveTestHarness.ts
Normal file
|
|
@ -0,0 +1,350 @@
|
|||
import { constants as fsConstants, promises as fs } from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
|
||||
import Fastify from 'fastify';
|
||||
|
||||
import { registerTeamRoutes } from '../../../../src/main/http/teams';
|
||||
import { applyOpenCodeAutoUpdatePolicy } from '../../../../src/main/services/runtime/openCodeAutoUpdatePolicy';
|
||||
import { OpenCodeBridgeCommandClient } from '../../../../src/main/services/team/opencode/bridge/OpenCodeBridgeCommandClient';
|
||||
import {
|
||||
createOpenCodeBridgeCommandLeaseStore,
|
||||
createOpenCodeBridgeCommandLedgerStore,
|
||||
} from '../../../../src/main/services/team/opencode/bridge/OpenCodeBridgeCommandLedgerStore';
|
||||
import {
|
||||
createOpenCodeBridgeClientIdentity,
|
||||
OpenCodeBridgeCommandHandshakePort,
|
||||
} from '../../../../src/main/services/team/opencode/bridge/OpenCodeBridgeHandshakeClient';
|
||||
import type { RuntimeStoreManifestEvidence } from '../../../../src/main/services/team/opencode/bridge/OpenCodeBridgeCommandContract';
|
||||
import { OpenCodeReadinessBridge } from '../../../../src/main/services/team/opencode/bridge/OpenCodeReadinessBridge';
|
||||
import {
|
||||
OpenCodeStateChangingBridgeCommandService,
|
||||
type OpenCodeBridgeCommandExecutor,
|
||||
type RuntimeStoreManifestReader,
|
||||
} from '../../../../src/main/services/team/opencode/bridge/OpenCodeStateChangingBridgeCommandService';
|
||||
import { readOpenCodeRuntimeLaneIndex } from '../../../../src/main/services/team/opencode/store/OpenCodeRuntimeManifestEvidenceReader';
|
||||
import { OpenCodeTeamRuntimeAdapter } from '../../../../src/main/services/team/runtime/OpenCodeTeamRuntimeAdapter';
|
||||
import { TeamRuntimeAdapterRegistry } from '../../../../src/main/services/team/runtime/TeamRuntimeAdapter';
|
||||
import { resolveAgentTeamsMcpLaunchSpec } from '../../../../src/main/services/team/TeamMcpConfigBuilder';
|
||||
import { TeamProvisioningService } from '../../../../src/main/services/team/TeamProvisioningService';
|
||||
import { getClaudeBasePath, getTeamsBasePath } from '../../../../src/main/utils/pathDecoder';
|
||||
|
||||
import type { HttpServices } from '../../../../src/main/http';
|
||||
import type { TaskRef } from '../../../../src/shared/types';
|
||||
|
||||
const DEFAULT_ORCHESTRATOR_CLI = '/Users/belief/dev/projects/claude/agent_teams_orchestrator/cli';
|
||||
|
||||
export interface InboxMessage {
|
||||
from?: string;
|
||||
to?: string;
|
||||
text?: string;
|
||||
messageId?: string;
|
||||
read?: boolean;
|
||||
taskRefs?: TaskRef[];
|
||||
source?: string;
|
||||
}
|
||||
|
||||
export interface OpenCodeLiveHarness {
|
||||
bridgeClient: OpenCodeBridgeCommandClient;
|
||||
selectedModel: string;
|
||||
svc: TeamProvisioningService;
|
||||
dispose: () => Promise<void>;
|
||||
}
|
||||
|
||||
export async function createOpenCodeLiveHarness(input: {
|
||||
tempDir: string;
|
||||
selectedModel: string;
|
||||
}): Promise<OpenCodeLiveHarness> {
|
||||
const orchestratorCli =
|
||||
process.env.CLAUDE_AGENT_TEAMS_ORCHESTRATOR_CLI_PATH?.trim() || DEFAULT_ORCHESTRATOR_CLI;
|
||||
await assertExecutable(orchestratorCli);
|
||||
|
||||
const svc = new TeamProvisioningService();
|
||||
const controlApi = await startLiveTeamControlApi(svc);
|
||||
svc.setControlApiBaseUrlResolver(async () => controlApi.baseUrl);
|
||||
|
||||
const mcpLaunchSpec = await resolveAgentTeamsMcpLaunchSpec();
|
||||
const bridgeEnv = {
|
||||
...createStableBridgeEnv(),
|
||||
PATH: withBunOnPath(process.env.PATH ?? ''),
|
||||
XDG_DATA_HOME: path.join(input.tempDir, 'xdg-data'),
|
||||
AGENT_TEAMS_MCP_CLAUDE_DIR: getClaudeBasePath(),
|
||||
CLAUDE_TEAM_CONTROL_URL: controlApi.baseUrl,
|
||||
CLAUDE_MULTIMODEL_AGENT_TEAMS_MCP_COMMAND: mcpLaunchSpec.command,
|
||||
CLAUDE_MULTIMODEL_AGENT_TEAMS_MCP_ENTRY: mcpLaunchSpec.args[0] ?? '',
|
||||
};
|
||||
const bridgeClient = new OpenCodeBridgeCommandClient({
|
||||
binaryPath: orchestratorCli,
|
||||
tempDirectory: path.join(input.tempDir, 'bridge-input'),
|
||||
env: bridgeEnv,
|
||||
});
|
||||
const stateChangingCommands = createStateChangingCommands({
|
||||
bridge: bridgeClient,
|
||||
controlDir: path.join(input.tempDir, 'control'),
|
||||
});
|
||||
const readinessBridge = new OpenCodeReadinessBridge(bridgeClient, {
|
||||
stateChangingCommands,
|
||||
timeoutMs: 180_000,
|
||||
launchTimeoutMs: 180_000,
|
||||
reconcileTimeoutMs: 90_000,
|
||||
stopTimeoutMs: 90_000,
|
||||
});
|
||||
const adapter = new OpenCodeTeamRuntimeAdapter(readinessBridge);
|
||||
svc.setRuntimeAdapterRegistry(new TeamRuntimeAdapterRegistry([adapter]));
|
||||
return {
|
||||
bridgeClient,
|
||||
selectedModel: input.selectedModel,
|
||||
svc,
|
||||
dispose: async () => {
|
||||
svc.setControlApiBaseUrlResolver(null);
|
||||
await controlApi.close();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function waitForUserInboxReply(
|
||||
teamName: string,
|
||||
from: string,
|
||||
expectedText: string,
|
||||
timeoutMs: number
|
||||
): Promise<InboxMessage> {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
const inboxPath = path.join(getTeamsBasePath(), teamName, 'inboxes', 'user.json');
|
||||
let lastMessages: InboxMessage[] = [];
|
||||
|
||||
while (Date.now() < deadline) {
|
||||
lastMessages = await readInboxMessages(inboxPath);
|
||||
const match = lastMessages.find(
|
||||
(message) =>
|
||||
message.from === from &&
|
||||
message.to === 'user' &&
|
||||
typeof message.text === 'string' &&
|
||||
message.text.includes(expectedText)
|
||||
);
|
||||
if (match) {
|
||||
return match;
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 1_500));
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
`Timed out waiting for OpenCode reply in ${inboxPath}. Last messages: ${JSON.stringify(
|
||||
lastMessages,
|
||||
null,
|
||||
2
|
||||
)}`
|
||||
);
|
||||
}
|
||||
|
||||
export async function waitForMemberInboxMessage(
|
||||
teamName: string,
|
||||
memberName: string,
|
||||
from: string,
|
||||
expectedText: string | string[],
|
||||
timeoutMs: number
|
||||
): Promise<InboxMessage & { messageId: string }> {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
const inboxPath = path.join(getTeamsBasePath(), teamName, 'inboxes', `${memberName}.json`);
|
||||
let lastMessages: InboxMessage[] = [];
|
||||
const expectedTexts = Array.isArray(expectedText) ? expectedText : [expectedText];
|
||||
|
||||
while (Date.now() < deadline) {
|
||||
lastMessages = await readInboxMessages(inboxPath);
|
||||
const match = lastMessages.find(
|
||||
(message): message is InboxMessage & { messageId: string; text: string } => {
|
||||
if (message.from !== from || message.to !== memberName) return false;
|
||||
if (typeof message.messageId !== 'string' || !message.messageId.trim()) return false;
|
||||
const text = message.text;
|
||||
if (typeof text !== 'string') return false;
|
||||
return expectedTexts.every((expected) => text.includes(expected));
|
||||
}
|
||||
);
|
||||
if (match) {
|
||||
return match;
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 1_500));
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
`Timed out waiting for OpenCode member message in ${inboxPath}. Last messages: ${JSON.stringify(
|
||||
lastMessages,
|
||||
null,
|
||||
2
|
||||
)}`
|
||||
);
|
||||
}
|
||||
|
||||
export async function waitForOpenCodePeerRelay(
|
||||
svc: TeamProvisioningService,
|
||||
teamName: string,
|
||||
memberName: string,
|
||||
messageId: string,
|
||||
timeoutMs: number
|
||||
): Promise<void> {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
let lastRelay: Awaited<ReturnType<TeamProvisioningService['relayOpenCodeMemberInboxMessages']>> | null =
|
||||
null;
|
||||
|
||||
while (Date.now() < deadline) {
|
||||
lastRelay = await svc.relayOpenCodeMemberInboxMessages(teamName, memberName, {
|
||||
onlyMessageId: messageId,
|
||||
source: 'manual',
|
||||
deliveryMetadata: {
|
||||
replyRecipient: 'user',
|
||||
},
|
||||
});
|
||||
if (lastRelay.delivered >= 1) {
|
||||
return;
|
||||
}
|
||||
if (lastRelay.failed > 0 && lastRelay.lastDelivery?.responsePending !== true) {
|
||||
break;
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 3_000));
|
||||
}
|
||||
|
||||
throw new Error(`OpenCode peer relay failed: ${JSON.stringify(lastRelay, null, 2)}`);
|
||||
}
|
||||
|
||||
export async function readInboxMessages(inboxPath: string): Promise<InboxMessage[]> {
|
||||
try {
|
||||
const parsed = JSON.parse(await fs.readFile(inboxPath, 'utf8'));
|
||||
return Array.isArray(parsed) ? (parsed as InboxMessage[]) : [];
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
|
||||
return [];
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export async function waitUntil(
|
||||
predicate: () => Promise<boolean>,
|
||||
timeoutMs: number,
|
||||
pollMs = 500
|
||||
): Promise<void> {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() < deadline) {
|
||||
if (await predicate()) {
|
||||
return;
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, pollMs));
|
||||
}
|
||||
throw new Error(`Timed out after ${timeoutMs}ms waiting for condition`);
|
||||
}
|
||||
|
||||
export async function waitForOpenCodeLanesStopped(
|
||||
teamName: string,
|
||||
timeoutMs = 90_000
|
||||
): Promise<void> {
|
||||
await waitUntil(async () => {
|
||||
const laneIndex = await readOpenCodeRuntimeLaneIndex(getTeamsBasePath(), teamName);
|
||||
return Object.keys(laneIndex.lanes).length === 0;
|
||||
}, timeoutMs).catch(() => undefined);
|
||||
}
|
||||
|
||||
export async function getRuntimeTranscript(input: {
|
||||
bridgeClient: OpenCodeBridgeCommandClient;
|
||||
teamName: string;
|
||||
memberName: string;
|
||||
projectPath: string;
|
||||
}): Promise<unknown> {
|
||||
return input.bridgeClient
|
||||
.execute<
|
||||
{ teamId: string; teamName: string; laneId: string; memberName: string },
|
||||
{ logProjection?: { messages?: unknown[] }; messages?: unknown[] }
|
||||
>(
|
||||
'opencode.getRuntimeTranscript',
|
||||
{
|
||||
teamId: input.teamName,
|
||||
teamName: input.teamName,
|
||||
laneId: 'primary',
|
||||
memberName: input.memberName,
|
||||
},
|
||||
{ cwd: input.projectPath, timeoutMs: 60_000 }
|
||||
)
|
||||
.catch((transcriptError) => ({
|
||||
ok: false as const,
|
||||
error: String(transcriptError),
|
||||
}));
|
||||
}
|
||||
|
||||
async function startLiveTeamControlApi(svc: TeamProvisioningService): Promise<{
|
||||
baseUrl: string;
|
||||
close: () => Promise<void>;
|
||||
}> {
|
||||
const app = Fastify({ logger: false });
|
||||
registerTeamRoutes(app, {
|
||||
teamProvisioningService: svc,
|
||||
} as HttpServices);
|
||||
await app.listen({ host: '127.0.0.1', port: 0 });
|
||||
const address = app.server.address();
|
||||
if (!address || typeof address === 'string') {
|
||||
await app.close();
|
||||
throw new Error('Failed to start live team control API');
|
||||
}
|
||||
|
||||
return {
|
||||
baseUrl: `http://127.0.0.1:${address.port}`,
|
||||
close: async () => {
|
||||
await app.close();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function createStateChangingCommands(input: {
|
||||
bridge: OpenCodeBridgeCommandExecutor;
|
||||
controlDir: string;
|
||||
}): OpenCodeStateChangingBridgeCommandService {
|
||||
const clientIdentity = createOpenCodeBridgeClientIdentity({
|
||||
appVersion: '1.3.0-e2e',
|
||||
gitSha: null,
|
||||
buildId: 'opencode-semantic-model-matrix-e2e',
|
||||
});
|
||||
|
||||
return new OpenCodeStateChangingBridgeCommandService({
|
||||
expectedClientIdentity: clientIdentity,
|
||||
handshakePort: new OpenCodeBridgeCommandHandshakePort({
|
||||
bridge: input.bridge,
|
||||
clientIdentity,
|
||||
}),
|
||||
leaseStore: createOpenCodeBridgeCommandLeaseStore({
|
||||
filePath: path.join(input.controlDir, 'leases.json'),
|
||||
}),
|
||||
ledger: createOpenCodeBridgeCommandLedgerStore({
|
||||
filePath: path.join(input.controlDir, 'ledger.json'),
|
||||
}),
|
||||
bridge: input.bridge,
|
||||
manifestReader: new StaticManifestReader(),
|
||||
});
|
||||
}
|
||||
|
||||
class StaticManifestReader implements RuntimeStoreManifestReader {
|
||||
async read(): Promise<RuntimeStoreManifestEvidence> {
|
||||
return {
|
||||
highWatermark: 0,
|
||||
activeRunId: null,
|
||||
capabilitySnapshotId: null,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function assertExecutable(filePath: string): Promise<void> {
|
||||
await fs.access(filePath, fsConstants.X_OK);
|
||||
}
|
||||
|
||||
function withBunOnPath(pathValue: string): string {
|
||||
const bunDir = '/Users/belief/.bun/bin';
|
||||
return pathValue.split(path.delimiter).includes(bunDir)
|
||||
? pathValue
|
||||
: `${bunDir}${path.delimiter}${pathValue}`;
|
||||
}
|
||||
|
||||
function createStableBridgeEnv(): NodeJS.ProcessEnv {
|
||||
const realHome = os.userInfo().homedir;
|
||||
const env = applyOpenCodeAutoUpdatePolicy({ ...process.env });
|
||||
return {
|
||||
...env,
|
||||
HOME: realHome,
|
||||
USERPROFILE: realHome,
|
||||
};
|
||||
}
|
||||
468
test/main/services/team/openCodeSemanticScenarioHarness.ts
Normal file
468
test/main/services/team/openCodeSemanticScenarioHarness.ts
Normal file
|
|
@ -0,0 +1,468 @@
|
|||
import { promises as fs } from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
import { OpenCodeTeamRuntimeAdapter } from '../../../../src/main/services/team/runtime/OpenCodeTeamRuntimeAdapter';
|
||||
import { TeamTaskWriter } from '../../../../src/main/services/team/TeamTaskWriter';
|
||||
import type {
|
||||
OpenCodeLaunchTeamCommandBody,
|
||||
OpenCodeLaunchTeamCommandData,
|
||||
OpenCodeSendMessageCommandBody,
|
||||
OpenCodeSendMessageCommandData,
|
||||
} from '../../../../src/main/services/team/opencode/bridge/OpenCodeBridgeCommandContract';
|
||||
import type { OpenCodeTeamLaunchReadiness } from '../../../../src/main/services/team/opencode/readiness/OpenCodeTeamLaunchReadiness';
|
||||
import { REQUIRED_AGENT_TEAMS_APP_TOOL_IDS } from '../../../../src/main/services/team/opencode/mcp/OpenCodeMcpToolAvailability';
|
||||
import type {
|
||||
OpenCodeTeamRuntimeBridgePort,
|
||||
OpenCodeTeamRuntimeMessageInput,
|
||||
} from '../../../../src/main/services/team/runtime';
|
||||
import type {
|
||||
TeamLaunchRuntimeAdapter,
|
||||
TeamRuntimeLaunchInput,
|
||||
TeamRuntimeLaunchResult,
|
||||
TeamRuntimeMemberLaunchEvidence,
|
||||
TeamRuntimePrepareResult,
|
||||
TeamRuntimeReconcileInput,
|
||||
TeamRuntimeReconcileResult,
|
||||
TeamRuntimeStopInput,
|
||||
TeamRuntimeStopResult,
|
||||
} from '../../../../src/main/services/team/runtime/TeamRuntimeAdapter';
|
||||
import type { AgentActionMode, TaskRef, TeamCreateRequest, TeamTask } from '../../../../src/shared/types';
|
||||
|
||||
const FIXTURE_PATH = path.join(
|
||||
process.cwd(),
|
||||
'test',
|
||||
'fixtures',
|
||||
'team',
|
||||
'opencode',
|
||||
'semantic-realistic-scenario.json'
|
||||
);
|
||||
|
||||
export interface OpenCodeSemanticScenario {
|
||||
teamNamePrefix: string;
|
||||
displayName: string;
|
||||
description: string;
|
||||
teamPromptLines: string[];
|
||||
members: Array<{
|
||||
name: string;
|
||||
role: string;
|
||||
workflowLines: string[];
|
||||
}>;
|
||||
projectFiles: Array<{
|
||||
path: string;
|
||||
contentLines: string[];
|
||||
}>;
|
||||
tasks: Array<{
|
||||
taskId: string;
|
||||
displayId: string;
|
||||
subject: string;
|
||||
owner: string;
|
||||
comment: string;
|
||||
}>;
|
||||
directDelivery: {
|
||||
memberName: string;
|
||||
replyRecipient: string;
|
||||
actionMode: AgentActionMode;
|
||||
taskIndex: number;
|
||||
expectedReplyToken: string;
|
||||
textLines: string[];
|
||||
};
|
||||
peerDelivery: {
|
||||
senderName: string;
|
||||
recipientName: string;
|
||||
replyRecipient: string;
|
||||
actionMode: AgentActionMode;
|
||||
taskIndex: number;
|
||||
peerToken: string;
|
||||
expectedReplyToken: string;
|
||||
textLines: string[];
|
||||
};
|
||||
}
|
||||
|
||||
export interface CapturedOpenCodeBridge {
|
||||
readonly launchCommands: OpenCodeLaunchTeamCommandBody[];
|
||||
readonly messageCommands: OpenCodeSendMessageCommandBody[];
|
||||
readonly bridge: OpenCodeTeamRuntimeBridgePort;
|
||||
}
|
||||
|
||||
export class CapturingOpenCodeRuntimeAdapter implements TeamLaunchRuntimeAdapter {
|
||||
readonly providerId = 'opencode' as const;
|
||||
readonly launchInputs: TeamRuntimeLaunchInput[] = [];
|
||||
|
||||
async prepare(input: TeamRuntimeLaunchInput): Promise<TeamRuntimePrepareResult> {
|
||||
return {
|
||||
ok: true,
|
||||
providerId: 'opencode',
|
||||
modelId: input.model ?? null,
|
||||
diagnostics: [],
|
||||
warnings: [],
|
||||
};
|
||||
}
|
||||
|
||||
async launch(input: TeamRuntimeLaunchInput): Promise<TeamRuntimeLaunchResult> {
|
||||
this.launchInputs.push(input);
|
||||
return {
|
||||
runId: input.runId,
|
||||
teamName: input.teamName,
|
||||
launchPhase: 'finished',
|
||||
teamLaunchState: 'clean_success',
|
||||
members: Object.fromEntries(
|
||||
input.expectedMembers.map((member, index) => [
|
||||
member.name,
|
||||
buildConfirmedMemberEvidence(member.name, member.model ?? input.model ?? null, index),
|
||||
])
|
||||
),
|
||||
warnings: [],
|
||||
diagnostics: ['captured OpenCode launch input'],
|
||||
};
|
||||
}
|
||||
|
||||
async reconcile(input: TeamRuntimeReconcileInput): Promise<TeamRuntimeReconcileResult> {
|
||||
return {
|
||||
runId: input.runId,
|
||||
teamName: input.teamName,
|
||||
launchPhase: 'reconciled',
|
||||
teamLaunchState: 'clean_success',
|
||||
members: Object.fromEntries(
|
||||
input.expectedMembers.map((member, index) => [
|
||||
member.name,
|
||||
buildConfirmedMemberEvidence(member.name, member.model ?? null, index),
|
||||
])
|
||||
),
|
||||
snapshot: input.previousLaunchState,
|
||||
warnings: [],
|
||||
diagnostics: ['captured OpenCode reconcile input'],
|
||||
};
|
||||
}
|
||||
|
||||
async stop(input: TeamRuntimeStopInput): Promise<TeamRuntimeStopResult> {
|
||||
return {
|
||||
runId: input.runId,
|
||||
teamName: input.teamName,
|
||||
stopped: true,
|
||||
members: {},
|
||||
warnings: [],
|
||||
diagnostics: ['captured OpenCode stop input'],
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export async function loadOpenCodeSemanticScenario(): Promise<OpenCodeSemanticScenario> {
|
||||
const parsed = JSON.parse(await fs.readFile(FIXTURE_PATH, 'utf8')) as OpenCodeSemanticScenario;
|
||||
if (!Array.isArray(parsed.members) || parsed.members.length < 2) {
|
||||
throw new Error('OpenCode semantic scenario requires at least two members.');
|
||||
}
|
||||
if (!Array.isArray(parsed.tasks) || parsed.tasks.length < 2) {
|
||||
throw new Error('OpenCode semantic scenario requires at least two tasks.');
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
export async function materializeOpenCodeScenarioProject(
|
||||
scenario: OpenCodeSemanticScenario,
|
||||
projectPath: string
|
||||
): Promise<void> {
|
||||
for (const file of scenario.projectFiles) {
|
||||
const targetPath = path.join(projectPath, file.path);
|
||||
await fs.mkdir(path.dirname(targetPath), { recursive: true });
|
||||
await fs.writeFile(targetPath, `${file.contentLines.join('\n')}\n`, 'utf8');
|
||||
}
|
||||
}
|
||||
|
||||
export async function materializeOpenCodeScenarioTasks(input: {
|
||||
scenario: OpenCodeSemanticScenario;
|
||||
teamName: string;
|
||||
projectPath: string;
|
||||
}): Promise<void> {
|
||||
const writer = new TeamTaskWriter();
|
||||
const createdAt = '2026-04-21T00:00:00.000Z';
|
||||
for (const task of input.scenario.tasks) {
|
||||
const record: TeamTask = {
|
||||
id: task.taskId,
|
||||
displayId: task.displayId,
|
||||
subject: task.subject,
|
||||
description: task.comment,
|
||||
owner: task.owner,
|
||||
createdBy: 'lead',
|
||||
status: 'in_progress',
|
||||
projectPath: input.projectPath,
|
||||
createdAt,
|
||||
updatedAt: createdAt,
|
||||
comments: [
|
||||
{
|
||||
id: `${task.taskId}-comment`,
|
||||
author: 'lead',
|
||||
text: task.comment,
|
||||
createdAt,
|
||||
type: 'regular',
|
||||
},
|
||||
],
|
||||
};
|
||||
await writer.createTask(input.teamName, record);
|
||||
}
|
||||
}
|
||||
|
||||
export function buildOpenCodeScenarioTeamRequest(input: {
|
||||
scenario: OpenCodeSemanticScenario;
|
||||
teamName: string;
|
||||
projectPath: string;
|
||||
model: string;
|
||||
memberNames?: string[];
|
||||
}): TeamCreateRequest {
|
||||
const memberNames = new Set(input.memberNames ?? input.scenario.members.map((member) => member.name));
|
||||
return {
|
||||
teamName: input.teamName,
|
||||
displayName: input.scenario.displayName,
|
||||
description: input.scenario.description,
|
||||
cwd: input.projectPath,
|
||||
providerId: 'opencode',
|
||||
model: input.model,
|
||||
skipPermissions: true,
|
||||
prompt: input.scenario.teamPromptLines.join('\n'),
|
||||
members: input.scenario.members
|
||||
.filter((member) => memberNames.has(member.name))
|
||||
.map((member) => ({
|
||||
name: member.name,
|
||||
role: member.role,
|
||||
workflow: member.workflowLines.join('\n'),
|
||||
providerId: 'opencode' as const,
|
||||
model: input.model,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
export function buildScenarioRuntimeMessageInput(input: {
|
||||
scenario: OpenCodeSemanticScenario;
|
||||
teamName: string;
|
||||
projectPath: string;
|
||||
runId?: string;
|
||||
laneId?: string;
|
||||
kind: 'direct' | 'peer';
|
||||
}): OpenCodeTeamRuntimeMessageInput {
|
||||
if (input.kind === 'direct') {
|
||||
const delivery = input.scenario.directDelivery;
|
||||
return {
|
||||
runId: input.runId,
|
||||
teamName: input.teamName,
|
||||
laneId: input.laneId ?? 'primary',
|
||||
memberName: delivery.memberName,
|
||||
cwd: input.projectPath,
|
||||
text: delivery.textLines.join('\n'),
|
||||
messageId: `semantic-direct-${delivery.expectedReplyToken}`,
|
||||
replyRecipient: delivery.replyRecipient,
|
||||
actionMode: delivery.actionMode,
|
||||
taskRefs: [taskRefForScenario(input.scenario, delivery.taskIndex, input.teamName)],
|
||||
};
|
||||
}
|
||||
|
||||
const delivery = input.scenario.peerDelivery;
|
||||
return {
|
||||
runId: input.runId,
|
||||
teamName: input.teamName,
|
||||
laneId: input.laneId ?? 'primary',
|
||||
memberName: delivery.senderName,
|
||||
cwd: input.projectPath,
|
||||
text: delivery.textLines.join('\n'),
|
||||
messageId: `semantic-peer-${delivery.peerToken}`,
|
||||
replyRecipient: delivery.replyRecipient,
|
||||
actionMode: delivery.actionMode,
|
||||
taskRefs: [taskRefForScenario(input.scenario, delivery.taskIndex, input.teamName)],
|
||||
};
|
||||
}
|
||||
|
||||
export function taskRefForScenario(
|
||||
scenario: OpenCodeSemanticScenario,
|
||||
taskIndex: number,
|
||||
teamName: string
|
||||
): TaskRef {
|
||||
const task = scenario.tasks[taskIndex];
|
||||
if (!task) {
|
||||
throw new Error(`OpenCode semantic scenario task index ${taskIndex} is missing.`);
|
||||
}
|
||||
return {
|
||||
taskId: task.taskId,
|
||||
displayId: task.displayId,
|
||||
teamName,
|
||||
};
|
||||
}
|
||||
|
||||
export function createCapturingOpenCodeBridge(modelId: string): CapturedOpenCodeBridge {
|
||||
const launchCommands: OpenCodeLaunchTeamCommandBody[] = [];
|
||||
const messageCommands: OpenCodeSendMessageCommandBody[] = [];
|
||||
return {
|
||||
launchCommands,
|
||||
messageCommands,
|
||||
bridge: {
|
||||
checkOpenCodeTeamLaunchReadiness: async () => readyOpenCodeReadiness(modelId),
|
||||
getLastOpenCodeRuntimeSnapshot: () => ({
|
||||
providerId: 'opencode',
|
||||
binaryPath: '/opt/homebrew/bin/opencode',
|
||||
binaryFingerprint: 'version:1.14.19',
|
||||
version: '1.14.19',
|
||||
capabilitySnapshotId: 'capability-semantic-contract',
|
||||
}),
|
||||
launchOpenCodeTeam: async (command) => {
|
||||
launchCommands.push(command);
|
||||
return buildReadyLaunchData(command, modelId);
|
||||
},
|
||||
sendOpenCodeTeamMessage: async (command) => {
|
||||
messageCommands.push(command);
|
||||
return buildAcceptedMessageData(command);
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function dumpOpenCodePromptArtifacts(input: {
|
||||
outputDir: string;
|
||||
launchInput: TeamRuntimeLaunchInput;
|
||||
launchCommand: OpenCodeLaunchTeamCommandBody;
|
||||
messageCommands: OpenCodeSendMessageCommandBody[];
|
||||
}): Promise<void> {
|
||||
await fs.mkdir(input.outputDir, { recursive: true });
|
||||
const summary = {
|
||||
teamName: input.launchInput.teamName,
|
||||
launchPromptChars: input.launchInput.prompt?.length ?? 0,
|
||||
leadPromptChars: input.launchCommand.leadPrompt?.length ?? 0,
|
||||
memberPromptChars: input.launchCommand.members.map((member) => ({
|
||||
name: member.name,
|
||||
chars: member.prompt?.length ?? 0,
|
||||
})),
|
||||
messagePromptChars: input.messageCommands.map((command) => ({
|
||||
memberName: command.memberName,
|
||||
chars: command.text.length,
|
||||
actionMode: command.actionMode ?? null,
|
||||
taskRefs: command.taskRefs ?? [],
|
||||
})),
|
||||
};
|
||||
await fs.writeFile(path.join(input.outputDir, 'summary.json'), `${JSON.stringify(summary, null, 2)}\n`);
|
||||
await fs.writeFile(
|
||||
path.join(input.outputDir, 'launch-command.json'),
|
||||
`${JSON.stringify(input.launchCommand, null, 2)}\n`
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(input.outputDir, 'message-commands.json'),
|
||||
`${JSON.stringify(input.messageCommands, null, 2)}\n`
|
||||
);
|
||||
}
|
||||
|
||||
export function createOpenCodeRuntimeAdapterFromCapture(
|
||||
capture: CapturedOpenCodeBridge
|
||||
): OpenCodeTeamRuntimeAdapter {
|
||||
return new OpenCodeTeamRuntimeAdapter(capture.bridge);
|
||||
}
|
||||
|
||||
export function parseOpenCodeE2EModelList(): string[] {
|
||||
const raw = process.env.OPENCODE_E2E_MODELS?.trim();
|
||||
if (!raw) {
|
||||
const single = process.env.OPENCODE_E2E_MODEL?.trim();
|
||||
return single ? [single] : ['opencode/big-pickle'];
|
||||
}
|
||||
return raw
|
||||
.split(',')
|
||||
.map((model) => model.trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function readyOpenCodeReadiness(modelId: string): OpenCodeTeamLaunchReadiness {
|
||||
return {
|
||||
state: 'ready',
|
||||
launchAllowed: true,
|
||||
modelId,
|
||||
availableModels: [modelId],
|
||||
opencodeVersion: '1.14.19',
|
||||
installMethod: 'brew',
|
||||
binaryPath: '/opt/homebrew/bin/opencode',
|
||||
hostHealthy: true,
|
||||
appMcpConnected: true,
|
||||
requiredToolsPresent: true,
|
||||
permissionBridgeReady: true,
|
||||
runtimeStoresReady: true,
|
||||
supportLevel: 'production_supported',
|
||||
missing: [],
|
||||
diagnostics: [],
|
||||
evidence: {
|
||||
capabilitiesReady: true,
|
||||
mcpToolProofRoute: '/experimental/tool/ids',
|
||||
observedMcpTools: [...REQUIRED_AGENT_TEAMS_APP_TOOL_IDS],
|
||||
runtimeStoreReadinessReason: 'runtime_store_manifest_valid',
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function buildReadyLaunchData(
|
||||
command: OpenCodeLaunchTeamCommandBody,
|
||||
modelId: string
|
||||
): OpenCodeLaunchTeamCommandData {
|
||||
return {
|
||||
runId: command.runId,
|
||||
teamLaunchState: 'ready',
|
||||
durableCheckpoints: [
|
||||
{ name: 'required_tools_proven', observedAt: '2026-04-21T00:00:00.000Z' },
|
||||
{ name: 'delivery_ready', observedAt: '2026-04-21T00:00:00.000Z' },
|
||||
{ name: 'run_ready', observedAt: '2026-04-21T00:00:00.000Z' },
|
||||
],
|
||||
members: Object.fromEntries(
|
||||
command.members.map((member, index) => [
|
||||
member.name,
|
||||
{
|
||||
sessionId: `semantic-session-${member.name}`,
|
||||
launchState: 'confirmed_alive' as const,
|
||||
runtimePid: 31_000 + index,
|
||||
model: modelId,
|
||||
evidence: [
|
||||
{ kind: 'required_tools_proven', observedAt: '2026-04-21T00:00:00.000Z' },
|
||||
{ kind: 'delivery_ready', observedAt: '2026-04-21T00:00:00.000Z' },
|
||||
{ kind: 'member_ready', observedAt: '2026-04-21T00:00:00.000Z' },
|
||||
{ kind: 'run_ready', observedAt: '2026-04-21T00:00:00.000Z' },
|
||||
],
|
||||
diagnostics: [],
|
||||
},
|
||||
])
|
||||
),
|
||||
warnings: [],
|
||||
diagnostics: [],
|
||||
};
|
||||
}
|
||||
|
||||
function buildAcceptedMessageData(
|
||||
command: OpenCodeSendMessageCommandBody
|
||||
): OpenCodeSendMessageCommandData {
|
||||
return {
|
||||
accepted: true,
|
||||
sessionId: `semantic-session-${command.memberName}`,
|
||||
memberName: command.memberName,
|
||||
runtimePid: 41_000,
|
||||
prePromptCursor: 'semantic-pre-prompt-cursor',
|
||||
responseObservation: {
|
||||
state: 'responded_tool_call',
|
||||
deliveredUserMessageId: command.messageId ?? null,
|
||||
assistantMessageId: 'semantic-assistant-message',
|
||||
toolCallNames: ['agent-teams_message_send'],
|
||||
visibleMessageToolCallId: 'semantic-tool-call',
|
||||
visibleReplyMessageId: command.messageId ?? null,
|
||||
visibleReplyCorrelation: command.messageId ? 'relayOfMessageId' : null,
|
||||
latestAssistantPreview: 'semantic accepted message',
|
||||
reason: null,
|
||||
},
|
||||
diagnostics: [],
|
||||
};
|
||||
}
|
||||
|
||||
function buildConfirmedMemberEvidence(
|
||||
memberName: string,
|
||||
model: string | null,
|
||||
index: number
|
||||
): TeamRuntimeMemberLaunchEvidence {
|
||||
return {
|
||||
memberName,
|
||||
providerId: 'opencode',
|
||||
launchState: 'confirmed_alive',
|
||||
agentToolAccepted: true,
|
||||
runtimeAlive: true,
|
||||
bootstrapConfirmed: true,
|
||||
hardFailure: false,
|
||||
sessionId: `semantic-session-${memberName}`,
|
||||
runtimePid: 21_000 + index,
|
||||
diagnostics: [`captured OpenCode launch ready${model ? ` for ${model}` : ''}`],
|
||||
};
|
||||
}
|
||||
|
|
@ -257,6 +257,64 @@ describe('RuntimeProviderManagementPanelView', () => {
|
|||
expect(host.querySelector('[data-testid="runtime-provider-search"]')).not.toBeNull();
|
||||
});
|
||||
|
||||
it('does not open a model list for a render-only filtered fallback provider', async () => {
|
||||
const host = document.createElement('div');
|
||||
document.body.appendChild(host);
|
||||
const root = createRoot(host);
|
||||
const actions = createActions();
|
||||
const openRouterProvider = {
|
||||
...createState().view!.providers[0],
|
||||
state: 'connected' as const,
|
||||
modelCount: 174,
|
||||
actions: [],
|
||||
};
|
||||
const openAiProvider = {
|
||||
...openRouterProvider,
|
||||
providerId: 'openai',
|
||||
displayName: 'OpenAI',
|
||||
recommended: false,
|
||||
defaultModelId: 'openai/gpt-5.4-mini-fast',
|
||||
};
|
||||
|
||||
await act(async () => {
|
||||
root.render(
|
||||
React.createElement(RuntimeProviderManagementPanelView, {
|
||||
state: createState({
|
||||
view: {
|
||||
...createState().view!,
|
||||
providers: [openRouterProvider, openAiProvider],
|
||||
},
|
||||
providers: [openRouterProvider, openAiProvider],
|
||||
selectedProviderId: 'openrouter',
|
||||
modelPickerProviderId: 'openrouter',
|
||||
modelPickerMode: 'use',
|
||||
providerQuery: 'openai',
|
||||
models: [
|
||||
{
|
||||
providerId: 'openrouter',
|
||||
modelId: 'openrouter/openai/gpt-oss-20b:free',
|
||||
displayName: 'openai/gpt-oss-20b:free',
|
||||
sourceLabel: 'OpenRouter',
|
||||
free: true,
|
||||
default: false,
|
||||
availability: 'untested',
|
||||
},
|
||||
],
|
||||
}),
|
||||
actions,
|
||||
disabled: false,
|
||||
})
|
||||
);
|
||||
await Promise.resolve();
|
||||
});
|
||||
|
||||
expect(host.textContent).toContain('OpenAI');
|
||||
expect(host.textContent).not.toContain('OpenRouter');
|
||||
expect(
|
||||
host.querySelector('[data-testid="runtime-provider-model-loading-skeleton"]')
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
it('opens the OpenCode provider directory and renders directory rows', async () => {
|
||||
const host = document.createElement('div');
|
||||
document.body.appendChild(host);
|
||||
|
|
|
|||
Loading…
Reference in a new issue