admin-valentin/agent-ecosystem
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

test(opencode): harden deep live launch smoke

Browse source
This commit is contained in:
777genius 2026-05-20 18:24:13 +03:00
parent a88e9f74b7
commit 187a2697f7
3 changed files with 125 additions and 129 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

234
test-results/opencode-semantic-model-gauntlet/model-gauntlet-results.json
View file

@ -1,5 +1,5 @@
{ {
"generatedAt": "2026-05-18T13:16:59.867Z", "generatedAt": "2026-05-20T15:19:19.600Z",
"runsPerModel": 1, "runsPerModel": 1,
"qualification": { "qualification": {
"minimumAverageScore": 80, "minimumAverageScore": 80,
@ -10,23 +10,23 @@
"models": [ "models": [
{ {
"model": "opencode/big-pickle", "model": "opencode/big-pickle",
"verdict": "tested-only", "verdict": "recommended",
"confidence": "low", "confidence": "low",
"qualified": false, "qualified": true,
"readinessScore": 54, "readinessScore": 100,
"averageScore": 35, "averageScore": 100,
"consistencyScore": 100, "consistencyScore": 100,
"behavioralAverageScore": 35, "behavioralAverageScore": 100,
"minScore": 35, "minScore": 100,
"successfulRuns": 0, "successfulRuns": 1,
"countedRuns": 1, "countedRuns": 1,
"hardFailures": 1, "hardFailures": 0,
"providerInfraFailures": 0, "providerInfraFailures": 0,
"runtimeTransportFailures": 0, "runtimeTransportFailures": 0,
"modelBehaviorFailures": 1, "modelBehaviorFailures": 0,
"harnessFailures": 0, "harnessFailures": 0,
"p50DurationMs": 133048, "p50DurationMs": 201546,
"p95DurationMs": 133048, "p95DurationMs": 201546,
"stagePassRates": { "stagePassRates": {
"launchBootstrap": { "launchBootstrap": {
"passed": 1, "passed": 1,
@ -39,19 +39,19 @@
"rate": 100 "rate": 100
}, },
"peerRelayAB": { "peerRelayAB": {
"passed": 0, "passed": 1,
"total": 1, "total": 1,
"rate": 0 "rate": 100
}, },
"peerRelayBC": { "peerRelayBC": {
"passed": 0, "passed": 1,
"total": 1, "total": 1,
"rate": 0 "rate": 100
}, },
"concurrentReplies": { "concurrentReplies": {
"passed": 0, "passed": 1,
"total": 1, "total": 1,
"rate": 0 "rate": 100
}, },
"taskRefs": { "taskRefs": {
"passed": 1, "passed": 1,
@ -59,19 +59,19 @@
"rate": 100 "rate": 100
}, },
"cleanTranscript": { "cleanTranscript": {
"passed": 0, "passed": 1,
"total": 1, "total": 1,
"rate": 0 "rate": 100
}, },
"noDuplicateTokens": { "noDuplicateTokens": {
"passed": 0, "passed": 1,
"total": 1, "total": 1,
"rate": 0 "rate": 100
}, },
"latencyStable": { "latencyStable": {
"passed": 0, "passed": 1,
"total": 1, "total": 1,
"rate": 0 "rate": 100
} }
}, },
"taskRefPassRates": { "taskRefPassRates": {
@ -81,24 +81,24 @@
"rate": 100 "rate": 100
}, },
"peerRelayAB": { "peerRelayAB": {
"passed": 0, "passed": 1,
"total": 0, "total": 1,
"rate": null "rate": 100
}, },
"peerRelayBC": { "peerRelayBC": {
"passed": 0, "passed": 1,
"total": 0, "total": 1,
"rate": null "rate": 100
}, },
"concurrentBob": { "concurrentBob": {
"passed": 0, "passed": 1,
"total": 0, "total": 1,
"rate": null "rate": 100
}, },
"concurrentTom": { "concurrentTom": {
"passed": 0, "passed": 1,
"total": 0, "total": 1,
"rate": null "rate": 100
} }
}, },
"protocolViolationTotals": { "protocolViolationTotals": {
@ -107,64 +107,24 @@
"affectedRuns": 0 "affectedRuns": 0
}, },
"stageFailureImpact": [ "stageFailureImpact": [
{
"stage": "concurrentReplies",
"failedRuns": 1,
"weightedLoss": 15,
"passRate": {
"passed": 0,
"total": 1,
"rate": 0
}
},
{
"stage": "peerRelayAB",
"failedRuns": 1,
"weightedLoss": 15,
"passRate": {
"passed": 0,
"total": 1,
"rate": 0
}
},
{
"stage": "peerRelayBC",
"failedRuns": 1,
"weightedLoss": 15,
"passRate": {
"passed": 0,
"total": 1,
"rate": 0
}
},
{ {
"stage": "cleanTranscript", "stage": "cleanTranscript",
"failedRuns": 1, "failedRuns": 0,
"weightedLoss": 10, "weightedLoss": 0,
"passRate": { "passRate": {
"passed": 0, "passed": 1,
"total": 1, "total": 1,
"rate": 0 "rate": 100
} }
}, },
{ {
"stage": "latencyStable", "stage": "concurrentReplies",
"failedRuns": 1, "failedRuns": 0,
"weightedLoss": 5, "weightedLoss": 0,
"passRate": { "passRate": {
"passed": 0, "passed": 1,
"total": 1, "total": 1,
"rate": 0 "rate": 100
}
},
{
"stage": "noDuplicateTokens",
"failedRuns": 1,
"weightedLoss": 5,
"passRate": {
"passed": 0,
"total": 1,
"rate": 0
} }
}, },
{ {
@ -177,6 +137,16 @@
"rate": 100 "rate": 100
} }
}, },
{
"stage": "latencyStable",
"failedRuns": 0,
"weightedLoss": 0,
"passRate": {
"passed": 1,
"total": 1,
"rate": 100
}
},
{ {
"stage": "launchBootstrap", "stage": "launchBootstrap",
"failedRuns": 0, "failedRuns": 0,
@ -187,6 +157,36 @@
"rate": 100 "rate": 100
} }
}, },
{
"stage": "noDuplicateTokens",
"failedRuns": 0,
"weightedLoss": 0,
"passRate": {
"passed": 1,
"total": 1,
"rate": 100
}
},
{
"stage": "peerRelayAB",
"failedRuns": 0,
"weightedLoss": 0,
"passRate": {
"passed": 1,
"total": 1,
"rate": 100
}
},
{
"stage": "peerRelayBC",
"failedRuns": 0,
"weightedLoss": 0,
"passRate": {
"passed": 1,
"total": 1,
"rate": 100
}
},
{ {
"stage": "taskRefs", "stage": "taskRefs",
"failedRuns": 0, "failedRuns": 0,
@ -200,45 +200,42 @@
], ],
"scoreStability": { "scoreStability": {
"sampleSize": 1, "sampleSize": 1,
"minScore": 35, "minScore": 100,
"maxScore": 35, "maxScore": 100,
"spread": 0, "spread": 0,
"standardDeviation": 0, "standardDeviation": 0,
"consistencyScore": 100 "consistencyScore": 100
}, },
"dominantFailureCategory": "model-behavior", "dominantFailureCategory": "none",
"recommendationBlockers": [ "recommendationBlockers": [],
"overall average 35 < 80",
"behavioral average 35 < 80",
"successful runs 0 < 1",
"hard failures 1",
"model-behavior failures 1",
"highest weighted stage loss concurrentReplies=15"
],
"runs": [ "runs": [
{ {
"runIndex": 1, "runIndex": 1,
"passed": false, "passed": true,
"score": 35, "score": 100,
"countedForRecommendation": true, "countedForRecommendation": true,
"outcome": "behavioral-fail", "outcome": "passed",
"failureCategory": "model-behavior", "failureCategory": "none",
"primaryFailure": "Timed out waiting for OpenCode member bob to become idle. Last durableState: reply_pending", "primaryFailure": null,
"durationMs": 133048, "durationMs": 201546,
"hardFailure": true, "hardFailure": false,
"stageDurationsMs": { "stageDurationsMs": {
"setup": 371, "setup": 171,
"launchBootstrap": 23735, "launchBootstrap": 41905,
"materializeTasks": 36, "materializeTasks": 39,
"directReply": 17327 "directReply": 29109,
"peerRelayAB": 45148,
"peerRelayBC": 39967,
"concurrentReplies": 28807,
"hygiene": 1
}, },
"stageFailures": {}, "stageFailures": {},
"taskRefChecks": { "taskRefChecks": {
"directReply": true, "directReply": true,
"peerRelayAB": null, "peerRelayAB": true,
"peerRelayBC": null, "peerRelayBC": true,
"concurrentBob": null, "concurrentBob": true,
"concurrentTom": null "concurrentTom": true
}, },
"protocolViolations": { "protocolViolations": {
"badMessages": 0, "badMessages": 0,
@ -247,17 +244,16 @@
"stages": { "stages": {
"launchBootstrap": true, "launchBootstrap": true,
"directReply": true, "directReply": true,
"peerRelayAB": false, "peerRelayAB": true,
"peerRelayBC": false, "peerRelayBC": true,
"concurrentReplies": false, "concurrentReplies": true,
"taskRefs": true, "taskRefs": true,
"cleanTranscript": false, "cleanTranscript": true,
"noDuplicateTokens": false, "noDuplicateTokens": true,
"latencyStable": false "latencyStable": true
}, },
"diagnostics": [ "diagnostics": [
"runId=9fb17ac5-fc66-4b01-831e-90a04b1e2304", "runId=1d6a50c3-c5cc-4c1e-91a0-d0e34a2229a3"
"Timed out waiting for OpenCode member bob to become idle. Last durableState: reply_pending"
] ]
} }
] ]

18
test-results/opencode-semantic-model-gauntlet/model-gauntlet-results.md
View file

@ -1,6 +1,6 @@
# OpenCode Model Gauntlet Results # OpenCode Model Gauntlet Results
Generated: 2026-05-18T13:16:59.867Z Generated: 2026-05-20T15:19:19.600Z
Runs per model: 1 Runs per model: 1
Recommended threshold: average >= 80, successful runs >= 1, consistency >= 85, hard failures = 0 Recommended threshold: average >= 80, successful runs >= 1, consistency >= 85, hard failures = 0
@ -13,25 +13,25 @@ Scoring weights: launchBootstrap=15, directReply=10, peerRelayAB=15, peerRelayBC
| Model | Verdict | Confidence | Readiness | Consistency | Score Spread | Behavior Avg | Overall Avg | Counted | Pass Runs | Weakest Stage | Weakest TaskRef | Dominant Failure | Blockers | Provider Infra | Runtime Transport | Model Fails | Protocol Runs | p50 | p95 | | Model | Verdict | Confidence | Readiness | Consistency | Score Spread | Behavior Avg | Overall Avg | Counted | Pass Runs | Weakest Stage | Weakest TaskRef | Dominant Failure | Blockers | Provider Infra | Runtime Transport | Model Fails | Protocol Runs | p50 | p95 |
| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | | --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: |
| `opencode/big-pickle` | Tested only | low | 54 | 100 | 0 | 35 | 35 | 1/1 | 0/1 | cleanTranscript 0/1 (0%) | directReply 1/1 (100%) | model-behavior | overall average 35 < 80; behavioral average 35 < 80; successful runs 0 < 1; hard failures 1; model-behavior failures 1; highest weighted stage loss concurrentReplies=15 | 0 | 0 | 1 | 0 | 133048ms | 133048ms | | `opencode/big-pickle` | Recommended | low | 100 | 100 | 0 | 100 | 100 | 1/1 | 1/1 | cleanTranscript 1/1 (100%) | concurrentBob 1/1 (100%) | none | - | 0 | 0 | 0 | 0 | 201546ms | 201546ms |
## opencode/big-pickle ## opencode/big-pickle
Readiness score: 54. Readiness score: 100.
Score stability: consistency=100, min=35, max=35, spread=0, stdDev=0, samples=1. Score stability: consistency=100, min=100, max=100, spread=0, stdDev=0, samples=1.
Recommendation blockers: overall average 35 < 80; behavioral average 35 < 80; successful runs 0 < 1; hard failures 1; model-behavior failures 1; highest weighted stage loss concurrentReplies=15. Recommendation blockers: -.
Weighted stage impact: concurrentReplies:loss=15, failed=1, pass=0/1 (0%); peerRelayAB:loss=15, failed=1, pass=0/1 (0%); peerRelayBC:loss=15, failed=1, pass=0/1 (0%); cleanTranscript:loss=10, failed=1, pass=0/1 (0%); latencyStable:loss=5, failed=1, pass=0/1 (0%). Weighted stage impact: -.
Stage pass rates: launchBootstrap:1/1 (100%), directReply:1/1 (100%), peerRelayAB:0/1 (0%), peerRelayBC:0/1 (0%), concurrentReplies:0/1 (0%), taskRefs:1/1 (100%), cleanTranscript:0/1 (0%), noDuplicateTokens:0/1 (0%), latencyStable:0/1 (0%). Stage pass rates: launchBootstrap:1/1 (100%), directReply:1/1 (100%), peerRelayAB:1/1 (100%), peerRelayBC:1/1 (100%), concurrentReplies:1/1 (100%), taskRefs:1/1 (100%), cleanTranscript:1/1 (100%), noDuplicateTokens:1/1 (100%), latencyStable:1/1 (100%).
TaskRef pass rates: directReply:1/1 (100%), peerRelayAB:n/a, peerRelayBC:n/a, concurrentBob:n/a, concurrentTom:n/a. TaskRef pass rates: directReply:1/1 (100%), peerRelayAB:1/1 (100%), peerRelayBC:1/1 (100%), concurrentBob:1/1 (100%), concurrentTom:1/1 (100%).
Protocol totals: badMessages=0, duplicateOrMissingTokens=0, affectedRuns=0. Protocol totals: badMessages=0, duplicateOrMissingTokens=0, affectedRuns=0.
| Run | Outcome | Category | Score | Counted | Duration | Failed Stages | Slowest Stage | TaskRefs | Protocol | Diagnostics | | Run | Outcome | Category | Score | Counted | Duration | Failed Stages | Slowest Stage | TaskRefs | Protocol | Diagnostics |
| ---: | --- | --- | ---: | --- | ---: | --- | --- | --- | --- | --- | | ---: | --- | --- | ---: | --- | ---: | --- | --- | --- | --- | --- |
| 1 | behavioral-fail | model-behavior | 35 | yes | 133048ms | peerRelayAB, peerRelayBC, concurrentReplies, cleanTranscript, noDuplicateTokens, latencyStable | launchBootstrap:23735ms | directReply:ok | - | Timed out waiting for OpenCode member bob to become idle. Last durableState: reply_pending | | 1 | passed | none | 100 | yes | 201546ms | - | peerRelayAB:45148ms | directReply:ok, peerRelayAB:ok, peerRelayBC:ok, concurrentBob:ok, concurrentTom:ok | - | runId=1d6a50c3-c5cc-4c1e-91a0-d0e34a2229a3 |

2
test/main/services/team/OpenCodeAcceptFastDelivery.live-e2e.test.ts
View file

@ -78,7 +78,7 @@ liveDescribe('OpenCode accept-fast delivery live e2e', () => {
} else { } else {
await fs.rm(tempDir, { recursive: true, force: true }); await fs.rm(tempDir, { recursive: true, force: true });
} }
}); }, 90_000);
it( it(
'returns after prompt acceptance and later projects exact-session task logs', 'returns after prompt acceptance and later projects exact-session task logs',