266 lines
6.4 KiB
JSON
266 lines
6.4 KiB
JSON
{
|
|
"generatedAt": "2026-05-18T13:16:59.867Z",
|
|
"runsPerModel": 1,
|
|
"qualification": {
|
|
"minimumAverageScore": 80,
|
|
"minimumSuccessfulRuns": 1,
|
|
"minimumConsistencyScore": 85,
|
|
"requireNoHardFailures": true
|
|
},
|
|
"models": [
|
|
{
|
|
"model": "opencode/big-pickle",
|
|
"verdict": "tested-only",
|
|
"confidence": "low",
|
|
"qualified": false,
|
|
"readinessScore": 54,
|
|
"averageScore": 35,
|
|
"consistencyScore": 100,
|
|
"behavioralAverageScore": 35,
|
|
"minScore": 35,
|
|
"successfulRuns": 0,
|
|
"countedRuns": 1,
|
|
"hardFailures": 1,
|
|
"providerInfraFailures": 0,
|
|
"runtimeTransportFailures": 0,
|
|
"modelBehaviorFailures": 1,
|
|
"harnessFailures": 0,
|
|
"p50DurationMs": 133048,
|
|
"p95DurationMs": 133048,
|
|
"stagePassRates": {
|
|
"launchBootstrap": {
|
|
"passed": 1,
|
|
"total": 1,
|
|
"rate": 100
|
|
},
|
|
"directReply": {
|
|
"passed": 1,
|
|
"total": 1,
|
|
"rate": 100
|
|
},
|
|
"peerRelayAB": {
|
|
"passed": 0,
|
|
"total": 1,
|
|
"rate": 0
|
|
},
|
|
"peerRelayBC": {
|
|
"passed": 0,
|
|
"total": 1,
|
|
"rate": 0
|
|
},
|
|
"concurrentReplies": {
|
|
"passed": 0,
|
|
"total": 1,
|
|
"rate": 0
|
|
},
|
|
"taskRefs": {
|
|
"passed": 1,
|
|
"total": 1,
|
|
"rate": 100
|
|
},
|
|
"cleanTranscript": {
|
|
"passed": 0,
|
|
"total": 1,
|
|
"rate": 0
|
|
},
|
|
"noDuplicateTokens": {
|
|
"passed": 0,
|
|
"total": 1,
|
|
"rate": 0
|
|
},
|
|
"latencyStable": {
|
|
"passed": 0,
|
|
"total": 1,
|
|
"rate": 0
|
|
}
|
|
},
|
|
"taskRefPassRates": {
|
|
"directReply": {
|
|
"passed": 1,
|
|
"total": 1,
|
|
"rate": 100
|
|
},
|
|
"peerRelayAB": {
|
|
"passed": 0,
|
|
"total": 0,
|
|
"rate": null
|
|
},
|
|
"peerRelayBC": {
|
|
"passed": 0,
|
|
"total": 0,
|
|
"rate": null
|
|
},
|
|
"concurrentBob": {
|
|
"passed": 0,
|
|
"total": 0,
|
|
"rate": null
|
|
},
|
|
"concurrentTom": {
|
|
"passed": 0,
|
|
"total": 0,
|
|
"rate": null
|
|
}
|
|
},
|
|
"protocolViolationTotals": {
|
|
"badMessages": 0,
|
|
"duplicateOrMissingTokens": 0,
|
|
"affectedRuns": 0
|
|
},
|
|
"stageFailureImpact": [
|
|
{
|
|
"stage": "concurrentReplies",
|
|
"failedRuns": 1,
|
|
"weightedLoss": 15,
|
|
"passRate": {
|
|
"passed": 0,
|
|
"total": 1,
|
|
"rate": 0
|
|
}
|
|
},
|
|
{
|
|
"stage": "peerRelayAB",
|
|
"failedRuns": 1,
|
|
"weightedLoss": 15,
|
|
"passRate": {
|
|
"passed": 0,
|
|
"total": 1,
|
|
"rate": 0
|
|
}
|
|
},
|
|
{
|
|
"stage": "peerRelayBC",
|
|
"failedRuns": 1,
|
|
"weightedLoss": 15,
|
|
"passRate": {
|
|
"passed": 0,
|
|
"total": 1,
|
|
"rate": 0
|
|
}
|
|
},
|
|
{
|
|
"stage": "cleanTranscript",
|
|
"failedRuns": 1,
|
|
"weightedLoss": 10,
|
|
"passRate": {
|
|
"passed": 0,
|
|
"total": 1,
|
|
"rate": 0
|
|
}
|
|
},
|
|
{
|
|
"stage": "latencyStable",
|
|
"failedRuns": 1,
|
|
"weightedLoss": 5,
|
|
"passRate": {
|
|
"passed": 0,
|
|
"total": 1,
|
|
"rate": 0
|
|
}
|
|
},
|
|
{
|
|
"stage": "noDuplicateTokens",
|
|
"failedRuns": 1,
|
|
"weightedLoss": 5,
|
|
"passRate": {
|
|
"passed": 0,
|
|
"total": 1,
|
|
"rate": 0
|
|
}
|
|
},
|
|
{
|
|
"stage": "directReply",
|
|
"failedRuns": 0,
|
|
"weightedLoss": 0,
|
|
"passRate": {
|
|
"passed": 1,
|
|
"total": 1,
|
|
"rate": 100
|
|
}
|
|
},
|
|
{
|
|
"stage": "launchBootstrap",
|
|
"failedRuns": 0,
|
|
"weightedLoss": 0,
|
|
"passRate": {
|
|
"passed": 1,
|
|
"total": 1,
|
|
"rate": 100
|
|
}
|
|
},
|
|
{
|
|
"stage": "taskRefs",
|
|
"failedRuns": 0,
|
|
"weightedLoss": 0,
|
|
"passRate": {
|
|
"passed": 1,
|
|
"total": 1,
|
|
"rate": 100
|
|
}
|
|
}
|
|
],
|
|
"scoreStability": {
|
|
"sampleSize": 1,
|
|
"minScore": 35,
|
|
"maxScore": 35,
|
|
"spread": 0,
|
|
"standardDeviation": 0,
|
|
"consistencyScore": 100
|
|
},
|
|
"dominantFailureCategory": "model-behavior",
|
|
"recommendationBlockers": [
|
|
"overall average 35 < 80",
|
|
"behavioral average 35 < 80",
|
|
"successful runs 0 < 1",
|
|
"hard failures 1",
|
|
"model-behavior failures 1",
|
|
"highest weighted stage loss concurrentReplies=15"
|
|
],
|
|
"runs": [
|
|
{
|
|
"runIndex": 1,
|
|
"passed": false,
|
|
"score": 35,
|
|
"countedForRecommendation": true,
|
|
"outcome": "behavioral-fail",
|
|
"failureCategory": "model-behavior",
|
|
"primaryFailure": "Timed out waiting for OpenCode member bob to become idle. Last durableState: reply_pending",
|
|
"durationMs": 133048,
|
|
"hardFailure": true,
|
|
"stageDurationsMs": {
|
|
"setup": 371,
|
|
"launchBootstrap": 23735,
|
|
"materializeTasks": 36,
|
|
"directReply": 17327
|
|
},
|
|
"stageFailures": {},
|
|
"taskRefChecks": {
|
|
"directReply": true,
|
|
"peerRelayAB": null,
|
|
"peerRelayBC": null,
|
|
"concurrentBob": null,
|
|
"concurrentTom": null
|
|
},
|
|
"protocolViolations": {
|
|
"badMessages": 0,
|
|
"duplicateOrMissingTokens": []
|
|
},
|
|
"stages": {
|
|
"launchBootstrap": true,
|
|
"directReply": true,
|
|
"peerRelayAB": false,
|
|
"peerRelayBC": false,
|
|
"concurrentReplies": false,
|
|
"taskRefs": true,
|
|
"cleanTranscript": false,
|
|
"noDuplicateTokens": false,
|
|
"latencyStable": false
|
|
},
|
|
"diagnostics": [
|
|
"runId=9fb17ac5-fc66-4b01-831e-90a04b1e2304",
|
|
"Timed out waiting for OpenCode member bob to become idle. Last durableState: reply_pending"
|
|
]
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|