mirror of
https://github.com/usips/trufflepig.git
synced 2026-10-27 13:38:01 -04:00
Joshua MoonandClaude Fable 5.1
7d7782aed0
Rerank the top 32 fused files with a pinned bge-reranker-v2-m3 cross-encoder, opt-in via --rerank or [semantic] rerank = true. The shared worker runs the reranker on its own executor thread and GPU (rerank_model_dir, rerank_gpu_uuid) with a 1500 ms deadline; failures keep fused order and report rerank_status in coverage. Adds the cuda_rerank evaluation arm and the measured development replay: file recall@5 0.542 and recall@10 0.625 against 0.292 and 0.542 for file-first lexical on the same day. Co-Authored-By: Claude Fable 5.1 <[email protected]>
763 lines
22 KiB
JSON
763 lines
22 KiB
JSON
{
|
|
"schema_version": 1,
|
|
"split": "development",
|
|
"held_out_claim": false,
|
|
"measured_commit": "3835a69",
|
|
"default_semantic_enabled": false,
|
|
"arms": {
|
|
"oldlexical": {
|
|
"binary_sha256": "727057b4771a19ffa2c2b68273fb316e3c8991d3b790df248db32f814d7400e6",
|
|
"page_budget_tokens": 600,
|
|
"summary": {
|
|
"tasks": 12,
|
|
"snapshot_invalid": 0,
|
|
"unavailable": 0,
|
|
"errors": 0,
|
|
"truncated_searches": 11,
|
|
"misses": 8,
|
|
"mean_file_recall_at_5": 0.25,
|
|
"mean_file_recall_at_10": 0.25,
|
|
"mean_latency_seconds": 0.6679960291100239,
|
|
"usage": {
|
|
"tool_calls": 36,
|
|
"stdout_bytes": 52077,
|
|
"stderr_bytes": 0,
|
|
"emitted_tokens": 16801,
|
|
"tokenizer": "o200k_base",
|
|
"latency_seconds": 24.04785704796086,
|
|
"token_cost_censored": false
|
|
}
|
|
},
|
|
"tasks": [
|
|
{
|
|
"task_id": "lunatic-document-append-guard",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 0.0,
|
|
"file_recall_at_10": 0.0,
|
|
"hits_emitted": 2,
|
|
"page_complete": false
|
|
}
|
|
},
|
|
{
|
|
"task_id": "lunatic-legacy-airlock-fixture",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 0.0,
|
|
"file_recall_at_10": 0.0,
|
|
"hits_emitted": 2,
|
|
"page_complete": false
|
|
}
|
|
},
|
|
{
|
|
"task_id": "lunatic-cycle-link-validation",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 0.0,
|
|
"file_recall_at_10": 0.0,
|
|
"hits_emitted": 2,
|
|
"page_complete": false
|
|
}
|
|
},
|
|
{
|
|
"task_id": "lunatic-script-permission-interface",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 0.0,
|
|
"file_recall_at_10": 0.0,
|
|
"hits_emitted": 2,
|
|
"page_complete": false
|
|
}
|
|
},
|
|
{
|
|
"task_id": "tfs-airlock-permission-fixture",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 0.5,
|
|
"file_recall_at_10": 0.5,
|
|
"hits_emitted": 2,
|
|
"page_complete": false
|
|
}
|
|
},
|
|
{
|
|
"task_id": "tfs-airlock-cycle-state",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 1.0,
|
|
"file_recall_at_10": 1.0,
|
|
"hits_emitted": 2,
|
|
"page_complete": false
|
|
}
|
|
},
|
|
{
|
|
"task_id": "tfs-airlock-rebuild-fixture",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 1.0,
|
|
"file_recall_at_10": 1.0,
|
|
"hits_emitted": 2,
|
|
"page_complete": false
|
|
}
|
|
},
|
|
{
|
|
"task_id": "tfs-writing-commit",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 0.5,
|
|
"file_recall_at_10": 0.5,
|
|
"hits_emitted": 2,
|
|
"page_complete": true
|
|
}
|
|
},
|
|
{
|
|
"task_id": "tg-airlock-controller-cycle",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 0.0,
|
|
"file_recall_at_10": 0.0,
|
|
"hits_emitted": 2,
|
|
"page_complete": false
|
|
}
|
|
},
|
|
{
|
|
"task_id": "tg-airlock-pump-pressure",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 0.0,
|
|
"file_recall_at_10": 0.0,
|
|
"hits_emitted": 2,
|
|
"page_complete": false
|
|
}
|
|
},
|
|
{
|
|
"task_id": "tg-airlock-secure-open",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 0.0,
|
|
"file_recall_at_10": 0.0,
|
|
"hits_emitted": 2,
|
|
"page_complete": false
|
|
}
|
|
},
|
|
{
|
|
"task_id": "tg-airlock-wire-permissions",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 0.0,
|
|
"file_recall_at_10": 0.0,
|
|
"hits_emitted": 2,
|
|
"page_complete": false
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"newfilefirstlexical": {
|
|
"binary_sha256": "92865aa758bdcd5e552e7d1e5aaef69365b1c8a11bc2f59cfbdb6d79df5b01db",
|
|
"page_budget_tokens": 600,
|
|
"summary": {
|
|
"tasks": 12,
|
|
"snapshot_invalid": 0,
|
|
"unavailable": 0,
|
|
"errors": 0,
|
|
"truncated_searches": 10,
|
|
"misses": 6,
|
|
"mean_file_recall_at_5": 0.2916666666666667,
|
|
"mean_file_recall_at_10": 0.4583333333333333,
|
|
"mean_latency_seconds": 0.4801186444632809,
|
|
"usage": {
|
|
"tool_calls": 84,
|
|
"stdout_bytes": 215074,
|
|
"stderr_bytes": 0,
|
|
"emitted_tokens": 67921,
|
|
"tokenizer": "o200k_base",
|
|
"latency_seconds": 40.329966134915594,
|
|
"token_cost_censored": false
|
|
}
|
|
},
|
|
"tasks": [
|
|
{
|
|
"task_id": "lunatic-document-append-guard",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 1.0,
|
|
"file_recall_at_10": 1.0,
|
|
"hits_emitted": 6,
|
|
"page_complete": true
|
|
}
|
|
},
|
|
{
|
|
"task_id": "lunatic-legacy-airlock-fixture",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 0.0,
|
|
"file_recall_at_10": 0.0,
|
|
"hits_emitted": 6,
|
|
"page_complete": false
|
|
}
|
|
},
|
|
{
|
|
"task_id": "lunatic-cycle-link-validation",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 0.0,
|
|
"file_recall_at_10": 0.0,
|
|
"hits_emitted": 6,
|
|
"page_complete": false
|
|
}
|
|
},
|
|
{
|
|
"task_id": "lunatic-script-permission-interface",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 0.0,
|
|
"file_recall_at_10": 0.0,
|
|
"hits_emitted": 6,
|
|
"page_complete": false
|
|
}
|
|
},
|
|
{
|
|
"task_id": "tfs-airlock-permission-fixture",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 0.5,
|
|
"file_recall_at_10": 0.5,
|
|
"hits_emitted": 6,
|
|
"page_complete": false
|
|
}
|
|
},
|
|
{
|
|
"task_id": "tfs-airlock-cycle-state",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 1.0,
|
|
"file_recall_at_10": 1.0,
|
|
"hits_emitted": 6,
|
|
"page_complete": false
|
|
}
|
|
},
|
|
{
|
|
"task_id": "tfs-airlock-rebuild-fixture",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 0.0,
|
|
"file_recall_at_10": 0.0,
|
|
"hits_emitted": 6,
|
|
"page_complete": false
|
|
}
|
|
},
|
|
{
|
|
"task_id": "tfs-writing-commit",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 0.0,
|
|
"file_recall_at_10": 0.0,
|
|
"hits_emitted": 6,
|
|
"page_complete": true
|
|
}
|
|
},
|
|
{
|
|
"task_id": "tg-airlock-controller-cycle",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 1.0,
|
|
"file_recall_at_10": 1.0,
|
|
"hits_emitted": 6,
|
|
"page_complete": false
|
|
}
|
|
},
|
|
{
|
|
"task_id": "tg-airlock-pump-pressure",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 0.0,
|
|
"file_recall_at_10": 0.0,
|
|
"hits_emitted": 5,
|
|
"page_complete": false
|
|
}
|
|
},
|
|
{
|
|
"task_id": "tg-airlock-secure-open",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 0.0,
|
|
"file_recall_at_10": 1.0,
|
|
"hits_emitted": 6,
|
|
"page_complete": false
|
|
}
|
|
},
|
|
{
|
|
"task_id": "tg-airlock-wire-permissions",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 0.0,
|
|
"file_recall_at_10": 1.0,
|
|
"hits_emitted": 7,
|
|
"page_complete": false
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"cuda": {
|
|
"binary_sha256": "e5f671a87bef34bdec417f126f87dede2755fcf59d36530f4c2553ad8f73dd3f",
|
|
"page_budget_tokens": 600,
|
|
"summary": {
|
|
"tasks": 12,
|
|
"snapshot_invalid": 0,
|
|
"unavailable": 0,
|
|
"errors": 0,
|
|
"truncated_searches": 12,
|
|
"misses": 8,
|
|
"mean_file_recall_at_5": 0.3333333333333333,
|
|
"mean_file_recall_at_10": 0.3333333333333333,
|
|
"mean_latency_seconds": 0.4304255027680306,
|
|
"usage": {
|
|
"tool_calls": 82,
|
|
"stdout_bytes": 214106,
|
|
"stderr_bytes": 0,
|
|
"emitted_tokens": 67137,
|
|
"tokenizer": "o200k_base",
|
|
"latency_seconds": 35.29489122697851,
|
|
"token_cost_censored": false
|
|
}
|
|
},
|
|
"tasks": [
|
|
{
|
|
"task_id": "lunatic-document-append-guard",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 1.0,
|
|
"file_recall_at_10": 1.0,
|
|
"hits_emitted": 6,
|
|
"page_complete": false
|
|
}
|
|
},
|
|
{
|
|
"task_id": "lunatic-legacy-airlock-fixture",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 0.0,
|
|
"file_recall_at_10": 0.0,
|
|
"hits_emitted": 6,
|
|
"page_complete": false
|
|
}
|
|
},
|
|
{
|
|
"task_id": "lunatic-cycle-link-validation",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 0.0,
|
|
"file_recall_at_10": 0.0,
|
|
"hits_emitted": 6,
|
|
"page_complete": false
|
|
}
|
|
},
|
|
{
|
|
"task_id": "lunatic-script-permission-interface",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 1.0,
|
|
"file_recall_at_10": 1.0,
|
|
"hits_emitted": 6,
|
|
"page_complete": false
|
|
}
|
|
},
|
|
{
|
|
"task_id": "tfs-airlock-permission-fixture",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 0.0,
|
|
"file_recall_at_10": 0.0,
|
|
"hits_emitted": 6,
|
|
"page_complete": false
|
|
}
|
|
},
|
|
{
|
|
"task_id": "tfs-airlock-cycle-state",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 0.0,
|
|
"file_recall_at_10": 0.0,
|
|
"hits_emitted": 6,
|
|
"page_complete": false
|
|
}
|
|
},
|
|
{
|
|
"task_id": "tfs-airlock-rebuild-fixture",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 0.0,
|
|
"file_recall_at_10": 0.0,
|
|
"hits_emitted": 6,
|
|
"page_complete": false
|
|
}
|
|
},
|
|
{
|
|
"task_id": "tfs-writing-commit",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 0.0,
|
|
"file_recall_at_10": 0.0,
|
|
"hits_emitted": 6,
|
|
"page_complete": false
|
|
}
|
|
},
|
|
{
|
|
"task_id": "tg-airlock-controller-cycle",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 1.0,
|
|
"file_recall_at_10": 1.0,
|
|
"hits_emitted": 5,
|
|
"page_complete": false
|
|
}
|
|
},
|
|
{
|
|
"task_id": "tg-airlock-pump-pressure",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 1.0,
|
|
"file_recall_at_10": 1.0,
|
|
"hits_emitted": 5,
|
|
"page_complete": false
|
|
}
|
|
},
|
|
{
|
|
"task_id": "tg-airlock-secure-open",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 0.0,
|
|
"file_recall_at_10": 0.0,
|
|
"hits_emitted": 6,
|
|
"page_complete": false
|
|
}
|
|
},
|
|
{
|
|
"task_id": "tg-airlock-wire-permissions",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 0.0,
|
|
"file_recall_at_10": 0.0,
|
|
"hits_emitted": 6,
|
|
"page_complete": false
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"cuda_rerank": {
|
|
"binary_sha256": "907b07c43e4c6e22702ad0e3d5e9ea4ec621f16d41346b6003f143a23551cd7c",
|
|
"page_budget_tokens": 600,
|
|
"summary": {
|
|
"tasks": 12,
|
|
"snapshot_invalid": 0,
|
|
"unavailable": 0,
|
|
"errors": 0,
|
|
"truncated_searches": 12,
|
|
"misses": 4,
|
|
"mean_file_recall_at_5": 0.5416666666666666,
|
|
"mean_file_recall_at_10": 0.625,
|
|
"mean_latency_seconds": 0.6553997579626967,
|
|
"usage": {
|
|
"tool_calls": 82,
|
|
"stdout_bytes": 218870,
|
|
"stderr_bytes": 0,
|
|
"emitted_tokens": 68316,
|
|
"tokenizer": "o200k_base",
|
|
"latency_seconds": 53.74278015294112,
|
|
"token_cost_censored": false
|
|
}
|
|
},
|
|
"tasks": [
|
|
{
|
|
"task_id": "lunatic-document-append-guard",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 1.0,
|
|
"file_recall_at_10": 1.0,
|
|
"hits_emitted": 6,
|
|
"page_complete": false
|
|
}
|
|
},
|
|
{
|
|
"task_id": "lunatic-legacy-airlock-fixture",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 0.0,
|
|
"file_recall_at_10": 0.0,
|
|
"hits_emitted": 6,
|
|
"page_complete": false
|
|
}
|
|
},
|
|
{
|
|
"task_id": "lunatic-cycle-link-validation",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 1.0,
|
|
"file_recall_at_10": 1.0,
|
|
"hits_emitted": 6,
|
|
"page_complete": false
|
|
}
|
|
},
|
|
{
|
|
"task_id": "lunatic-script-permission-interface",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 1.0,
|
|
"file_recall_at_10": 1.0,
|
|
"hits_emitted": 6,
|
|
"page_complete": false
|
|
}
|
|
},
|
|
{
|
|
"task_id": "tfs-airlock-permission-fixture",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 0.5,
|
|
"file_recall_at_10": 0.5,
|
|
"hits_emitted": 6,
|
|
"page_complete": false
|
|
}
|
|
},
|
|
{
|
|
"task_id": "tfs-airlock-cycle-state",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 0.0,
|
|
"file_recall_at_10": 0.0,
|
|
"hits_emitted": 5,
|
|
"page_complete": false
|
|
}
|
|
},
|
|
{
|
|
"task_id": "tfs-airlock-rebuild-fixture",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 1.0,
|
|
"file_recall_at_10": 1.0,
|
|
"hits_emitted": 6,
|
|
"page_complete": false
|
|
}
|
|
},
|
|
{
|
|
"task_id": "tfs-writing-commit",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 0.0,
|
|
"file_recall_at_10": 0.0,
|
|
"hits_emitted": 6,
|
|
"page_complete": false
|
|
}
|
|
},
|
|
{
|
|
"task_id": "tg-airlock-controller-cycle",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 1.0,
|
|
"file_recall_at_10": 1.0,
|
|
"hits_emitted": 6,
|
|
"page_complete": false
|
|
}
|
|
},
|
|
{
|
|
"task_id": "tg-airlock-pump-pressure",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 0.0,
|
|
"file_recall_at_10": 1.0,
|
|
"hits_emitted": 6,
|
|
"page_complete": false
|
|
}
|
|
},
|
|
{
|
|
"task_id": "tg-airlock-secure-open",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 1.0,
|
|
"file_recall_at_10": 1.0,
|
|
"hits_emitted": 6,
|
|
"page_complete": false
|
|
}
|
|
},
|
|
{
|
|
"task_id": "tg-airlock-wire-permissions",
|
|
"snapshot_valid": true,
|
|
"page": {
|
|
"file_recall_at_5": 0.0,
|
|
"file_recall_at_10": 0.0,
|
|
"hits_emitted": 5,
|
|
"page_complete": false
|
|
}
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"latency": {
|
|
"serial": {
|
|
"requests": 100,
|
|
"p50_seconds": 1.0717706709983759,
|
|
"p95_seconds": 1.4897110809979495,
|
|
"maximum_seconds": 4.224220871998114,
|
|
"errors": 0,
|
|
"nonready_responses": 6,
|
|
"over_budget_responses": 0,
|
|
"p95_limit_seconds": 2.0,
|
|
"passed": false
|
|
},
|
|
"concurrent": {
|
|
"requests": 120,
|
|
"p50_seconds": 3.3592287749925163,
|
|
"p95_seconds": 6.08325262699509,
|
|
"maximum_seconds": 6.460270354000386,
|
|
"errors": 0,
|
|
"nonready_responses": 0,
|
|
"over_budget_responses": 0,
|
|
"p95_limit_seconds": 5.0,
|
|
"passed": false
|
|
},
|
|
"clients": 6,
|
|
"waves": 20,
|
|
"warmup_requests_excluded": 12,
|
|
"tokenizer": "o200k_base",
|
|
"passed": false
|
|
},
|
|
"limitations": [
|
|
"Twelve development tasks, no held-out evaluation or statistical significance claim.",
|
|
"Selected source hashes validated; complete repositories were not frozen and Lunatic changed during measurement.",
|
|
"First-page file recall does not establish that source bytes were read or understood.",
|
|
"Tool output tokens are observed; model input, reasoning and billed tokens are unknown.",
|
|
"CUDA retrieval and latency use the measured commit; subsequent recovery-only fixes require separate regression checks."
|
|
],
|
|
"recovery_fix_commit": "8ffa665",
|
|
"raw_artifact_archive": "/home/josh/.cache/trufflepig/evaluation/gpu-search",
|
|
"verification": {
|
|
"cuda_library_checkpoint_passed": 289,
|
|
"final_preparation_cuda_tests_passed": 15,
|
|
"final_worker_semantic_tests_passed": 18,
|
|
"final_preparation_no_default_tests_passed": 15,
|
|
"python_tests_passed": 59,
|
|
"all_targets_cuda_check": "passed"
|
|
},
|
|
"rerank_measurement": {
|
|
"measured_tree": "uncommitted rerank change on 6c50a74",
|
|
"binary_sha256": "907b07c43e4c6e22702ad0e3d5e9ea4ec621f16d41346b6003f143a23551cd7c",
|
|
"reranker": "rozgo/bge-reranker-v2-m3@fbd57b17b4db111a9d16813bb08b4c804fac18e9",
|
|
"embedding_gpu": "NVIDIA GeForce RTX 4090",
|
|
"rerank_gpu": "NVIDIA RTX PRO 6000 Blackwell Workstation Edition",
|
|
"note": "All three arms were re-measured in one session on the current working trees, so the same-day baselines below are the comparison for cuda_rerank; the arms recorded above were measured at 3835a69 on earlier trees.",
|
|
"same_day_baselines": {
|
|
"newfilefirstlexical": {
|
|
"binary_sha256": "907b07c43e4c6e22702ad0e3d5e9ea4ec621f16d41346b6003f143a23551cd7c",
|
|
"summary": {
|
|
"tasks": 12,
|
|
"snapshot_invalid": 0,
|
|
"unavailable": 0,
|
|
"errors": 0,
|
|
"truncated_searches": 10,
|
|
"misses": 5,
|
|
"mean_file_recall_at_5": 0.2916666666666667,
|
|
"mean_file_recall_at_10": 0.5416666666666666,
|
|
"mean_latency_seconds": 0.3370037924220977,
|
|
"usage": {
|
|
"tool_calls": 85,
|
|
"stdout_bytes": 218053,
|
|
"stderr_bytes": 0,
|
|
"emitted_tokens": 68941,
|
|
"tokenizer": "o200k_base",
|
|
"latency_seconds": 28.645322355878307,
|
|
"token_cost_censored": false
|
|
}
|
|
}
|
|
},
|
|
"cuda": {
|
|
"binary_sha256": "907b07c43e4c6e22702ad0e3d5e9ea4ec621f16d41346b6003f143a23551cd7c",
|
|
"summary": {
|
|
"tasks": 12,
|
|
"snapshot_invalid": 0,
|
|
"unavailable": 0,
|
|
"errors": 0,
|
|
"truncated_searches": 11,
|
|
"misses": 8,
|
|
"mean_file_recall_at_5": 0.3333333333333333,
|
|
"mean_file_recall_at_10": 0.3333333333333333,
|
|
"mean_latency_seconds": 0.41867398462051003,
|
|
"usage": {
|
|
"tool_calls": 82,
|
|
"stdout_bytes": 214534,
|
|
"stderr_bytes": 0,
|
|
"emitted_tokens": 67192,
|
|
"tokenizer": "o200k_base",
|
|
"latency_seconds": 34.33126673888182,
|
|
"token_cost_censored": false
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"per_task_file_recall_at_5": {
|
|
"lunatic-document-append-guard": {
|
|
"newfilefirstlexical": 1.0,
|
|
"cuda": 1.0,
|
|
"cuda_rerank": 1.0
|
|
},
|
|
"lunatic-legacy-airlock-fixture": {
|
|
"newfilefirstlexical": 0.0,
|
|
"cuda": 0.0,
|
|
"cuda_rerank": 0.0
|
|
},
|
|
"lunatic-cycle-link-validation": {
|
|
"newfilefirstlexical": 0.0,
|
|
"cuda": 0.0,
|
|
"cuda_rerank": 1.0
|
|
},
|
|
"lunatic-script-permission-interface": {
|
|
"newfilefirstlexical": 0.0,
|
|
"cuda": 1.0,
|
|
"cuda_rerank": 1.0
|
|
},
|
|
"tfs-airlock-permission-fixture": {
|
|
"newfilefirstlexical": 0.5,
|
|
"cuda": 0.0,
|
|
"cuda_rerank": 0.5
|
|
},
|
|
"tfs-airlock-cycle-state": {
|
|
"newfilefirstlexical": 1.0,
|
|
"cuda": 0.0,
|
|
"cuda_rerank": 0.0
|
|
},
|
|
"tfs-airlock-rebuild-fixture": {
|
|
"newfilefirstlexical": 0.0,
|
|
"cuda": 0.0,
|
|
"cuda_rerank": 1.0
|
|
},
|
|
"tfs-writing-commit": {
|
|
"newfilefirstlexical": 0.0,
|
|
"cuda": 0.0,
|
|
"cuda_rerank": 0.0
|
|
},
|
|
"tg-airlock-controller-cycle": {
|
|
"newfilefirstlexical": 1.0,
|
|
"cuda": 1.0,
|
|
"cuda_rerank": 1.0
|
|
},
|
|
"tg-airlock-pump-pressure": {
|
|
"newfilefirstlexical": 0.0,
|
|
"cuda": 1.0,
|
|
"cuda_rerank": 0.0
|
|
},
|
|
"tg-airlock-secure-open": {
|
|
"newfilefirstlexical": 0.0,
|
|
"cuda": 0.0,
|
|
"cuda_rerank": 1.0
|
|
},
|
|
"tg-airlock-wire-permissions": {
|
|
"newfilefirstlexical": 0.0,
|
|
"cuda": 0.0,
|
|
"cuda_rerank": 0.0
|
|
}
|
|
},
|
|
"limitations": [
|
|
"Twelve development tasks, no held-out evaluation or statistical significance claim.",
|
|
"Workspace searches rerank each of the three members sequentially; mean search latency rose from about 1.1 s (cuda) to about 2.7 s (cuda_rerank) at a 600-token page.",
|
|
"Lunatic and tales-from-space working trees have moved since the recorded parent revisions; selected label hashes still validated for all twelve tasks."
|
|
]
|
|
}
|
|
}
|