138 lines
3.6 KiB
JSON
138 lines
3.6 KiB
JSON
{
|
|
"config": {
|
|
"bridge": "http://127.0.0.1:52052",
|
|
"llamas": "http://127.0.0.1:52072,http://127.0.0.1:52073",
|
|
"model_path": "/data2/wangzhe/gitlink/mooncake1/.omb-state/models/qwen2.5-coder-7b-instruct-q4_k_m.gguf",
|
|
"agents": 6,
|
|
"ctx_tokens": 30000,
|
|
"share_mode": "shared_prefix",
|
|
"n_predict": 8,
|
|
"block_size": 256,
|
|
"kv_type": "q8_0",
|
|
"namespace": "canon2",
|
|
"concurrency": 2,
|
|
"slots_per_server": 4,
|
|
"replica_num": 1,
|
|
"out": "/data2/wangzhe/gitlink/mooncake1/.omb-state/run/demo_result.json"
|
|
},
|
|
"baseline": {
|
|
"label": "cache_off",
|
|
"agents": 6,
|
|
"wall_s": 23.111,
|
|
"ttft_ms_mean": 5684.5,
|
|
"ttft_ms_median": 5685.4,
|
|
"ttft_ms_p90": 5687.4,
|
|
"prompt_tokens_recomputed": 173653,
|
|
"prompt_tokens_total": 173653,
|
|
"recompute_ratio": 1.0,
|
|
"hit_agents": 0,
|
|
"throughput_agents_per_s": 0.26
|
|
},
|
|
"cached": {
|
|
"label": "cache_on",
|
|
"agents": 6,
|
|
"wall_s": 20.186,
|
|
"ttft_ms_mean": 3966.9,
|
|
"ttft_ms_median": 3653.9,
|
|
"ttft_ms_p90": 3934.2,
|
|
"prompt_tokens_recomputed": 28998,
|
|
"prompt_tokens_total": 173653,
|
|
"recompute_ratio": 0.167,
|
|
"hit_agents": 5,
|
|
"throughput_agents_per_s": 0.297
|
|
},
|
|
"ttft_reduction_pct": 30.2,
|
|
"throughput_speedup_x": 1.14,
|
|
"per_agent_cached": [
|
|
{
|
|
"idx": 0,
|
|
"llama": "http://127.0.0.1:52072",
|
|
"tokens": 28942,
|
|
"hit": false,
|
|
"decision": "miss",
|
|
"restored": false,
|
|
"store_get_ms": 0,
|
|
"bytes": 0,
|
|
"prepare_ms": 68.9809100003913,
|
|
"prompt_n": 28942,
|
|
"prompt_ms": 5587.763,
|
|
"comp_wall_ms": 5798.791166977026,
|
|
"ttft_ms": 5656.743910000391
|
|
},
|
|
{
|
|
"idx": 1,
|
|
"llama": "http://127.0.0.1:52073",
|
|
"tokens": 28942,
|
|
"hit": true,
|
|
"decision": "restore",
|
|
"restored": true,
|
|
"store_get_ms": 1988.8248469796963,
|
|
"bytes": 882366232,
|
|
"prepare_ms": 3607.2255889885128,
|
|
"prompt_n": 11,
|
|
"prompt_ms": 16.065,
|
|
"comp_wall_ms": 382.30138598009944,
|
|
"ttft_ms": 3623.290588988513
|
|
},
|
|
{
|
|
"idx": 2,
|
|
"llama": "http://127.0.0.1:52072",
|
|
"tokens": 28944,
|
|
"hit": true,
|
|
"decision": "restore",
|
|
"restored": true,
|
|
"store_get_ms": 1912.454207020346,
|
|
"bytes": 882366232,
|
|
"prepare_ms": 3500.9051900124177,
|
|
"prompt_n": 13,
|
|
"prompt_ms": 16.111,
|
|
"comp_wall_ms": 306.38355697738007,
|
|
"ttft_ms": 3517.0161900124176
|
|
},
|
|
{
|
|
"idx": 3,
|
|
"llama": "http://127.0.0.1:52073",
|
|
"tokens": 28940,
|
|
"hit": true,
|
|
"decision": "restore",
|
|
"restored": true,
|
|
"store_get_ms": 1712.3380179982632,
|
|
"bytes": 882366232,
|
|
"prepare_ms": 3297.939590003807,
|
|
"prompt_n": 9,
|
|
"prompt_ms": 87.499,
|
|
"comp_wall_ms": 220.71896900888532,
|
|
"ttft_ms": 3385.438590003807
|
|
},
|
|
{
|
|
"idx": 4,
|
|
"llama": "http://127.0.0.1:52072",
|
|
"tokens": 28942,
|
|
"hit": true,
|
|
"decision": "restore",
|
|
"restored": true,
|
|
"store_get_ms": 1611.3499579951167,
|
|
"bytes": 882366232,
|
|
"prepare_ms": 3589.8023130139336,
|
|
"prompt_n": 11,
|
|
"prompt_ms": 94.8,
|
|
"comp_wall_ms": 3008.9015809935518,
|
|
"ttft_ms": 3684.602313013934
|
|
},
|
|
{
|
|
"idx": 5,
|
|
"llama": "http://127.0.0.1:52073",
|
|
"tokens": 28943,
|
|
"hit": true,
|
|
"decision": "restore",
|
|
"restored": true,
|
|
"store_get_ms": 2498.859048995655,
|
|
"bytes": 882366232,
|
|
"prepare_ms": 3919.863100978546,
|
|
"prompt_n": 12,
|
|
"prompt_ms": 14.367,
|
|
"comp_wall_ms": 287.0978229911998,
|
|
"ttft_ms": 3934.230100978546
|
|
}
|
|
]
|
|
} |