83 lines
2.1 KiB
JSON
83 lines
2.1 KiB
JSON
{
|
|
"cells": [
|
|
{
|
|
"name": "A1_1p5b_8k_5ag",
|
|
"backend": "rdma",
|
|
"agents": 5,
|
|
"ttft_off": 883.4,
|
|
"ttft_on": 917.9,
|
|
"ttft_red_pct": -3.9,
|
|
"reuser_ttft_on": 947.5,
|
|
"thru_x": 1.0,
|
|
"recompute_off_pct": 100.0,
|
|
"recompute_on_pct": 40.1,
|
|
"hit_agents": 4
|
|
},
|
|
{
|
|
"name": "A2_7b_30k_6ag",
|
|
"backend": "rdma",
|
|
"agents": 6,
|
|
"ttft_off": 5868.0,
|
|
"ttft_on": 3897.6,
|
|
"ttft_red_pct": 33.6,
|
|
"reuser_ttft_on": 3520.5,
|
|
"thru_x": 1.01,
|
|
"recompute_off_pct": 100.0,
|
|
"recompute_on_pct": 16.7,
|
|
"hit_agents": 5
|
|
},
|
|
{
|
|
"name": "A3_7b_16k_8ag",
|
|
"backend": "rdma",
|
|
"agents": 8,
|
|
"ttft_off": 3184.2,
|
|
"ttft_on": 2120.2,
|
|
"ttft_red_pct": 33.4,
|
|
"reuser_ttft_on": 1981.9,
|
|
"thru_x": 1.25,
|
|
"recompute_off_pct": 100.0,
|
|
"recompute_on_pct": 12.6,
|
|
"hit_agents": 7
|
|
}
|
|
],
|
|
"raw": {
|
|
"cells": [
|
|
{
|
|
"name": "A1_1p5b_8k_5ag",
|
|
"llamas": "http://127.0.0.1:52070,http://127.0.0.1:52071",
|
|
"model_path": "qwen2.5-coder-1.5b-instruct-q4_k_m.gguf",
|
|
"kv_type": "f16",
|
|
"agents": 5,
|
|
"ctx_tokens": 8000,
|
|
"share_mode": "shared_prefix",
|
|
"slots_per_server": 4,
|
|
"concurrency": 2,
|
|
"store_backend": "rdma"
|
|
},
|
|
{
|
|
"name": "A2_7b_30k_6ag",
|
|
"llamas": "http://127.0.0.1:52072,http://127.0.0.1:52073",
|
|
"model_path": "qwen2.5-coder-7b-instruct-q4_k_m.gguf",
|
|
"kv_type": "q8_0",
|
|
"agents": 6,
|
|
"ctx_tokens": 30000,
|
|
"share_mode": "shared_prefix",
|
|
"slots_per_server": 4,
|
|
"concurrency": 2,
|
|
"store_backend": "rdma"
|
|
},
|
|
{
|
|
"name": "A3_7b_16k_8ag",
|
|
"llamas": "http://127.0.0.1:52072,http://127.0.0.1:52073",
|
|
"model_path": "qwen2.5-coder-7b-instruct-q4_k_m.gguf",
|
|
"kv_type": "q8_0",
|
|
"agents": 8,
|
|
"ctx_tokens": 16000,
|
|
"share_mode": "shared_prefix",
|
|
"slots_per_server": 4,
|
|
"concurrency": 2,
|
|
"store_backend": "rdma"
|
|
}
|
|
]
|
|
}
|
|
} |