29 lines
1.2 KiB
Bash
29 lines
1.2 KiB
Bash
#!/usr/bin/env bash
|
|
# Bring up the full stack: master + store-proxy + N llama servers + sidecar.
|
|
# Usage: stack_up.sh [num_servers=2] [tcp|rdma=rdma] [model_gguf] [ctx=40960]
|
|
set -euo pipefail
|
|
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
source "$HERE/env.sh"
|
|
|
|
N="${1:-2}"; PROTO="${2:-rdma}"
|
|
MODEL="${3:-$MODELS_DIR/qwen2.5-coder-1.5b-instruct-q4_k_m.gguf}"
|
|
CTX="${4:-40960}"
|
|
|
|
echo "==> master"; bash "$HERE/master_start.sh"
|
|
echo "==> store proxy"; OMB_STORE_PROTOCOL="$PROTO" bash "$HERE/proxy_start.sh" "$PROTO"
|
|
echo "==> $N llama servers";
|
|
for i in $(seq 0 $((N-1))); do
|
|
bash "$HERE/llama_start.sh" "a$i" auto "$((OMB_LLAMA_PORT + i))" "$MODEL" "$CTX" 1
|
|
done
|
|
echo "==> sidecar"; bash "$HERE/bridged_start.sh" mooncake
|
|
|
|
echo
|
|
echo "stack is up:"
|
|
echo " master 127.0.0.1:$OMB_MASTER_PORT"
|
|
echo " store proxy 127.0.0.1:$OMB_STORE_PROXY_PORT (proto=$PROTO)"
|
|
for i in $(seq 0 $((N-1))); do echo " llama a$i 127.0.0.1:$((OMB_LLAMA_PORT + i))"; done
|
|
echo " sidecar gRPC 127.0.0.1:$OMB_BRIDGE_GRPC_PORT"
|
|
echo " sidecar HTTP 127.0.0.1:$OMB_BRIDGE_HTTP_PORT (/metrics /stats /healthz)"
|
|
echo
|
|
curl -fsS "http://127.0.0.1:$OMB_BRIDGE_HTTP_PORT/healthz" && echo
|