134 lines
3.7 KiB
YAML
134 lines
3.7 KiB
YAML
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
apiVersion: nvidia.com/v1alpha1
|
|
kind: DynamoGraphDeployment
|
|
metadata:
|
|
name: trtllm-disagg-planner
|
|
spec:
|
|
pvcs:
|
|
- name: dynamo-pvc
|
|
create: false
|
|
services:
|
|
Frontend:
|
|
dynamoNamespace: trtllm-disagg-planner
|
|
componentType: frontend
|
|
replicas: 1
|
|
extraPodSpec:
|
|
mainContainer:
|
|
image: my-registry/trtllm-runtime:my-tag
|
|
workingDir: /workspace/examples/backends/trtllm
|
|
command:
|
|
- python3
|
|
args:
|
|
- -m
|
|
- dynamo.frontend
|
|
- --http-port
|
|
- "8000"
|
|
- --kv-cache-block-size
|
|
- "128"
|
|
- --router-mode
|
|
- kv
|
|
- --kv-overlap-score-weight
|
|
- "0.0"
|
|
- --router-temperature
|
|
- "0.0"
|
|
- --no-kv-events
|
|
Planner:
|
|
dynamoNamespace: trtllm-disagg-planner
|
|
envFromSecret: hf-token-secret
|
|
componentType: planner
|
|
replicas: 1
|
|
volumeMounts:
|
|
- name: dynamo-pvc # Must be pre-created before deployment and SLA profiler must have been run
|
|
mountPoint: /data
|
|
extraPodSpec:
|
|
mainContainer:
|
|
image: my-registry/trtllm-runtime:my-tag
|
|
workingDir: /workspace/components/src/dynamo/planner
|
|
ports:
|
|
- name: metrics
|
|
containerPort: 9085
|
|
command:
|
|
- python3
|
|
args:
|
|
- -m
|
|
- planner_sla
|
|
- --environment=kubernetes
|
|
- --backend=trtllm
|
|
- --adjustment-interval=60
|
|
- --profile-results-dir=/data/profiling_results
|
|
- --prometheus-port=9085
|
|
TRTLLMDecodeWorker:
|
|
dynamoNamespace: trtllm-disagg-planner
|
|
envFromSecret: hf-token-secret
|
|
componentType: worker
|
|
subComponentType: decode
|
|
replicas: 1
|
|
livenessProbe:
|
|
httpGet:
|
|
path: /live
|
|
port: 9090
|
|
periodSeconds: 5
|
|
timeoutSeconds: 30
|
|
failureThreshold: 1
|
|
readinessProbe:
|
|
httpGet:
|
|
path: /health
|
|
port: 9090
|
|
periodSeconds: 10
|
|
timeoutSeconds: 30
|
|
failureThreshold: 60
|
|
resources:
|
|
limits:
|
|
gpu: "1"
|
|
extraPodSpec:
|
|
terminationGracePeriodSeconds: 600
|
|
mainContainer:
|
|
image: my-registry/trtllm-runtime:my-tag
|
|
workingDir: /workspace/
|
|
command:
|
|
- python3
|
|
args:
|
|
- -m
|
|
- dynamo.trtllm
|
|
- --model-path
|
|
- Qwen/Qwen3-0.6B
|
|
- --served-model-name
|
|
- Qwen/Qwen3-0.6B
|
|
- --extra-engine-args
|
|
- ./recipes/qwen3/trtllm/decode.yaml
|
|
- --disaggregation-mode
|
|
- decode
|
|
- --disaggregation-strategy
|
|
- decode_first
|
|
TRTLLMPrefillWorker:
|
|
dynamoNamespace: trtllm-disagg-planner
|
|
envFromSecret: hf-token-secret
|
|
componentType: worker
|
|
subComponentType: prefill
|
|
replicas: 1
|
|
resources:
|
|
limits:
|
|
gpu: "1"
|
|
extraPodSpec:
|
|
terminationGracePeriodSeconds: 600
|
|
mainContainer:
|
|
image: my-registry/trtllm-runtime:my-tag
|
|
workingDir: /workspace/
|
|
command:
|
|
- python3
|
|
args:
|
|
- -m
|
|
- dynamo.trtllm
|
|
- --model-path
|
|
- Qwen/Qwen3-0.6B
|
|
- --served-model-name
|
|
- Qwen/Qwen3-0.6B
|
|
- --extra-engine-args
|
|
- ./recipes/qwen3/trtllm/prefill.yaml
|
|
- --disaggregation-mode
|
|
- prefill
|
|
- --disaggregation-strategy
|
|
- decode_first
|