mooncake-track
/
dynamo
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: llama3-70-b-agg recipe model download failure (#4290)

This commit is contained in:
Thomas Montfort 2025-11-13 11:27:05 -05:00 committed by GitHub
parent f817c59576
commit bf21cc034e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 45 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
recipes/gpt-oss-120b/trtllm/agg/deploy.yaml
View File

@ -54,7 +54,7 @@ spec:
envFromSecret: hf-token-secret envFromSecret: hf-token-secret
volumeMounts: volumeMounts:
- name: model-cache - name: model-cache
mountPoint: /root/.cache/huggingface mountPoint: /opt/models
sharedMemory: sharedMemory:
size: 80Gi size: 80Gi
extraPodSpec: extraPodSpec:
@ -92,7 +92,9 @@ spec:
- name: ENGINE_ARGS - name: ENGINE_ARGS
value: "/opt/dynamo/configs/config.yaml" value: "/opt/dynamo/configs/config.yaml"
- name: MODEL_PATH - name: MODEL_PATH
value: "/root/.cache/huggingface/hub/models--openai--gpt-oss-120b/snapshots/b5c939de8f754692c1647ca79fbf85e8c1e70f8a" value: "/opt/models/hub/models--openai--gpt-oss-120b/snapshots/b5c939de8f754692c1647ca79fbf85e8c1e70f8a"
- name: HF_HOME
value: /opt/models
volumeMounts: volumeMounts:
- mountPath: /opt/dynamo/configs - mountPath: /opt/dynamo/configs
name: llm-config name: llm-config

11
recipes/llama-3-70b/vllm/agg/deploy.yaml
View File

@ -15,11 +15,14 @@ spec:
dynamoNamespace: llama3-70b-agg dynamoNamespace: llama3-70b-agg
volumeMounts: volumeMounts:
- name: model-cache - name: model-cache
mountPoint: /root/.cache/huggingface mountPoint: /opt/models
extraPodSpec: extraPodSpec:
mainContainer: mainContainer:
image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
workingDir: /workspace/examples/backends/vllm workingDir: /workspace/examples/backends/vllm
envs:
- name: HF_HOME
value: /opt/models
replicas: 1 replicas: 1
VllmPrefillWorker: VllmPrefillWorker:
componentType: worker componentType: worker
@ -27,7 +30,7 @@ spec:
envFromSecret: hf-token-secret envFromSecret: hf-token-secret
volumeMounts: volumeMounts:
- name: model-cache - name: model-cache
mountPoint: /root/.cache/huggingface mountPoint: /opt/models
sharedMemory: sharedMemory:
size: 20Gi size: 20Gi
extraPodSpec: extraPodSpec:
@ -36,7 +39,9 @@ spec:
- name: SERVED_MODEL_NAME - name: SERVED_MODEL_NAME
value: "RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic" value: "RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic"
- name: MODEL_PATH - name: MODEL_PATH
value: "/root/.cache/huggingface/hub/models--RedHatAI--Llama-3.3-70B-Instruct-FP8-dynamic/snapshots/ddb4128556dfcff99e0c41aee159ea6c3e655dcd" value: "/opt/models/hub/models--RedHatAI--Llama-3.3-70B-Instruct-FP8-dynamic/snapshots/ddb4128556dfcff99e0c41aee159ea6c3e655dcd"
- name: HF_HOME
value: /opt/models
args: args:
- "python3 -m dynamo.vllm --model $MODEL_PATH --served-model-name $SERVED_MODEL_NAME --tensor-parallel-size 4 --data-parallel-size 1 --disable-log-requests --gpu-memory-utilization 0.90 --no-enable-prefix-caching --block-size 128" - "python3 -m dynamo.vllm --model $MODEL_PATH --served-model-name $SERVED_MODEL_NAME --tensor-parallel-size 4 --data-parallel-size 1 --disable-log-requests --gpu-memory-utilization 0.90 --no-enable-prefix-caching --block-size 128"
command: command:

17
recipes/llama-3-70b/vllm/disagg-multi-node/deploy.yaml
View File

@ -15,11 +15,14 @@ spec:
dynamoNamespace: llama3-70b-disagg-mn dynamoNamespace: llama3-70b-disagg-mn
volumeMounts: volumeMounts:
- name: model-cache - name: model-cache
mountPoint: /root/.cache/huggingface mountPoint: /opt/models
extraPodSpec: extraPodSpec:
mainContainer: mainContainer:
image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
workingDir: /workspace/examples/backends/vllm workingDir: /workspace/examples/backends/vllm
envs:
- name: HF_HOME
value: /opt/models
replicas: 1 replicas: 1
VllmPrefillWorker: VllmPrefillWorker:
componentType: worker componentType: worker
@ -27,7 +30,7 @@ spec:
envFromSecret: hf-token-secret envFromSecret: hf-token-secret
volumeMounts: volumeMounts:
- name: model-cache - name: model-cache
mountPoint: /root/.cache/huggingface mountPoint: /opt/models
sharedMemory: sharedMemory:
size: 80Gi size: 80Gi
extraPodSpec: extraPodSpec:
@ -36,7 +39,9 @@ spec:
- name: SERVED_MODEL_NAME - name: SERVED_MODEL_NAME
value: "RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic" value: "RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic"
- name: MODEL_PATH - name: MODEL_PATH
value: "/root/.cache/huggingface/hub/models--RedHatAI--Llama-3.3-70B-Instruct-FP8-dynamic/snapshots/ddb4128556dfcff99e0c41aee159ea6c3e655dcd" value: "/opt/models/hub/models--RedHatAI--Llama-3.3-70B-Instruct-FP8-dynamic/snapshots/ddb4128556dfcff99e0c41aee159ea6c3e655dcd"
- name: HF_HOME
value: /opt/models
args: args:
- "python3 -m dynamo.vllm --model $MODEL_PATH --served-model-name $SERVED_MODEL_NAME --tensor-parallel-size 8 --data-parallel-size 1 --disable-log-requests --is-prefill-worker --gpu-memory-utilization 0.95 --no-enable-prefix-caching --block-size 128" - "python3 -m dynamo.vllm --model $MODEL_PATH --served-model-name $SERVED_MODEL_NAME --tensor-parallel-size 8 --data-parallel-size 1 --disable-log-requests --is-prefill-worker --gpu-memory-utilization 0.95 --no-enable-prefix-caching --block-size 128"
command: command:
@ -56,7 +61,7 @@ spec:
envFromSecret: hf-token-secret envFromSecret: hf-token-secret
volumeMounts: volumeMounts:
- name: model-cache - name: model-cache
mountPoint: /root/.cache/huggingface mountPoint: /opt/models
sharedMemory: sharedMemory:
size: 80Gi size: 80Gi
extraPodSpec: extraPodSpec:
@ -65,7 +70,9 @@ spec:
- name: SERVED_MODEL_NAME - name: SERVED_MODEL_NAME
value: "RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic" value: "RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic"
- name: MODEL_PATH - name: MODEL_PATH
value: "/root/.cache/huggingface/hub/models--RedHatAI--Llama-3.3-70B-Instruct-FP8-dynamic/snapshots/ddb4128556dfcff99e0c41aee159ea6c3e655dcd" value: "/opt/models/hub/models--RedHatAI--Llama-3.3-70B-Instruct-FP8-dynamic/snapshots/ddb4128556dfcff99e0c41aee159ea6c3e655dcd"
- name: HF_HOME
value: /opt/models
args: args:
- "python3 -m dynamo.vllm --model $MODEL_PATH --served-model-name $SERVED_MODEL_NAME --tensor-parallel-size 8 --data-parallel-size 1 --disable-log-requests --gpu-memory-utilization 0.90 --no-enable-prefix-caching --block-size 128" - "python3 -m dynamo.vllm --model $MODEL_PATH --served-model-name $SERVED_MODEL_NAME --tensor-parallel-size 8 --data-parallel-size 1 --disable-log-requests --gpu-memory-utilization 0.90 --no-enable-prefix-caching --block-size 128"
command: command:

17
recipes/llama-3-70b/vllm/disagg-single-node/deploy.yaml
View File

@ -15,11 +15,14 @@ spec:
dynamoNamespace: llama3-70b-disagg-sn dynamoNamespace: llama3-70b-disagg-sn
volumeMounts: volumeMounts:
- name: model-cache - name: model-cache
mountPoint: /root/.cache/huggingface mountPoint: /opt/models
extraPodSpec: extraPodSpec:
mainContainer: mainContainer:
image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
workingDir: /workspace/examples/backends/vllm workingDir: /workspace/examples/backends/vllm
envs:
- name: HF_HOME
value: /opt/models
replicas: 1 replicas: 1
VllmPrefillWorker: VllmPrefillWorker:
componentType: worker componentType: worker
@ -27,7 +30,7 @@ spec:
envFromSecret: hf-token-secret envFromSecret: hf-token-secret
volumeMounts: volumeMounts:
- name: model-cache - name: model-cache
mountPoint: /root/.cache/huggingface mountPoint: /opt/models
sharedMemory: sharedMemory:
size: 80Gi size: 80Gi
extraPodSpec: extraPodSpec:
@ -48,7 +51,9 @@ spec:
- name: SERVED_MODEL_NAME - name: SERVED_MODEL_NAME
value: "RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic" value: "RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic"
- name: MODEL_PATH - name: MODEL_PATH
value: "/root/.cache/huggingface/hub/models--RedHatAI--Llama-3.3-70B-Instruct-FP8-dynamic/snapshots/ddb4128556dfcff99e0c41aee159ea6c3e655dcd" value: "/opt/models/hub/models--RedHatAI--Llama-3.3-70B-Instruct-FP8-dynamic/snapshots/ddb4128556dfcff99e0c41aee159ea6c3e655dcd"
- name: HF_HOME
value: /opt/models
args: args:
- "python3 -m dynamo.vllm --model $MODEL_PATH --served-model-name $SERVED_MODEL_NAME --tensor-parallel-size 2 --data-parallel-size 1 --disable-log-requests --is-prefill-worker --gpu-memory-utilization 0.95 --no-enable-prefix-caching --block-size 128" - "python3 -m dynamo.vllm --model $MODEL_PATH --served-model-name $SERVED_MODEL_NAME --tensor-parallel-size 2 --data-parallel-size 1 --disable-log-requests --is-prefill-worker --gpu-memory-utilization 0.95 --no-enable-prefix-caching --block-size 128"
command: command:
@ -68,7 +73,7 @@ spec:
envFromSecret: hf-token-secret envFromSecret: hf-token-secret
volumeMounts: volumeMounts:
- name: model-cache - name: model-cache
mountPoint: /root/.cache/huggingface mountPoint: /opt/models
sharedMemory: sharedMemory:
size: 80Gi size: 80Gi
extraPodSpec: extraPodSpec:
@ -89,7 +94,9 @@ spec:
- name: SERVED_MODEL_NAME - name: SERVED_MODEL_NAME
value: "RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic" value: "RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic"
- name: MODEL_PATH - name: MODEL_PATH
value: "/root/.cache/huggingface/hub/models--RedHatAI--Llama-3.3-70B-Instruct-FP8-dynamic/snapshots/ddb4128556dfcff99e0c41aee159ea6c3e655dcd" value: "/opt/models/hub/models--RedHatAI--Llama-3.3-70B-Instruct-FP8-dynamic/snapshots/ddb4128556dfcff99e0c41aee159ea6c3e655dcd"
- name: HF_HOME
value: /opt/models
args: args:
- "python3 -m dynamo.vllm --model $MODEL_PATH --served-model-name $SERVED_MODEL_NAME --tensor-parallel-size 4 --data-parallel-size 1 --disable-log-requests --gpu-memory-utilization 0.90 --no-enable-prefix-caching --block-size 128" - "python3 -m dynamo.vllm --model $MODEL_PATH --served-model-name $SERVED_MODEL_NAME --tensor-parallel-size 4 --data-parallel-size 1 --disable-log-requests --gpu-memory-utilization 0.90 --no-enable-prefix-caching --block-size 128"
command: command:

4
recipes/qwen3-32b-fp8/trtllm/agg/deploy.yaml
View File

@ -70,7 +70,7 @@ spec:
envFromSecret: hf-token-secret envFromSecret: hf-token-secret
volumeMounts: volumeMounts:
- name: model-cache - name: model-cache
mountPoint: /root/.cache/huggingface mountPoint: /opt/models
sharedMemory: sharedMemory:
size: 80Gi size: 80Gi
extraPodSpec: extraPodSpec:
@ -106,6 +106,8 @@ spec:
value: "/opt/dynamo/configs/config.yaml" value: "/opt/dynamo/configs/config.yaml"
- name: MODEL_PATH - name: MODEL_PATH
value: "Qwen/Qwen3-32B-FP8" value: "Qwen/Qwen3-32B-FP8"
- name: HF_HOME
value: "/opt/models"
volumeMounts: volumeMounts:
- mountPath: /opt/dynamo/configs - mountPath: /opt/dynamo/configs
name: llm-config name: llm-config

8
recipes/qwen3-32b-fp8/trtllm/disagg/deploy.yaml
View File

@ -228,7 +228,7 @@ spec:
envFromSecret: hf-token-secret envFromSecret: hf-token-secret
volumeMounts: volumeMounts:
- name: model-cache - name: model-cache
mountPoint: /root/.cache/huggingface mountPoint: /opt/models
sharedMemory: sharedMemory:
size: 80Gi size: 80Gi
extraPodSpec: extraPodSpec:
@ -265,6 +265,8 @@ spec:
value: "/opt/dynamo/configs/config-prefill.yaml" value: "/opt/dynamo/configs/config-prefill.yaml"
- name: MODEL_PATH - name: MODEL_PATH
value: "Qwen/Qwen3-32B-FP8" value: "Qwen/Qwen3-32B-FP8"
- name: HF_HOME
value: "/opt/models"
volumeMounts: volumeMounts:
- mountPath: /opt/dynamo/configs - mountPath: /opt/dynamo/configs
name: llm-config-prefill name: llm-config-prefill
@ -287,7 +289,7 @@ spec:
envFromSecret: hf-token-secret envFromSecret: hf-token-secret
volumeMounts: volumeMounts:
- name: model-cache - name: model-cache
mountPoint: /root/.cache/huggingface mountPoint: /opt/models
sharedMemory: sharedMemory:
size: 80Gi size: 80Gi
extraPodSpec: extraPodSpec:
@ -324,6 +326,8 @@ spec:
value: "/opt/dynamo/configs/config-decode.yaml" value: "/opt/dynamo/configs/config-decode.yaml"
- name: MODEL_PATH - name: MODEL_PATH
value: "Qwen/Qwen3-32B-FP8" value: "Qwen/Qwen3-32B-FP8"
- name: HF_HOME
value: "/opt/models"
volumeMounts: volumeMounts:
- mountPath: /opt/dynamo/configs - mountPath: /opt/dynamo/configs
name: llm-config-decode name: llm-config-decode