diff --git a/recipes/gpt-oss-120b/trtllm/agg/deploy.yaml b/recipes/gpt-oss-120b/trtllm/agg/deploy.yaml index f5c12f1af..f14466030 100644 --- a/recipes/gpt-oss-120b/trtllm/agg/deploy.yaml +++ b/recipes/gpt-oss-120b/trtllm/agg/deploy.yaml @@ -54,7 +54,7 @@ spec: envFromSecret: hf-token-secret volumeMounts: - name: model-cache - mountPoint: /root/.cache/huggingface + mountPoint: /opt/models sharedMemory: size: 80Gi extraPodSpec: @@ -92,7 +92,9 @@ spec: - name: ENGINE_ARGS value: "/opt/dynamo/configs/config.yaml" - name: MODEL_PATH - value: "/root/.cache/huggingface/hub/models--openai--gpt-oss-120b/snapshots/b5c939de8f754692c1647ca79fbf85e8c1e70f8a" + value: "/opt/models/hub/models--openai--gpt-oss-120b/snapshots/b5c939de8f754692c1647ca79fbf85e8c1e70f8a" + - name: HF_HOME + value: /opt/models volumeMounts: - mountPath: /opt/dynamo/configs name: llm-config diff --git a/recipes/llama-3-70b/vllm/agg/deploy.yaml b/recipes/llama-3-70b/vllm/agg/deploy.yaml index f0d8a9394..2cca281b9 100644 --- a/recipes/llama-3-70b/vllm/agg/deploy.yaml +++ b/recipes/llama-3-70b/vllm/agg/deploy.yaml @@ -15,11 +15,14 @@ spec: dynamoNamespace: llama3-70b-agg volumeMounts: - name: model-cache - mountPoint: /root/.cache/huggingface + mountPoint: /opt/models extraPodSpec: mainContainer: image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag workingDir: /workspace/examples/backends/vllm + envs: + - name: HF_HOME + value: /opt/models replicas: 1 VllmPrefillWorker: componentType: worker @@ -27,7 +30,7 @@ spec: envFromSecret: hf-token-secret volumeMounts: - name: model-cache - mountPoint: /root/.cache/huggingface + mountPoint: /opt/models sharedMemory: size: 20Gi extraPodSpec: @@ -36,7 +39,9 @@ spec: - name: SERVED_MODEL_NAME value: "RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic" - name: MODEL_PATH - value: "/root/.cache/huggingface/hub/models--RedHatAI--Llama-3.3-70B-Instruct-FP8-dynamic/snapshots/ddb4128556dfcff99e0c41aee159ea6c3e655dcd" + value: "/opt/models/hub/models--RedHatAI--Llama-3.3-70B-Instruct-FP8-dynamic/snapshots/ddb4128556dfcff99e0c41aee159ea6c3e655dcd" + - name: HF_HOME + value: /opt/models args: - "python3 -m dynamo.vllm --model $MODEL_PATH --served-model-name $SERVED_MODEL_NAME --tensor-parallel-size 4 --data-parallel-size 1 --disable-log-requests --gpu-memory-utilization 0.90 --no-enable-prefix-caching --block-size 128" command: diff --git a/recipes/llama-3-70b/vllm/disagg-multi-node/deploy.yaml b/recipes/llama-3-70b/vllm/disagg-multi-node/deploy.yaml index 33177e79f..94acf7c84 100644 --- a/recipes/llama-3-70b/vllm/disagg-multi-node/deploy.yaml +++ b/recipes/llama-3-70b/vllm/disagg-multi-node/deploy.yaml @@ -15,11 +15,14 @@ spec: dynamoNamespace: llama3-70b-disagg-mn volumeMounts: - name: model-cache - mountPoint: /root/.cache/huggingface + mountPoint: /opt/models extraPodSpec: mainContainer: image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag workingDir: /workspace/examples/backends/vllm + envs: + - name: HF_HOME + value: /opt/models replicas: 1 VllmPrefillWorker: componentType: worker @@ -27,7 +30,7 @@ spec: envFromSecret: hf-token-secret volumeMounts: - name: model-cache - mountPoint: /root/.cache/huggingface + mountPoint: /opt/models sharedMemory: size: 80Gi extraPodSpec: @@ -36,7 +39,9 @@ spec: - name: SERVED_MODEL_NAME value: "RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic" - name: MODEL_PATH - value: "/root/.cache/huggingface/hub/models--RedHatAI--Llama-3.3-70B-Instruct-FP8-dynamic/snapshots/ddb4128556dfcff99e0c41aee159ea6c3e655dcd" + value: "/opt/models/hub/models--RedHatAI--Llama-3.3-70B-Instruct-FP8-dynamic/snapshots/ddb4128556dfcff99e0c41aee159ea6c3e655dcd" + - name: HF_HOME + value: /opt/models args: - "python3 -m dynamo.vllm --model $MODEL_PATH --served-model-name $SERVED_MODEL_NAME --tensor-parallel-size 8 --data-parallel-size 1 --disable-log-requests --is-prefill-worker --gpu-memory-utilization 0.95 --no-enable-prefix-caching --block-size 128" command: @@ -56,7 +61,7 @@ spec: envFromSecret: hf-token-secret volumeMounts: - name: model-cache - mountPoint: /root/.cache/huggingface + mountPoint: /opt/models sharedMemory: size: 80Gi extraPodSpec: @@ -65,7 +70,9 @@ spec: - name: SERVED_MODEL_NAME value: "RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic" - name: MODEL_PATH - value: "/root/.cache/huggingface/hub/models--RedHatAI--Llama-3.3-70B-Instruct-FP8-dynamic/snapshots/ddb4128556dfcff99e0c41aee159ea6c3e655dcd" + value: "/opt/models/hub/models--RedHatAI--Llama-3.3-70B-Instruct-FP8-dynamic/snapshots/ddb4128556dfcff99e0c41aee159ea6c3e655dcd" + - name: HF_HOME + value: /opt/models args: - "python3 -m dynamo.vllm --model $MODEL_PATH --served-model-name $SERVED_MODEL_NAME --tensor-parallel-size 8 --data-parallel-size 1 --disable-log-requests --gpu-memory-utilization 0.90 --no-enable-prefix-caching --block-size 128" command: diff --git a/recipes/llama-3-70b/vllm/disagg-single-node/deploy.yaml b/recipes/llama-3-70b/vllm/disagg-single-node/deploy.yaml index 053adfa18..e67996f06 100644 --- a/recipes/llama-3-70b/vllm/disagg-single-node/deploy.yaml +++ b/recipes/llama-3-70b/vllm/disagg-single-node/deploy.yaml @@ -15,11 +15,14 @@ spec: dynamoNamespace: llama3-70b-disagg-sn volumeMounts: - name: model-cache - mountPoint: /root/.cache/huggingface + mountPoint: /opt/models extraPodSpec: mainContainer: image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag workingDir: /workspace/examples/backends/vllm + envs: + - name: HF_HOME + value: /opt/models replicas: 1 VllmPrefillWorker: componentType: worker @@ -27,7 +30,7 @@ spec: envFromSecret: hf-token-secret volumeMounts: - name: model-cache - mountPoint: /root/.cache/huggingface + mountPoint: /opt/models sharedMemory: size: 80Gi extraPodSpec: @@ -48,7 +51,9 @@ spec: - name: SERVED_MODEL_NAME value: "RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic" - name: MODEL_PATH - value: "/root/.cache/huggingface/hub/models--RedHatAI--Llama-3.3-70B-Instruct-FP8-dynamic/snapshots/ddb4128556dfcff99e0c41aee159ea6c3e655dcd" + value: "/opt/models/hub/models--RedHatAI--Llama-3.3-70B-Instruct-FP8-dynamic/snapshots/ddb4128556dfcff99e0c41aee159ea6c3e655dcd" + - name: HF_HOME + value: /opt/models args: - "python3 -m dynamo.vllm --model $MODEL_PATH --served-model-name $SERVED_MODEL_NAME --tensor-parallel-size 2 --data-parallel-size 1 --disable-log-requests --is-prefill-worker --gpu-memory-utilization 0.95 --no-enable-prefix-caching --block-size 128" command: @@ -68,7 +73,7 @@ spec: envFromSecret: hf-token-secret volumeMounts: - name: model-cache - mountPoint: /root/.cache/huggingface + mountPoint: /opt/models sharedMemory: size: 80Gi extraPodSpec: @@ -89,7 +94,9 @@ spec: - name: SERVED_MODEL_NAME value: "RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic" - name: MODEL_PATH - value: "/root/.cache/huggingface/hub/models--RedHatAI--Llama-3.3-70B-Instruct-FP8-dynamic/snapshots/ddb4128556dfcff99e0c41aee159ea6c3e655dcd" + value: "/opt/models/hub/models--RedHatAI--Llama-3.3-70B-Instruct-FP8-dynamic/snapshots/ddb4128556dfcff99e0c41aee159ea6c3e655dcd" + - name: HF_HOME + value: /opt/models args: - "python3 -m dynamo.vllm --model $MODEL_PATH --served-model-name $SERVED_MODEL_NAME --tensor-parallel-size 4 --data-parallel-size 1 --disable-log-requests --gpu-memory-utilization 0.90 --no-enable-prefix-caching --block-size 128" command: diff --git a/recipes/qwen3-32b-fp8/trtllm/agg/deploy.yaml b/recipes/qwen3-32b-fp8/trtllm/agg/deploy.yaml index 440091907..6dbca3a52 100644 --- a/recipes/qwen3-32b-fp8/trtllm/agg/deploy.yaml +++ b/recipes/qwen3-32b-fp8/trtllm/agg/deploy.yaml @@ -70,7 +70,7 @@ spec: envFromSecret: hf-token-secret volumeMounts: - name: model-cache - mountPoint: /root/.cache/huggingface + mountPoint: /opt/models sharedMemory: size: 80Gi extraPodSpec: @@ -106,6 +106,8 @@ spec: value: "/opt/dynamo/configs/config.yaml" - name: MODEL_PATH value: "Qwen/Qwen3-32B-FP8" + - name: HF_HOME + value: "/opt/models" volumeMounts: - mountPath: /opt/dynamo/configs name: llm-config diff --git a/recipes/qwen3-32b-fp8/trtllm/disagg/deploy.yaml b/recipes/qwen3-32b-fp8/trtllm/disagg/deploy.yaml index ef7e63626..6c43df5e7 100644 --- a/recipes/qwen3-32b-fp8/trtllm/disagg/deploy.yaml +++ b/recipes/qwen3-32b-fp8/trtllm/disagg/deploy.yaml @@ -228,7 +228,7 @@ spec: envFromSecret: hf-token-secret volumeMounts: - name: model-cache - mountPoint: /root/.cache/huggingface + mountPoint: /opt/models sharedMemory: size: 80Gi extraPodSpec: @@ -265,6 +265,8 @@ spec: value: "/opt/dynamo/configs/config-prefill.yaml" - name: MODEL_PATH value: "Qwen/Qwen3-32B-FP8" + - name: HF_HOME + value: "/opt/models" volumeMounts: - mountPath: /opt/dynamo/configs name: llm-config-prefill @@ -287,7 +289,7 @@ spec: envFromSecret: hf-token-secret volumeMounts: - name: model-cache - mountPoint: /root/.cache/huggingface + mountPoint: /opt/models sharedMemory: size: 80Gi extraPodSpec: @@ -324,6 +326,8 @@ spec: value: "/opt/dynamo/configs/config-decode.yaml" - name: MODEL_PATH value: "Qwen/Qwen3-32B-FP8" + - name: HF_HOME + value: "/opt/models" volumeMounts: - mountPath: /opt/dynamo/configs name: llm-config-decode