dynamo/.github/workflows/container-validation-dynamo.yml

# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

name: NVIDIA Dynamo Github Validation

on:
  push:
    branches:
    - main
    - release/*.*.*
    - "pull-request/[0-9]+"

concurrency:
    # The group name is a ternary operation. If the ref_name is 'main',
    # then the group name uses the run_id to ensure a unique group for
    # 'main' pushes. Otherwise, the group name is the ref_name, so that
    # workflows on the same PR/branch have the same group name for cancelling.
    group: dynamo-build-test-${{ github.ref_name == 'main' && github.run_id || github.ref_name }}
    cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}

env:
  CUDA_VERSION: '12.9'
  BUILDER_NAME: b-${{ github.run_id }}-${{ github.run_attempt }}

jobs:
  changed-files:
    runs-on: ubuntu-latest
    outputs:
      core: ${{ github.ref_name == 'main' || startsWith(github.ref_name, 'release/') || steps.changes.outputs.core == 'true' }}
      planner: ${{ github.ref_name == 'main' || startsWith(github.ref_name, 'release/') || steps.changes.outputs.planner == 'true' }}
      frontend: ${{ github.ref_name == 'main' || startsWith(github.ref_name, 'release/') || steps.changes.outputs.frontend == 'true' }}
      builder_name: ${{ steps.export-builder-name.outputs.builder_name }}
    steps:
      - name: Checkout code
        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955  # v4.3.0
        with:
          fetch-depth: 0
      - name: Check for changes
        id: changes
        uses: ./.github/actions/changed-files
        with:
          gh_token: ${{ github.token }}
      - name: Export builder name
        id: export-builder-name
        run: |
          echo "builder_name=${{ env.BUILDER_NAME }}" >> $GITHUB_OUTPUT          

  dynamo-status-check:
    runs-on: ubuntu-latest
    needs: [changed-files, build, rust-checks, test-parallel, test-sequential]
    if: always()
    steps:
      - name: "Check all dependent jobs"
        run: |
          echo '${{ toJson(needs) }}' | jq -e 'to_entries | map(.value.result) | all(. as $result | ["success", "skipped"] | any($result == .))'          

  build:
    needs: changed-files
    if: needs.changed-files.outputs.core == 'true' || needs.changed-files.outputs.planner == 'true' || needs.changed-files.outputs.frontend == 'true'
    runs-on: prod-builder-v3
    name: Build
    timeout-minutes: 60
    outputs:
      # Only pass the non-secret tag suffix between jobs (GitHub blanks outputs containing secrets)
      runtime_tag_suffix: ${{ steps.define_image_tag.outputs.runtime_tag_suffix }}
      test_tag_suffix: ${{ steps.define_image_tag.outputs.test_tag_suffix }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955  # v4.3.0
        with:
          lfs: true
      - name: Initialize Dynamo Builder
        uses: ./.github/actions/init-dynamo-builder
        with:
          builder_name: ${{ needs.changed-files.outputs.builder_name }}
          flavor: general
          arch: amd64
      - name: Docker Login
        if: github.event.pull_request.head.repo.full_name == github.repository || github.event_name == 'push'
        uses: ./.github/actions/docker-login
        with:
          ngc_ci_access_token: ${{ secrets.NGC_CI_ACCESS_TOKEN }}
          aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }}
          aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }}
          azure_acr_hostname: ${{ secrets.AZURE_ACR_HOSTNAME }}
          azure_acr_user: ${{ secrets.AZURE_ACR_USER }}
          azure_acr_password: ${{ secrets.AZURE_ACR_PASSWORD }}
      - name: Define Image Tag
        id: define_image_tag
        env:
          ECR_HOSTNAME: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com
        run: |
          CUDA_MAJOR=${CUDA_VERSION%%.*}
          RUNTIME_TAG_SUFFIX="${{ github.sha }}-dynamo-runtime-cuda${CUDA_MAJOR}-amd64"
          TEST_TAG_SUFFIX="${{ github.sha }}-dynamo-test-cuda${CUDA_MAJOR}-amd64"
          echo "runtime_tag_suffix=${RUNTIME_TAG_SUFFIX}" >> $GITHUB_OUTPUT
          echo "test_tag_suffix=${TEST_TAG_SUFFIX}" >> $GITHUB_OUTPUT
          echo "runtime_remote_tag=${ECR_HOSTNAME}/ai-dynamo/dynamo:${RUNTIME_TAG_SUFFIX}" >> $GITHUB_OUTPUT
          echo "test_remote_tag=${ECR_HOSTNAME}/ai-dynamo/dynamo:${TEST_TAG_SUFFIX}" >> $GITHUB_OUTPUT          
      - name: Generate Dockerfile
        shell: bash
        run: |
          echo "Generating Dockerfile for target: runtime and framework: dynamo"
          python ./container/render.py \
              --target=runtime \
              --framework=dynamo \
              --platform=amd64 \
              --cuda-version=${{ env.CUDA_VERSION }} \
              --show-result \
              --output-short-filename          
      - name: Build and Push Runtime Image
        uses: ./.github/actions/docker-remote-build
        with:
          image_tag: ${{ steps.define_image_tag.outputs.runtime_remote_tag }}
          framework: dynamo
          target: runtime
          platform: amd64
          cuda_version: ${{ env.CUDA_VERSION }}
          ci_token: ${{ secrets.CI_TOKEN }}
          aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }}
          aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }}
          sccache_s3_bucket: ${{ secrets.SCCACHE_S3_BUCKET }}
          aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          push_image: 'true'
      - name: Build and Push Test Image
        env:
          ECR_HOSTNAME: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com
        shell: bash
        run: |
          CACHE_TAG="test-dynamo-cuda${CUDA_VERSION%%.*}-amd64-cache"
          CACHE_ARGS="--cache-from type=registry,ref=${ECR_HOSTNAME}/ai-dynamo/dynamo:main-${CACHE_TAG}"
          CACHE_ARGS+=" --cache-from type=registry,ref=${ECR_HOSTNAME}/ai-dynamo/dynamo:release-${CACHE_TAG}"
          if [[ "$GITHUB_REF_NAME" == "main" ]]; then
            CACHE_ARGS+=" --cache-to type=registry,ref=${ECR_HOSTNAME}/ai-dynamo/dynamo:main-${CACHE_TAG},mode=max"
          elif [[ "$GITHUB_REF_NAME" =~ ^release ]]; then
            CACHE_ARGS+=" --cache-to type=registry,ref=${ECR_HOSTNAME}/ai-dynamo/dynamo:release-${CACHE_TAG},mode=max"
          fi

          docker buildx build \
            --progress=plain \
            --push \
            --platform linux/amd64 \
            -f container/Dockerfile.test \
            --build-arg BASE_IMAGE=${{ steps.define_image_tag.outputs.runtime_remote_tag }} \
            ${CACHE_ARGS} \
            -t ${{ steps.define_image_tag.outputs.test_remote_tag }} .          

  rust-checks:
    needs: [changed-files, build]
    if: needs.changed-files.outputs.core == 'true'
    runs-on: prod-tester-amd-gpu-v1
    name: Rust Checks
    timeout-minutes: 30
    env:
      CONTAINER_ID: test_${{ github.run_id }}_${{ github.run_attempt }}_rust_dynamo
      IMAGE_TAG: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com/ai-dynamo/dynamo:${{ needs.build.outputs.runtime_tag_suffix }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955  # v4.3.0
      - name: Docker Login
        uses: ./.github/actions/docker-login
        with:
          aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }}
          aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }}
      - name: Pull runtime image
        run: |
          source ./.github/scripts/retry_docker.sh
          retry_pull ${{ env.IMAGE_TAG }}          
      - name: Run Rust checks (block-manager + media-ffmpeg + integration tests)
        run: |
          docker run --rm --runtime=nvidia --gpus all --user root -w /workspace/lib/llm \
            --name ${{ env.CONTAINER_ID }}_rust_checks \
            -e SCCACHE_BUCKET=${{ secrets.SCCACHE_S3_BUCKET }} \
            -e SCCACHE_REGION=${{ secrets.AWS_DEFAULT_REGION }} \
            -e AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \
            -e AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
            ${{ env.IMAGE_TAG }} \
            bash -ec 'ARCH_ALT=x86_64 /workspace/container/use-sccache.sh install && \
                      eval $(/workspace/container/use-sccache.sh setup-env) && \
                      rustup component add rustfmt clippy && \
                      cargo fmt -- --check && \
                      cargo clippy --features block-manager,media-ffmpeg,testing-nixl,integration --no-deps --all-targets -- -D warnings && \
                      cargo test --locked --all-targets --features=block-manager,media-ffmpeg,testing-nixl,integration -- --nocapture && \
                      cargo clippy -p kvbm-physical --no-deps --all-targets -- -D warnings && \
                      cargo test --locked -p kvbm-physical --features testing-kvbm -- --nocapture --test-threads=4 && \
                      /workspace/container/use-sccache.sh show-stats "Rust Checks"'          

  test-parallel:
    needs: [changed-files, build]
    if: needs.changed-files.outputs.core == 'true' || needs.changed-files.outputs.planner == 'true' || needs.changed-files.outputs.frontend == 'true'
    runs-on: prod-builder-amd-v1
    name: Pytest (parallel)
    timeout-minutes: 30
    env:
      IMAGE_TAG: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com/ai-dynamo/dynamo:${{ needs.build.outputs.test_tag_suffix }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955  # v4.3.0
      - name: Docker Login
        uses: ./.github/actions/docker-login
        with:
          aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }}
          aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }}
      - name: Pull test image
        run: |
          source ./.github/scripts/retry_docker.sh
          retry_pull ${{ env.IMAGE_TAG }}          
      - name: Run pytest (parallel tests with xdist)
        uses: ./.github/actions/pytest
        with:
          image_tag: ${{ env.IMAGE_TAG }}
          pytest_marks: "pre_merge and parallel and not (vllm or sglang or trtllm) and (gpu_0)"
          framework: dynamo
          test_type: "pre_merge_parallel"
          platform_arch: amd64
          enable_mypy: 'true'
          hf_token: ${{ secrets.HF_TOKEN }}
          parallel_mode: '4'
          dind_as_sidecar: 'false'

  test-sequential:
    needs: [changed-files, build]
    if: needs.changed-files.outputs.core == 'true' || needs.changed-files.outputs.planner == 'true' || needs.changed-files.outputs.frontend == 'true'
    runs-on: prod-builder-amd-v1
    name: Pytest (sequential)
    timeout-minutes: 30
    env:
      IMAGE_TAG: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com/ai-dynamo/dynamo:${{ needs.build.outputs.test_tag_suffix }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955  # v4.3.0
      - name: Docker Login
        uses: ./.github/actions/docker-login
        with:
          aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }}
          aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }}
      - name: Pull test image
        run: |
          source ./.github/scripts/retry_docker.sh
          retry_pull ${{ env.IMAGE_TAG }}          
      - name: Run pytest (sequential tests)
        uses: ./.github/actions/pytest
        with:
          image_tag: ${{ env.IMAGE_TAG }}
          pytest_marks: "pre_merge and not parallel and not (vllm or sglang or trtllm) and (gpu_0)"
          framework: dynamo
          test_type: "pre_merge_sequential"
          platform_arch: amd64
          enable_mypy: 'false'
          hf_token: ${{ secrets.HF_TOKEN }}
          parallel_mode: 'none'
          dind_as_sidecar: 'false'