Mooncake/.github/workflows/pre-release.yaml

name: Pre-Release

# Dry-run of the release pipelines: build wheels like Release / Release Non-CUDA /
# Release CUDA 13, validate artifacts, but do not create a GitHub Release or publish to PyPI.
#
# Trigger by pushing a pre-release tag, for example:
#   git tag v1.0.0-rc1 && git push origin v1.0.0-rc1
on:
  push:
    tags:
      - 'v*-rc*'
      - 'v*-alpha*'
      - 'v*-beta*'
      - 'v*-pre*'

env:
  SCCACHE_GHA_ENABLED: "true"

jobs:
  build-cuda:
    name: Build (CUDA 12)
    runs-on: ubuntu-22.04
    permissions:
      contents: read
    strategy:
      matrix:
        python-version: ['3.10', '3.11', '3.12', '3.13']
    env:
      BUILD_WITH_EP: "1"
      TORCH_CUDA_ARCH_LIST: "8.0;9.0"
    steps:
      - name: Checkout source
        uses: actions/checkout@v4

      - name: Set version from tag
        run: echo "VERSION=${GITHUB_REF_NAME#v}" >> $GITHUB_ENV

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - name: Free up disk space
        run: |
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /opt/ghc
          sudo rm -rf /opt/hostedtoolcache/CodeQL
          sudo rm -rf /usr/local/lib/android
          df -h          

      - name: Install CUDA Toolkit
        uses: Jimver/cuda-toolkit@v0.2.24
        with:
          cuda: '12.8.1'
          linux-local-args: '["--toolkit"]'
          method: 'network'
          sub-packages: '["nvcc", "nvrtc-dev"]'
          non-cuda-sub-packages: '["libcusparse-dev", "libcublas-dev", "libcusolver-dev"]'

      - name: Run sccache-cache
        uses: mozilla-actions/sccache-action@v0.0.9

      - name: Configure sccache
        uses: actions/github-script@v7
        with:
          script: |
            core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || '');
            core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');            

      - name: Configure project
        run: |
          sudo apt update -y
          sudo bash -x dependencies.sh -y
          mkdir build
          cd build
          cmake .. -DBUILD_UNIT_TESTS=OFF -DUSE_HTTP=ON -DUSE_ETCD=ON -DUSE_CUDA=ON -DWITH_EP=ON -DEP_TORCH_VERSIONS="2.9.1;2.10.0;2.11.0;2.12.0" -DSTORE_USE_ETCD=ON -DENABLE_SCCACHE=ON -DCMAKE_BUILD_TYPE=Release          
        shell: bash

      - name: Build project
        run: |
          export LIBRARY_PATH=/usr/local/cuda/lib64/stubs:$LIBRARY_PATH
          cd build
          make -j
          sudo -E make install          
        shell: bash

      - name: Build nvlink_allocator.so
        run: |
          export PATH=/usr/local/nvidia/bin:/usr/local/nvidia/lib64:$PATH
          export LD_LIBRARY_PATH=/usr/local/cuda/lib64/stubs:$LD_LIBRARY_PATH
          export LIBRARY_PATH=/usr/local/cuda/lib64/stubs:$LIBRARY_PATH
          mkdir -p build/mooncake-transfer-engine/nvlink-allocator
          cd mooncake-transfer-engine/nvlink-allocator
          bash build.sh ../../build/mooncake-transfer-engine/nvlink-allocator/          
        shell: bash

      - name: Run sccache stat for check
        if: ${{ env.SCCACHE_PATH != '' }}
        shell: bash
        run: ${SCCACHE_PATH} --show-stats

      - name: Generate Python version tag
        id: generate_tag_release
        run: |
          echo "python_version_tag=$(echo ${{ matrix.python-version }} | tr -d '.')" >> $GITHUB_OUTPUT          
        shell: bash

      - name: Build Python wheel
        run: |
          export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib
          PYTHON_VERSION=${{ matrix.python-version }} OUTPUT_DIR=dist-py${{ steps.generate_tag_release.outputs.python_version_tag }} ./scripts/build_wheel.sh          
        env:
          VERSION: ${{ env.VERSION }}

      - name: Upload Python wheel artifact
        uses: actions/upload-artifact@v4
        with:
          name: pre-release-cuda-py${{ steps.generate_tag_release.outputs.python_version_tag }}
          path: mooncake-wheel/dist-py${{ steps.generate_tag_release.outputs.python_version_tag }}/*.whl

  build-non-cuda:
    name: Build (Non-CUDA)
    runs-on: ubuntu-22.04
    permissions:
      contents: read
    strategy:
      matrix:
        python-version: ['3.10', '3.11', '3.12', '3.13']
    env:
      BUILD_WITH_EP: "0"
      NON_CUDA_BUILD: "1"
    steps:
      - name: Checkout source
        uses: actions/checkout@v4

      - name: Set version from tag
        run: echo "VERSION=${GITHUB_REF_NAME#v}" >> $GITHUB_ENV

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - name: Free up disk space
        run: |
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /opt/ghc
          sudo rm -rf /opt/hostedtoolcache/CodeQL          

      - name: Run sccache-cache
        uses: mozilla-actions/sccache-action@v0.0.9

      - name: Configure sccache
        uses: actions/github-script@v7
        with:
          script: |
            core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || '');
            core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');            

      - name: Configure project
        run: |
          sudo apt update -y
          sudo bash -x dependencies.sh -y
          mkdir build
          cd build
          cmake .. -DUSE_HTTP=ON -DUSE_ETCD=ON -DUSE_CUDA=OFF -DWITH_EP=OFF -DSTORE_USE_ETCD=ON -DENABLE_SCCACHE=ON -DCMAKE_BUILD_TYPE=Release          
        shell: bash

      - name: Build project
        run: |
          cd build
          make -j
          sudo make install          
        shell: bash

      - name: Run sccache stat for check
        if: ${{ env.SCCACHE_PATH != '' }}
        shell: bash
        run: ${SCCACHE_PATH} --show-stats

      - name: Generate Python version tag
        id: generate_tag_release
        run: |
          echo "python_version_tag=$(echo ${{ matrix.python-version }} | tr -d '.')" >> $GITHUB_OUTPUT          
        shell: bash

      - name: Build Python wheel
        run: |
          export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib
          PYTHON_VERSION=${{ matrix.python-version }} OUTPUT_DIR=dist-py${{ steps.generate_tag_release.outputs.python_version_tag }} ./scripts/build_wheel.sh          
        env:
          VERSION: ${{ env.VERSION }}

      - name: Upload Python wheel artifact
        uses: actions/upload-artifact@v4
        with:
          name: pre-release-non-cuda-py${{ steps.generate_tag_release.outputs.python_version_tag }}
          path: mooncake-wheel/dist-py${{ steps.generate_tag_release.outputs.python_version_tag }}/*.whl

  build-cuda13:
    name: Build (CUDA 13)
    runs-on: ubuntu-22.04
    permissions:
      contents: read
    strategy:
      matrix:
        python-version: ['3.10', '3.11', '3.12', '3.13']
    env:
      BUILD_WITH_EP: "1"
      CU13_BUILD: "1"
      TORCH_CUDA_ARCH_LIST: "8.0;9.0"
    steps:
      - name: Checkout source
        uses: actions/checkout@v4

      - name: Set version from tag
        run: echo "VERSION=${GITHUB_REF_NAME#v}" >> $GITHUB_ENV

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - name: Free up disk space
        run: |
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /opt/ghc
          sudo rm -rf /opt/hostedtoolcache/CodeQL
          sudo rm -rf /usr/local/lib/android
          df -h          

      - name: Install CUDA Toolkit 13
        uses: Jimver/cuda-toolkit@v0.2.29
        with:
          cuda: '13.0.2'
          linux-local-args: '["--toolkit"]'
          method: 'network'
          sub-packages: '["nvcc", "nvrtc-dev"]'
          non-cuda-sub-packages: '["libcusparse-dev", "libcublas-dev", "libcusolver-dev"]'

      - name: Run sccache-cache
        uses: mozilla-actions/sccache-action@v0.0.9

      - name: Configure sccache
        uses: actions/github-script@v7
        with:
          script: |
            core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || '');
            core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');            

      - name: Configure project
        run: |
          sudo apt update -y
          sudo bash -x dependencies.sh -y
          mkdir build
          cd build
          cmake .. -DBUILD_UNIT_TESTS=OFF -DUSE_HTTP=ON -DUSE_ETCD=ON -DUSE_CUDA=ON -DWITH_EP=ON -DEP_TORCH_VERSIONS="2.9.0;2.9.1;2.10.0;2.11.0" -DSTORE_USE_ETCD=ON -DENABLE_SCCACHE=ON -DCMAKE_BUILD_TYPE=Release          
        shell: bash

      - name: Build project
        run: |
          export LIBRARY_PATH=/usr/local/cuda/lib64/stubs:$LIBRARY_PATH
          cd build
          make -j
          sudo make install          
        shell: bash

      - name: Build nvlink_allocator.so
        run: |
          export PATH=/usr/local/nvidia/bin:/usr/local/nvidia/lib64:$PATH
          export LD_LIBRARY_PATH=/usr/local/cuda/lib64/stubs:$LD_LIBRARY_PATH
          export LIBRARY_PATH=/usr/local/cuda/lib64/stubs:$LIBRARY_PATH
          mkdir -p build/mooncake-transfer-engine/nvlink-allocator
          cd mooncake-transfer-engine/nvlink-allocator
          bash build.sh ../../build/mooncake-transfer-engine/nvlink-allocator/          
        shell: bash

      - name: Run sccache stat for check
        if: ${{ env.SCCACHE_PATH != '' }}
        shell: bash
        run: ${SCCACHE_PATH} --show-stats

      - name: Generate Python version tag
        id: generate_tag_release
        run: |
          echo "python_version_tag=$(echo ${{ matrix.python-version }} | tr -d '.')" >> $GITHUB_OUTPUT          
        shell: bash

      - name: Build Python wheel
        run: |
          export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib
          PYTHON_VERSION=${{ matrix.python-version }} OUTPUT_DIR=dist-py${{ steps.generate_tag_release.outputs.python_version_tag }} ./scripts/build_wheel.sh          
        env:
          VERSION: ${{ env.VERSION }}

      - name: Upload Python wheel artifact
        uses: actions/upload-artifact@v4
        with:
          name: pre-release-cuda13-py${{ steps.generate_tag_release.outputs.python_version_tag }}
          path: mooncake-wheel/dist-py${{ steps.generate_tag_release.outputs.python_version_tag }}/*.whl

  validate-release:
    name: Validate release artifacts
    needs: [build-cuda, build-non-cuda, build-cuda13]
    runs-on: ubuntu-22.04
    permissions:
      contents: read
    steps:
      - name: Checkout source
        uses: actions/checkout@v4

      - name: Download all pre-release wheel artifacts
        uses: actions/download-artifact@v4
        with:
          path: mooncake-wheel/dist-all
          pattern: pre-release-*

      - name: Prepare wheels for validation
        run: |
          mkdir -p mooncake-wheel/dist-release
          find mooncake-wheel/dist-all -name "*.whl" -exec cp {} mooncake-wheel/dist-release/ \;
          echo "Pre-release tag: ${GITHUB_REF_NAME}"
          echo "Collected wheels:"
          ls -la mooncake-wheel/dist-release/
          wheel_count=$(find mooncake-wheel/dist-release -name "*.whl" | wc -l)
          echo "wheel_count=${wheel_count}" >> "$GITHUB_ENV"
          if [ "${wheel_count}" -lt 12 ]; then
            echo "Expected at least 12 wheels (4 Python versions x 3 variants), found ${wheel_count}"
            exit 1
          fi          

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.12'

      - name: Validate wheels with twine
        run: |
          pip install twine
          twine check mooncake-wheel/dist-release/*.whl          

      - name: Upload validated wheels as workflow artifacts
        uses: actions/upload-artifact@v4
        with:
          name: pre-release-wheels-${{ github.ref_name }}
          path: mooncake-wheel/dist-release/*.whl
          retention-days: 14

      - name: Pre-release validation summary
        run: |
          echo "## Pre-Release validation passed" >> "$GITHUB_STEP_SUMMARY"
          echo "" >> "$GITHUB_STEP_SUMMARY"
          echo "- Tag: \`${GITHUB_REF_NAME}\`" >> "$GITHUB_STEP_SUMMARY"
          echo "- Wheels built: ${wheel_count}" >> "$GITHUB_STEP_SUMMARY"
          echo "- PyPI publish: skipped (pre-release dry run)" >> "$GITHUB_STEP_SUMMARY"
          echo "- GitHub Release upload: skipped (pre-release dry run)" >> "$GITHUB_STEP_SUMMARY"