839 lines
30 KiB
YAML
839 lines
30 KiB
YAML
name: 'Build & Test (Linux)'
|
|
|
|
on:
|
|
push:
|
|
branches: [ "main" ]
|
|
pull_request:
|
|
branches: [ "main" ]
|
|
types: [opened, synchronize, reopened, labeled]
|
|
workflow_dispatch: {}
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number || github.sha }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
build:
|
|
needs: [spell-check, clang-format, check-paths]
|
|
if: >-
|
|
(needs.check-paths.outputs.should-run-downstream == 'true' ||
|
|
github.event_name == 'workflow_dispatch') &&
|
|
(github.event_name == 'push' ||
|
|
github.event_name == 'workflow_dispatch' ||
|
|
github.event.action == 'opened' ||
|
|
contains(github.event.pull_request.labels.*.name, 'run-ci'))
|
|
runs-on: ubuntu-22.04
|
|
strategy:
|
|
matrix:
|
|
python-version: ['3.10', '3.12']
|
|
env:
|
|
CI: "true"
|
|
SCCACHE_GHA_ENABLED: "true"
|
|
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- name: Set up Python ${{ matrix.python-version }}
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: ${{ matrix.python-version }}
|
|
|
|
- name: Install Rust toolchain
|
|
uses: dtolnay/rust-toolchain@stable
|
|
|
|
- name: Install and start etcd
|
|
run: |
|
|
wget https://github.com/etcd-io/etcd/releases/download/v3.6.1/etcd-v3.6.1-linux-amd64.tar.gz
|
|
tar xzf etcd-v3.6.1-linux-amd64.tar.gz
|
|
sudo mv etcd-v3.6.1-linux-amd64/etcd* /usr/local/bin/
|
|
etcd --advertise-client-urls http://127.0.0.1:2379 --listen-client-urls http://127.0.0.1:2379 &
|
|
sleep 3 # Give etcd time to start
|
|
etcdctl --endpoints=http://127.0.0.1:2379 endpoint health
|
|
shell: bash
|
|
|
|
- name: Free up disk space
|
|
run: |
|
|
sudo rm -rf /usr/share/dotnet
|
|
sudo rm -rf /opt/ghc
|
|
sudo rm -rf /opt/hostedtoolcache/CodeQL
|
|
|
|
- name: Install CUDA Toolkit
|
|
uses: Jimver/cuda-toolkit@v0.2.24
|
|
with:
|
|
cuda: '12.8.1'
|
|
linux-local-args: '["--toolkit"]'
|
|
method: 'network'
|
|
sub-packages: '["nvcc"]'
|
|
|
|
- name: Install coverage tools and build utilities
|
|
run: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y lcov gcovr ninja-build
|
|
|
|
- name: Set up coverage compilation flags
|
|
run: |
|
|
echo "Setting up coverage compilation flags..."
|
|
echo "CXXFLAGS=--coverage" >> $GITHUB_ENV
|
|
echo "CFLAGS=--coverage" >> $GITHUB_ENV
|
|
echo "LDFLAGS=--coverage" >> $GITHUB_ENV
|
|
shell: bash
|
|
|
|
- name: Run sccache-cache
|
|
uses: mozilla-actions/sccache-action@v0.0.9
|
|
|
|
- name: Configure sccache
|
|
uses: actions/github-script@v7
|
|
with:
|
|
script: |
|
|
core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || '');
|
|
core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
|
|
|
|
- name: Run sccache stat for check
|
|
shell: bash
|
|
run: ${SCCACHE_PATH} --show-stats
|
|
|
|
- name: Configure project with coverage support
|
|
run: |
|
|
sudo apt update -y
|
|
sudo bash -x dependencies.sh -y
|
|
mkdir build
|
|
cd build
|
|
cmake -G Ninja .. -DUSE_HTTP=ON -DUSE_CXL=ON -DUSE_UB=ON -DUSE_ETCD=ON -DSTORE_USE_ETCD=ON -DENABLE_ASAN=ON -DENABLE_SCCACHE=ON -DCMAKE_BUILD_TYPE=Debug
|
|
shell: bash
|
|
|
|
- name: Build project
|
|
run: |
|
|
cd build
|
|
cmake --build .
|
|
sudo cmake --install .
|
|
shell: bash
|
|
|
|
- name: Build nvlink_allocator.so
|
|
run: |
|
|
mkdir -p build/mooncake-transfer-engine/nvlink-allocator
|
|
cd mooncake-transfer-engine/nvlink-allocator
|
|
export LIBRARY_PATH=/usr/local/cuda/lib64/stubs:$LIBRARY_PATH
|
|
bash build.sh ../../build/mooncake-transfer-engine/nvlink-allocator/
|
|
shell: bash
|
|
|
|
- name: Start Metadata Server
|
|
run: |
|
|
cd mooncake-transfer-engine/example/http-metadata-server-python
|
|
pip install aiohttp
|
|
python ./bootstrap_server.py &
|
|
shell: bash
|
|
|
|
- name: Run Mooncake Store Rust smoke test and benchmark
|
|
run: |
|
|
$GITHUB_WORKSPACE/build/mooncake-store/src/mooncake_master \
|
|
--eviction_high_watermark_ratio=0.95 \
|
|
--cluster_id=ci_rust_test_cluster \
|
|
--port 50051 &
|
|
MASTER_PID=$!
|
|
sleep 3
|
|
cd mooncake-store/rust
|
|
export LD_LIBRARY_PATH=$GITHUB_WORKSPACE/build/mooncake-asio:$GITHUB_WORKSPACE/build/mooncake-store/src:$GITHUB_WORKSPACE/build/mooncake-store/src/cachelib_memory_allocator:$GITHUB_WORKSPACE/build/mooncake-transfer-engine/src:$GITHUB_WORKSPACE/build/mooncake-transfer-engine/src/common/base:$GITHUB_WORKSPACE/build/mooncake-common/etcd:$LD_LIBRARY_PATH
|
|
export MOONCAKE_BUILD_DIR=$GITHUB_WORKSPACE/build
|
|
export MOONCAKE_STORE_LIB_DIR=$GITHUB_WORKSPACE/build/mooncake-store/src
|
|
export MOONCAKE_STORE_INCLUDE_DIR=$GITHUB_WORKSPACE/mooncake-store/include
|
|
export MC_METADATA_SERVER=http://127.0.0.1:8080/metadata
|
|
export MC_RUST_STORE_RUN_INTEGRATION=true
|
|
export MC_RUST_STORE_MASTER_ADDR=127.0.0.1:50051
|
|
export MC_RUST_STORE_LOCAL_HOSTNAME=127.0.0.1
|
|
export MC_RUST_STORE_PROTOCOL=tcp
|
|
export MC_RUST_STORE_DEVICE_NAME=
|
|
cargo test --test minimal_smoke -- --nocapture
|
|
MC_RUST_BENCH_ITERATIONS=4 \
|
|
MC_RUST_BENCH_VALUE_SIZE=4096 \
|
|
MC_RUST_BENCH_WARMUP=1 \
|
|
cargo run --release --example store_benchmark
|
|
kill $MASTER_PID 2>/dev/null || true
|
|
shell: bash
|
|
|
|
- name: Run Go store binding integration tests
|
|
run: |
|
|
$GITHUB_WORKSPACE/build/mooncake-store/src/mooncake_master \
|
|
--eviction_high_watermark_ratio=0.95 \
|
|
--cluster_id=ci_go_test_cluster \
|
|
--port 50051 &
|
|
MASTER_PID=$!
|
|
sleep 3
|
|
cd mooncake-store/go
|
|
export LD_LIBRARY_PATH=$GITHUB_WORKSPACE/build/mooncake-common:$GITHUB_WORKSPACE/build/mooncake-store/src:$GITHUB_WORKSPACE/build/mooncake-transfer-engine/src:$GITHUB_WORKSPACE/build/mooncake-transfer-engine/src/common/base:$GITHUB_WORKSPACE/build/mooncake-common/etcd
|
|
export CGO_ENABLED=1
|
|
export CGO_CFLAGS="-I$GITHUB_WORKSPACE/mooncake-store/include -I$GITHUB_WORKSPACE/mooncake-transfer-engine/include"
|
|
export CGO_LDFLAGS="-L$GITHUB_WORKSPACE/build/mooncake-store/src -L$GITHUB_WORKSPACE/build/mooncake-store/src/cachelib_memory_allocator -L$GITHUB_WORKSPACE/build/mooncake-transfer-engine/src -L$GITHUB_WORKSPACE/build/mooncake-transfer-engine/src/common/base -L$GITHUB_WORKSPACE/build/mooncake-common -L$GITHUB_WORKSPACE/build/mooncake-common/etcd -lmooncake_store -lcachelib_memory_allocator -ltransfer_engine -lbase -lasio -letcd_wrapper -lstdc++ -lnuma -lglog -lgflags -libverbs -ljsoncpp -lzstd -lcurl -luring -lasan -lm -lgcov"
|
|
# Link cudart if CUDA is available (needed for D2H staging in mooncake_store)
|
|
if [ -d /usr/local/cuda/lib64 ]; then export CGO_LDFLAGS="$CGO_LDFLAGS -L/usr/local/cuda/lib64 -lcudart"; fi
|
|
ASAN_OPTIONS=detect_leaks=0:verify_asan_link_order=0 MC_METADATA_SERVER=http://127.0.0.1:8080/metadata go test -v ./tests/...
|
|
kill $MASTER_PID 2>/dev/null || true
|
|
shell: bash
|
|
|
|
- name: Test (in build env) with coverage
|
|
run: |
|
|
cd build
|
|
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib
|
|
ldconfig -v || echo "always continue"
|
|
MC_METADATA_SERVER=http://127.0.0.1:8080/metadata DEFAULT_KV_LEASE_TTL=500 ctest -j --output-on-failure -E ub_transport_test
|
|
shell: bash
|
|
|
|
- name: Drain HTTP E2E test
|
|
if: matrix.python-version == '3.12'
|
|
run: |
|
|
cd build
|
|
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib
|
|
# Keep the sanitizer gate on the C++ integration test. The Python
|
|
# drain script is manual/nightly only because pybind + ASan teardown in
|
|
# a Python host process is not stable.
|
|
DEFAULT_KV_LEASE_TTL=500 ./mooncake-store/tests/task_integration_test --gtest_filter='TaskExecutorIntegrationTest.DrainJobCompleteFlow'
|
|
shell: bash
|
|
|
|
- name: Generate coverage report
|
|
id: coverage
|
|
run: |
|
|
cd build
|
|
echo "=== Starting coverage report generation ==="
|
|
echo "Current directory: $(pwd)"
|
|
|
|
echo "=== Looking for .gcda files ==="
|
|
find . -name "*.gcda" 2>/dev/null | head -10 || echo "No .gcda files found"
|
|
|
|
echo "=== Running lcov ==="
|
|
lcov --capture --directory . --output-file coverage.info 2>&1 || {
|
|
echo "WARNING: lcov failed to capture coverage data"
|
|
echo "Creating minimal lcov-compliant coverage file to allow CI to continue"
|
|
echo "TN:dummy" > coverage.filtered.info
|
|
echo "SF:/dev/null" >> coverage.filtered.info
|
|
echo "DA:0,0" >> coverage.filtered.info
|
|
echo "end_of_record" >> coverage.filtered.info
|
|
echo "coverage_failed=true" >> $GITHUB_OUTPUT
|
|
exit 0 # Exit successfully, do not block CI
|
|
}
|
|
|
|
echo "=== Processing coverage data ==="
|
|
lcov --remove coverage.info '/usr/*' '*/test/*' '*/third_party/*' --output-file coverage.filtered.info 2>&1 || true
|
|
|
|
echo "=== Generating HTML report ==="
|
|
genhtml coverage.filtered.info --output-directory coverage_report 2>&1 || echo "genhtml failed, continuing..."
|
|
|
|
echo "=== Coverage summary ==="
|
|
lcov --list coverage.filtered.info 2>&1 || echo "lcov list failed"
|
|
|
|
echo "=== Coverage report generation completed ==="
|
|
shell: bash
|
|
|
|
- name: Upload coverage to Codecov
|
|
uses: codecov/codecov-action@v4
|
|
with:
|
|
files: build/coverage.filtered.info
|
|
flags: unittests
|
|
name: code-coverage-report
|
|
token: ${{ secrets.CODECOV_TOKEN }}
|
|
fail_ci_if_error: false
|
|
continue-on-error: true
|
|
|
|
- name: Check coverage status
|
|
if: always()
|
|
run: |
|
|
if [ "${{ steps.coverage.outputs.coverage_failed }}" = "true" ]; then
|
|
echo "⚠️ Coverage collection failed but CI continued"
|
|
echo "::warning::Code coverage collection failed. Please check the build logs."
|
|
else
|
|
echo "✅ Coverage collected successfully"
|
|
fi
|
|
|
|
- name: Generate Python version tag
|
|
id: generate_tag_build
|
|
run: |
|
|
echo "python_version_tag=$(echo ${{ matrix.python-version }} | tr -d '.')" >> $GITHUB_OUTPUT
|
|
shell: bash
|
|
|
|
# In CI, build_wheel.sh removes build/ to free disk (CI=true); set FREE_BUILD_DIR=1 locally to enable.
|
|
- name: Build Python wheel
|
|
run: |
|
|
PYTHON_VERSION=${{ matrix.python-version }} OUTPUT_DIR=dist-py${{ steps.generate_tag_build.outputs.python_version_tag }} ./scripts/build_wheel.sh
|
|
shell: bash
|
|
|
|
- name: Upload wheel for ZMQ test job
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: wheel-build-py${{ steps.generate_tag_build.outputs.python_version_tag }}
|
|
path: mooncake-wheel/dist-py${{ steps.generate_tag_build.outputs.python_version_tag }}/*.whl
|
|
|
|
build-musa:
|
|
needs: [spell-check, clang-format, check-paths]
|
|
if: >-
|
|
(needs.check-paths.outputs.should-run-downstream == 'true' ||
|
|
github.event_name == 'workflow_dispatch') &&
|
|
(github.event_name == 'push' ||
|
|
github.event_name == 'workflow_dispatch' ||
|
|
github.event.action == 'opened' ||
|
|
contains(github.event.pull_request.labels.*.name, 'run-ci'))
|
|
runs-on: ubuntu-22.04
|
|
container: mthreads/musa:rc4.3.0-devel-ubuntu22.04-amd64
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- name: Mark repository as safe
|
|
run: git config --global --add safe.directory $GITHUB_WORKSPACE
|
|
shell: bash
|
|
|
|
- name: Configure project
|
|
run: |
|
|
apt update -y
|
|
apt install -y ninja-build
|
|
bash -x dependencies.sh -y
|
|
mkdir build
|
|
cd build
|
|
cmake -G Ninja .. -DUSE_MUSA=ON -DUSE_MNNVL=ON -DUSE_ETCD=ON -DSTORE_USE_ETCD=ON -DUSE_CXL=ON -DUSE_TCP=ON -DBUILD_UNIT_TESTS=OFF -DBUILD_EXAMPLES=OFF
|
|
shell: bash
|
|
|
|
- name: Build project
|
|
run: |
|
|
cd build
|
|
source ~/.bashrc
|
|
cmake --build .
|
|
cmake --install .
|
|
shell: bash
|
|
|
|
test-wheel-ubuntu:
|
|
needs: [spell-check, clang-format, build-flags]
|
|
if: >-
|
|
needs.build-flags.result == 'success' &&
|
|
(github.event_name == 'push' ||
|
|
github.event_name == 'workflow_dispatch' ||
|
|
github.event.action == 'opened' ||
|
|
contains(github.event.pull_request.labels.*.name, 'run-ci'))
|
|
strategy:
|
|
matrix:
|
|
ubuntu-version: [ubuntu-22.04, ubuntu-24.04]
|
|
python-version: ['3.10', '3.12']
|
|
runs-on: ${{ matrix.ubuntu-version }}
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- name: Set up Python ${{ matrix.python-version }}
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: ${{ matrix.python-version }}
|
|
|
|
- name: Generate Python version tag
|
|
id: generate_tag_test
|
|
run: |
|
|
echo "python_version_tag=$(echo ${{ matrix.python-version }} | tr -d '.')" >> $GITHUB_OUTPUT
|
|
shell: bash
|
|
|
|
- name: Download wheel artifact
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
name: mooncake-wheel-ubuntu-py${{ steps.generate_tag_test.outputs.python_version_tag }}
|
|
path: mooncake-wheel/dist
|
|
|
|
- name: Verify wheel file exists
|
|
run: |
|
|
ls -la mooncake-wheel/dist/
|
|
if [ ! -f mooncake-wheel/dist/*.whl ]; then
|
|
echo "ERROR: No wheel file found in mooncake-wheel/dist/"
|
|
exit 1
|
|
fi
|
|
shell: bash
|
|
|
|
- name: Free up disk space
|
|
run: |
|
|
sudo rm -rf /usr/share/dotnet
|
|
sudo rm -rf /opt/ghc
|
|
sudo rm -rf /opt/hostedtoolcache/CodeQL
|
|
sudo rm -rf /usr/local/lib/android
|
|
df -h
|
|
|
|
- name: Install CUDA Toolkit
|
|
uses: Jimver/cuda-toolkit@v0.2.24
|
|
with:
|
|
cuda: '12.8.1'
|
|
linux-local-args: '["--toolkit"]'
|
|
method: 'network'
|
|
|
|
- name: Run installation test script
|
|
run: |
|
|
bash scripts/test_installation.sh
|
|
shell: bash
|
|
|
|
- name: Start metadata server
|
|
run: |
|
|
source test_env/bin/activate
|
|
mooncake_http_metadata_server --port 8080 &
|
|
shell: bash
|
|
|
|
- name: Run tests with ssd
|
|
run: |
|
|
# Reserve port 50052 (mooncake_client RPC port) so the kernel never
|
|
# auto-allocates it as ephemeral source port for other outbound
|
|
# connections in the test suite. Without this, a random Python test
|
|
# connection can pick src_port=50052, leave a TIME_WAIT on
|
|
# <eth0_ip>:50052 for 60s, and block mooncake_client's bind to
|
|
# 0.0.0.0:50052 even with SO_REUSEADDR (Linux only relaxes
|
|
# TIME_WAIT+bind conflict for same-IP or loopback).
|
|
sudo sysctl -w net.ipv4.ip_local_reserved_ports=50052
|
|
source test_env/bin/activate
|
|
MC_STORE_MEMCPY=false TEST_SSD_OFFLOAD_IN_EVICT=true ./scripts/run_tests.sh
|
|
rm -rf /tmp/mooncake_test_ssd
|
|
deactivate
|
|
shell: bash
|
|
|
|
- name: Start Mooncake Master
|
|
run: |
|
|
source test_env/bin/activate
|
|
mkdir -p /tmp/mooncake_storage
|
|
mooncake_master \
|
|
--eviction_high_watermark_ratio=0.95 \
|
|
--cluster_id=ci_test_cluster \
|
|
--port 50051 &
|
|
sleep 3
|
|
shell: bash
|
|
|
|
- name: Run Python Tensor API Performance Test (CI check)
|
|
env:
|
|
MOONCAKE_MASTER: "127.0.0.1:50051"
|
|
MOONCAKE_TE_META_DATA_SERVER: "http://127.0.0.1:8080/metadata"
|
|
MOONCAKE_PROTOCOL: "tcp"
|
|
LOCAL_HOSTNAME: "127.0.0.1"
|
|
run: |
|
|
source test_env/bin/activate
|
|
python scripts/test_tensor_api.py -n 1
|
|
shell: bash
|
|
|
|
- name: Run Python Async API Test (CI check)
|
|
env:
|
|
MOONCAKE_MASTER: "127.0.0.1:50051"
|
|
MOONCAKE_TE_META_DATA_SERVER: "http://127.0.0.1:8080/metadata"
|
|
MOONCAKE_PROTOCOL: "tcp"
|
|
LOCAL_HOSTNAME: "127.0.0.1"
|
|
run: |
|
|
source test_env/bin/activate
|
|
python scripts/test_async_store.py
|
|
shell: bash
|
|
|
|
- name: Test Mooncake Copy/Move API
|
|
env:
|
|
MOONCAKE_MASTER: "127.0.0.1:50051"
|
|
MOONCAKE_TE_META_DATA_SERVER: "http://127.0.0.1:8080/metadata"
|
|
MOONCAKE_PROTOCOL: "tcp"
|
|
LOCAL_HOSTNAME: "127.0.0.1"
|
|
run: |
|
|
source test_env/bin/activate
|
|
python scripts/test_copy_move_api.py
|
|
shell: bash
|
|
|
|
- name: Run Python Drain HTTP E2E Test (CI check)
|
|
env:
|
|
MOONCAKE_MASTER: "127.0.0.1:50051"
|
|
MOONCAKE_TE_META_DATA_SERVER: "http://127.0.0.1:8080/metadata"
|
|
MOONCAKE_PROTOCOL: "tcp"
|
|
LOCAL_HOSTNAME: "127.0.0.1"
|
|
run: |
|
|
source test_env/bin/activate
|
|
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib
|
|
python scripts/test_drain_http_api.py --timeout-sec 90
|
|
shell: bash
|
|
|
|
- name: Run RPC Communicator Bandwidth Test
|
|
run: |
|
|
source test_env/bin/activate
|
|
python mooncake-transfer-engine/tests/rpc_communicator_test.py server --url 127.0.0.1:9004 --data-size 1 &
|
|
SERVER_PID=$!
|
|
sleep 5
|
|
timeout 10 python mooncake-transfer-engine/tests/rpc_communicator_test.py client --url 127.0.0.1:9004 --threads 2 --data-size 1 || true
|
|
kill $SERVER_PID 2>/dev/null || true
|
|
wait $SERVER_PID 2>/dev/null || true
|
|
|
|
- name: Test Mooncake PyTorch Backend (CPU Only)
|
|
env:
|
|
MC_FORCE_TCP: "true"
|
|
run: |
|
|
source test_env/bin/activate
|
|
python -m unittest mooncake-wheel.tests.test_mooncake_backend_cpu
|
|
shell: bash
|
|
|
|
- name: Test Safetensor Functions
|
|
run: |
|
|
source test_env/bin/activate
|
|
pip install safetensors
|
|
python -m unittest mooncake-wheel.tests.test_safetensor_functions
|
|
shell: bash
|
|
|
|
build-flags:
|
|
needs: [spell-check, clang-format, check-paths]
|
|
if: >-
|
|
(needs.check-paths.outputs.should-run-downstream == 'true' ||
|
|
github.event_name == 'workflow_dispatch') &&
|
|
(github.event_name == 'push' ||
|
|
github.event_name == 'workflow_dispatch' ||
|
|
github.event.action == 'opened' ||
|
|
contains(github.event.pull_request.labels.*.name, 'run-ci'))
|
|
runs-on: ubuntu-22.04
|
|
strategy:
|
|
matrix:
|
|
python-version: ['3.10', '3.12']
|
|
env:
|
|
CI: "true"
|
|
BUILD_WITH_EP: "1"
|
|
TORCH_CUDA_ARCH_LIST: "8.0;9.0"
|
|
SCCACHE_GHA_ENABLED: "true"
|
|
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- name: Set up Python ${{ matrix.python-version }}
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: ${{ matrix.python-version }}
|
|
|
|
- name: Free up disk space
|
|
run: |
|
|
sudo rm -rf /usr/share/dotnet
|
|
sudo rm -rf /opt/ghc
|
|
sudo rm -rf /opt/hostedtoolcache/CodeQL
|
|
sudo rm -rf /usr/local/lib/android
|
|
df -h
|
|
|
|
- name: Install CUDA Toolkit
|
|
uses: Jimver/cuda-toolkit@v0.2.24
|
|
with:
|
|
cuda: '12.8.1'
|
|
linux-local-args: '["--toolkit"]'
|
|
method: 'network'
|
|
sub-packages: '["nvcc", "nvrtc-dev"]'
|
|
non-cuda-sub-packages: '["libcusparse-dev", "libcublas-dev", "libcusolver-dev"]'
|
|
|
|
- name: Run sccache-cache
|
|
uses: mozilla-actions/sccache-action@v0.0.9
|
|
|
|
- name: Configure sccache
|
|
uses: actions/github-script@v7
|
|
with:
|
|
script: |
|
|
core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || '');
|
|
core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
|
|
|
|
- name: Run sccache stat for check
|
|
shell: bash
|
|
run: ${SCCACHE_PATH} --show-stats
|
|
|
|
- name: Install dependencies
|
|
run: |
|
|
sudo apt update -y
|
|
sudo apt install -y ninja-build
|
|
sudo bash -x dependencies.sh -y
|
|
df -h
|
|
shell: bash
|
|
|
|
- name: Install Rust toolchain
|
|
uses: dtolnay/rust-toolchain@stable
|
|
|
|
- name: Build transfer engine only
|
|
run: |
|
|
cd mooncake-transfer-engine
|
|
mkdir build
|
|
cd build
|
|
export PATH=/usr/local/nvidia/bin:/usr/local/nvidia/lib64:$PATH
|
|
export LD_LIBRARY_PATH=/usr/local/cuda/lib64/stubs:$LD_LIBRARY_PATH
|
|
cmake -G Ninja .. -DUSE_ETCD=OFF -DUSE_CXL=ON -DUSE_REDIS=ON -DUSE_HTTP=ON -DWITH_METRICS=ON -DBUILD_UNIT_TESTS=ON -DBUILD_EXAMPLES=ON -DENABLE_SCCACHE=ON -DUSE_CUDA=OFF -DUSE_MNNVL=OFF -DUSE_UB=OFF -DCMAKE_EXE_LINKER_FLAGS="-L/usr/local/cuda/lib64/stubs"
|
|
cmake --build .
|
|
sudo cmake --install .
|
|
df -h
|
|
shell: bash
|
|
|
|
- name: Configure project with all settings are ON
|
|
run: |
|
|
mkdir build
|
|
cd build
|
|
cmake -G Ninja .. -DUSE_ETCD=ON -DUSE_CXL=ON -DUSE_REDIS=ON -DUSE_HTTP=ON -DWITH_STORE=ON -DWITH_P2P_STORE=ON -DWITH_METRICS=ON -DBUILD_UNIT_TESTS=ON -DBUILD_EXAMPLES=ON -DENABLE_SCCACHE=ON -DUSE_CUDA=ON -DUSE_MNNVL=OFF -DUSE_UB=OFF -DCMAKE_EXE_LINKER_FLAGS="-L/usr/local/cuda/lib64/stubs"
|
|
shell: bash
|
|
# TODO: lack USE_NVMEOF,USE_MNNVL
|
|
|
|
- name: Build project with all settings are ON
|
|
run: |
|
|
export LD_LIBRARY_PATH=/usr/local/cuda/lib64/stubs:$LD_LIBRARY_PATH
|
|
export LIBRARY_PATH=/usr/local/cuda/lib64/stubs:$LIBRARY_PATH
|
|
cd build
|
|
cmake --build .
|
|
sudo cmake --install .
|
|
df -h
|
|
shell: bash
|
|
|
|
- name: Configure project with unit tests and examples
|
|
run: |
|
|
cd build
|
|
cmake -G Ninja .. -DBUILD_UNIT_TESTS=ON -DBUILD_EXAMPLES=ON -DWITH_STORE_RUST=ON -DENABLE_SCCACHE=ON
|
|
shell: bash
|
|
|
|
- name: Build project with unit tests and examples
|
|
run: |
|
|
export LD_LIBRARY_PATH=/usr/local/cuda/lib64/stubs:$LD_LIBRARY_PATH
|
|
export LIBRARY_PATH=/usr/local/cuda/lib64/stubs:$LIBRARY_PATH
|
|
cd build
|
|
cmake --build .
|
|
sudo cmake --install .
|
|
shell: bash
|
|
|
|
- name: Check Mooncake Store Rust bindings, examples, and tests
|
|
run: |
|
|
cd mooncake-store/rust
|
|
export MOONCAKE_BUILD_DIR=$GITHUB_WORKSPACE/build
|
|
cargo test --lib
|
|
MOONCAKE_STORE_LIB_DIR=$GITHUB_WORKSPACE/build/mooncake-store/src \
|
|
MOONCAKE_STORE_INCLUDE_DIR=$GITHUB_WORKSPACE/mooncake-store/include \
|
|
cargo test --examples --tests --no-run
|
|
shell: bash
|
|
|
|
- name: Configure project
|
|
run: |
|
|
cd build
|
|
rm -r */tests
|
|
cmake -G Ninja .. -DBUILD_UNIT_TESTS=OFF -DBUILD_EXAMPLES=OFF -DUSE_HTTP=ON -DENABLE_SCCACHE=ON -DUSE_CXL=ON -DWITH_EP=ON -DEP_TORCH_VERSIONS="2.9.0;2.9.1;2.10.0;2.11.0"
|
|
shell: bash
|
|
|
|
- name: Build project
|
|
run: |
|
|
export LD_LIBRARY_PATH=/usr/local/cuda/lib64/stubs:$LD_LIBRARY_PATH
|
|
export LIBRARY_PATH=/usr/local/cuda/lib64/stubs:$LIBRARY_PATH
|
|
cd build
|
|
cmake --build .
|
|
sudo cmake --install .
|
|
shell: bash
|
|
|
|
- name: Configure project with TENT
|
|
run: |
|
|
mkdir build-tent
|
|
cd build-tent
|
|
cmake -G Ninja .. -DUSE_TENT=ON -DUSE_HTTP=ON -DUSE_CUDA=ON -DENABLE_SCCACHE=ON -DBUILD_UNIT_TESTS=ON -DBUILD_EXAMPLES=ON -DCMAKE_EXE_LINKER_FLAGS="-L/usr/local/cuda/lib64/stubs"
|
|
shell: bash
|
|
|
|
- name: Build project with TENT
|
|
run: |
|
|
export LD_LIBRARY_PATH=/usr/local/cuda/lib64/stubs:$LD_LIBRARY_PATH
|
|
export LIBRARY_PATH=/usr/local/cuda/lib64/stubs:$LIBRARY_PATH
|
|
cd build-tent
|
|
cmake --build .
|
|
sudo cmake --install .
|
|
shell: bash
|
|
|
|
- name: Build nvlink_allocator.so
|
|
run: |
|
|
mkdir -p build/mooncake-transfer-engine/nvlink-allocator
|
|
cd mooncake-transfer-engine/nvlink-allocator
|
|
export PATH=/usr/local/nvidia/bin:/usr/local/nvidia/lib64:$PATH
|
|
export LD_LIBRARY_PATH=/usr/local/cuda/lib64/stubs:$LD_LIBRARY_PATH
|
|
export LIBRARY_PATH=/usr/local/cuda/lib64/stubs:$LIBRARY_PATH
|
|
bash build.sh ../../build/mooncake-transfer-engine/nvlink-allocator/
|
|
shell: bash
|
|
|
|
- name: Generate Python version tag
|
|
id: generate_tag_flags
|
|
run: |
|
|
echo "python_version_tag=$(echo ${{ matrix.python-version }} | tr -d '.')" >> $GITHUB_OUTPUT
|
|
shell: bash
|
|
|
|
# In CI, build_wheel.sh removes build/ to free disk (CI=true); set FREE_BUILD_DIR=1 locally to enable.
|
|
- name: Build Python wheel
|
|
run: |
|
|
PYTHON_VERSION=${{ matrix.python-version }} OUTPUT_DIR=dist-py${{ steps.generate_tag_flags.outputs.python_version_tag }} ./scripts/build_wheel.sh
|
|
shell: bash
|
|
|
|
- name: Upload Python wheel artifact
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: mooncake-wheel-ubuntu-py${{ steps.generate_tag_flags.outputs.python_version_tag }}
|
|
path: mooncake-wheel/dist-py${{ steps.generate_tag_flags.outputs.python_version_tag }}/*.whl
|
|
|
|
build-docker:
|
|
name: Build Docker Image
|
|
needs: [spell-check, clang-format, check-paths]
|
|
if: >-
|
|
(needs.check-paths.outputs.should-run-downstream == 'true' ||
|
|
github.event_name == 'workflow_dispatch') &&
|
|
(github.event_name == 'push' ||
|
|
github.event_name == 'workflow_dispatch' ||
|
|
github.event.action == 'opened' ||
|
|
contains(github.event.pull_request.labels.*.name, 'run-ci'))
|
|
runs-on: ubuntu-22.04
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- name: Set up Docker Buildx
|
|
uses: docker/setup-buildx-action@v2
|
|
|
|
- name: Build Docker image
|
|
run: |
|
|
docker build -f docker/mooncake.Dockerfile \
|
|
--build-arg PYTHON_VERSION=3.10 \
|
|
--build-arg EP_TORCH_VERSIONS="2.9.1" \
|
|
-t mooncake:from-source .
|
|
|
|
spell-check:
|
|
name: Spell Check with Typos
|
|
if: >-
|
|
(github.event_name == 'push' ||
|
|
github.event_name == 'workflow_dispatch' ||
|
|
github.event.action == 'opened' ||
|
|
contains(github.event.pull_request.labels.*.name, 'run-ci'))
|
|
runs-on: ubuntu-22.04
|
|
steps:
|
|
- name: Checkout Actions Repository
|
|
uses: actions/checkout@v4
|
|
with:
|
|
persist-credentials: false
|
|
- name: Spell Check Repo
|
|
uses: crate-ci/typos@v1.30.2
|
|
|
|
clang-format:
|
|
name: Check code format
|
|
if: >-
|
|
(github.event_name == 'push' ||
|
|
github.event_name == 'workflow_dispatch' ||
|
|
github.event.action == 'opened' ||
|
|
contains(github.event.pull_request.labels.*.name, 'run-ci'))
|
|
runs-on: ubuntu-22.04
|
|
steps:
|
|
- name: Checkout Actions Repository
|
|
uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: 0 # Need full history for branch comparison
|
|
persist-credentials: false
|
|
|
|
- name: Install clang-format 20
|
|
run: |
|
|
wget https://apt.llvm.org/llvm.sh
|
|
chmod +x llvm.sh
|
|
sudo ./llvm.sh 20
|
|
sudo apt-get install -y clang-format-20
|
|
|
|
- name: Check code format
|
|
run: |
|
|
# Check script exists and is executable
|
|
if [[ ! -x ./scripts/code_format.sh ]]; then
|
|
echo "Error: code_format.sh not found or not executable"
|
|
exit 1
|
|
fi
|
|
|
|
# Determine base ref for comparison
|
|
if [ "${{ github.event_name }}" == "pull_request" ]; then
|
|
# For PRs: compare against the target branch
|
|
BASE_REF="origin/${{ github.base_ref }}"
|
|
else
|
|
# For push events: use github.event.before to handle multi-commit pushes
|
|
BEFORE_SHA="${{ github.event.before }}"
|
|
if [ "${BEFORE_SHA}" == "0000000000000000000000000000000000000000" ]; then
|
|
# New branch push, compare against default branch
|
|
BASE_REF="origin/${{ github.event.repository.default_branch }}"
|
|
else
|
|
# Normal push (single or multiple commits)
|
|
BASE_REF="${BEFORE_SHA}"
|
|
fi
|
|
fi
|
|
echo "Comparing against: ${BASE_REF}"
|
|
./scripts/code_format.sh --check --base "${BASE_REF}"
|
|
shell: bash
|
|
|
|
|
|
check-paths:
|
|
if: >-
|
|
github.event_name == 'push' ||
|
|
github.event_name == 'workflow_dispatch' ||
|
|
github.event.action == 'opened' ||
|
|
contains(github.event.pull_request.labels.*.name, 'run-ci')
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
should-run-downstream: ${{ steps.dispatch-override.outputs.src || steps.filter.outputs.src }}
|
|
steps:
|
|
# workflow_dispatch has no PR/push diff context — skip paths-filter and default to true
|
|
- name: Default to true for workflow_dispatch
|
|
id: dispatch-override
|
|
if: github.event_name == 'workflow_dispatch'
|
|
run: echo "src=true" >> $GITHUB_OUTPUT
|
|
- uses: actions/checkout@v4
|
|
if: github.event_name != 'workflow_dispatch'
|
|
with:
|
|
fetch-depth: 2
|
|
persist-credentials: false
|
|
- uses: dorny/paths-filter@v3
|
|
if: github.event_name != 'workflow_dispatch'
|
|
id: filter
|
|
with:
|
|
filters: |
|
|
src:
|
|
- 'mooncake-*/**'
|
|
- 'extern/**'
|
|
- 'CMakeLists.txt'
|
|
- 'dependencies.sh'
|
|
- 'scripts/**'
|
|
- '.github/workflows/**'
|
|
|
|
build-wheel-cu13:
|
|
needs: [spell-check, clang-format, check-paths]
|
|
if: >-
|
|
(needs.check-paths.outputs.should-run-downstream == 'true' ||
|
|
github.event_name == 'workflow_dispatch') &&
|
|
(github.event_name == 'push' ||
|
|
github.event_name == 'workflow_dispatch' ||
|
|
github.event.action == 'opened' ||
|
|
contains(github.event.pull_request.labels.*.name, 'run-ci'))
|
|
uses: ./.github/workflows/ci_cu13.yml
|
|
secrets: inherit
|
|
|
|
ascend-test:
|
|
needs: [build, check-paths]
|
|
if: needs.check-paths.outputs.should-run-downstream == 'true'
|
|
uses: ./.github/workflows/ci_ascend.yml
|
|
secrets: inherit
|
|
|
|
integration-test:
|
|
needs: [build, check-paths]
|
|
if: needs.check-paths.outputs.should-run-downstream == 'true'
|
|
uses: ./.github/workflows/integration-test.yml
|
|
secrets: inherit
|
|
|
|
ci-gate:
|
|
name: CI Gate
|
|
if: always()
|
|
needs:
|
|
- spell-check
|
|
- clang-format
|
|
- build
|
|
- build-musa
|
|
- build-flags
|
|
- build-docker
|
|
- test-wheel-ubuntu
|
|
- build-wheel-cu13
|
|
- ascend-test
|
|
- integration-test
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Check required job results
|
|
run: |
|
|
failing=$(echo "$NEEDS_JSON" | jq -r '
|
|
to_entries[] |
|
|
select(.value.result != "success" and .value.result != "skipped") |
|
|
"\(.key): \(.value.result)"')
|
|
if [ -n "$failing" ]; then
|
|
echo "::error::The following jobs failed or were cancelled:"
|
|
echo "$failing"
|
|
exit 1
|
|
fi
|
|
echo "All checks passed or were acceptably skipped."
|
|
env:
|
|
NEEDS_JSON: ${{ toJSON(needs) }}
|