diff --git a/doc/en/ascend_transport.md b/doc/en/ascend_transport.md index 05d27e49..526d4b97 100644 --- a/doc/en/ascend_transport.md +++ b/doc/en/ascend_transport.md @@ -157,7 +157,7 @@ You can configure various scenarios (e.g., 1-to-1, 1-to-2, 2-to-1) and performan **Start Initiator Node:** ```bash -./transfer_engine_ascend_perf --metadata_server=P2PHANDSHAKE --local_server_name=10.0.0.0:12345 --protocol=hccl --operation=write --segment_id=10.0.0.0:12346 --device_id=0 --mode=initiator --block_size=8388608 +./transfer_engine_ascend_perf --metadata_server=P2PHANDSHAKE --local_server_name=10.0.0.0:12345 --protocol=hccl --operation=write --segment_id=10.0.0.0:12346 --device_id=0 --mode=initiator --block_size=16384 ``` **Start Target Node:** @@ -187,13 +187,13 @@ export ASCEND_TRANSPORT_PRINT=1 ``` ### Timeout Configuration -Ascend Transport uses TCP-based out-of-band communication on the host side, with a receive timeout set to 120 seconds. +Ascend Transport based on TCP for out-of-band communication, has a connection timeout configured via the environment variable `Ascend_TCP_TIMEOUT`, with a default value of 30 seconds. On the host side, the `recv` timeout is set to 30 seconds, meaning that if no message is received from the peer within 30 seconds, an error will be reported. -Connection timeout is controlled by the environment variable HCCL_CONNECT_TIMEOUT. -Execution timeout is configured via HCCL_EXEC_TIMEOUT. -If no communication occurs within this timeout, the hccl_socket connection will be terminated. +The connection timeout for `hccl_socket` is configured through the environment variable `Ascend_HCCL_SOCKET_TIMEOUT`, with a default value of 30 seconds. If this timeout is exceeded, the current transmission will report an error and return. -Point-to-point communication between endpoints involves a connection handshake with a timeout of 120 seconds. +`hccl_socket` has a keep-alive requirement, and the execution timeout is configured via the environment variable `HCCL_EXEC_TIMEOUT`. If no communication occurs within the `HCCL_EXEC_TIMEOUT` period, the `hccl_socket` connection will be disconnected. + +In `transport_mem`, the point-to-point communication between endpoints involves a connection handshake process, and its timeout is configured via `Ascend_TRANSPORT_MEM_TIMEOUT`, with a default value of 120 seconds. ### Error Code Ascend transport error codes reuse the HCCL collective communication transport error codes. diff --git a/doc/zh/ascend_transport.md b/doc/zh/ascend_transport.md index 381b4bba..2a8e418c 100644 --- a/doc/zh/ascend_transport.md +++ b/doc/zh/ascend_transport.md @@ -97,7 +97,7 @@ Ascend Transport提供多场景测试mooncake-transfer-engine/example/transfer_e 性能用例执行命令如： ```启动发起节点：``` -./transfer_engine_ascend_perf --metadata_server=P2PHANDSHAKE --local_server_name=10.0.0.0:12345 --protocol=hccl --operation=write --segment_id=10.0.0.0:12346 --device_id=0 --mode=initiator --block_size=8388608 +./transfer_engine_ascend_perf --metadata_server=P2PHANDSHAKE --local_server_name=10.0.0.0:12345 --protocol=hccl --operation=write --segment_id=10.0.0.0:12346 --device_id=0 --mode=initiator --block_size=16384 ```启动目标节点：``` ./transfer_engine_ascend_perf --metadata_server=P2PHANDSHAKE --local_server_name=10.0.0.0:12346 --protocol=hccl --operation=write --device_id=1 --mode=target @@ -112,11 +112,13 @@ Ascend Transport提供多场景测试mooncake-transfer-engine/example/transfer_e export ASCEND_TRANSPORT_PRINT=1 ### 超时时间配置 -Ascend Transport基于TCP的带外通信，在主机侧接收超时设置为 120 秒。 +Ascend Transport基于TCP的带外通信，连接的超时时间通过环境变量Ascend_TCP_TIMEOUT配置，默认为30秒，在主机侧recv接收超时设置为30秒，即recv阻塞超过30s未收到对端的消息会报错。 -在hccl_socket中，连接超时时间由环境变量HCCL_CONNECT_TIMEOUT配置，执行超时通过环境变量HCCL_EXEC_TIMEOUT配置，超过HCCL_EXEC_TIMEOUT未进行通信，会断开hccl_socket连接。 +hccl_socket的连接超时时间通过环境变量Ascend_HCCL_SOCKET_TIMEOUT配置，默认为30秒，超时则本次传输会报错并返回。 -在transport_mem中，端到端之间的点对点通信涉及连接握手过程，其超时时间为 120 秒。 +hccl_socket有保活要求，执行超时通过环境变量HCCL_EXEC_TIMEOUT配置，超过HCCL_EXEC_TIMEOUT未进行通信，会断开hccl_socket连接。 + +在transport_mem中，端到端之间的点对点通信涉及连接握手过程，其超时时间通过Ascend_TRANSPORT_MEM_TIMEOUT配置，默认为120秒。 ### 错误码 Ascend传输错误码沿用HCCL集合通信传输错误码。 diff --git a/mooncake-common/FindMpi.cmake b/mooncake-common/FindMpi.cmake new file mode 100644 index 00000000..ee3ae5fa --- /dev/null +++ b/mooncake-common/FindMpi.cmake @@ -0,0 +1,44 @@ +find_package(MPI QUIET CONFIG) + +if(TARGET MPI::MPI) + set(MPI_FOUND TRUE) + set(MPI_TARGET MPI::MPI) +else() + find_package(PkgConfig QUIET) + if(PKG_CONFIG_FOUND) + pkg_check_modules(PC_MPI QUIET mpi) + endif() + + file(GLOB MPI_INCLUDE_DIRS "/usr/include/mpich*" "/usr/local/include/mpich*") + + set(MPI_INCLUDE_SEARCH_PATHS "") + foreach(dir IN LISTS MPI_INCLUDE_DIRS) + list(APPEND MPI_INCLUDE_SEARCH_PATHS "${dir}") + endforeach() + + find_path(MPI_INCLUDE_DIR mpi.h + HINTS ${PC_MPI_INCLUDEDIR} ${PC_MPI_INCLUDE_DIRS} + PATHS /usr/include /usr/local/include ${MPI_INCLUDE_SEARCH_PATHS}) + + file(GLOB MPI_LIB_DIRS "/usr/lib/mpich*" "/usr/local/lib/mpich*" "/usr/local/mpich*/lib") + + set(MPI_LIB_SEARCH_PATHS "") + foreach(dir IN LISTS MPI_LIB_DIRS) + list(APPEND MPI_LIB_SEARCH_PATHS "${dir}") + endforeach() + + find_library(MPI_LIBRARY mpi + HINTS ${PC_MPI_LIBDIR} ${PC_MPI_LIBRARY_DIRS} + PATHS /usr/lib /usr/lib64 /usr/local/lib /usr/local/lib64 ${MPI_LIB_SEARCH_PATHS}) + + if(MPI_INCLUDE_DIR AND MPI_LIBRARY) + set(MPI_FOUND TRUE) + add_library(MPI::MPI INTERFACE IMPORTED) + target_include_directories(MPI::MPI INTERFACE ${MPI_INCLUDE_DIR}) + target_link_libraries(MPI::MPI INTERFACE ${MPI_LIBRARY}) + set(MPI_TARGET MPI::MPI) + endif() +endif() + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(MPI DEFAULT_MSG MPI_TARGET) \ No newline at end of file diff --git a/mooncake-common/include/default_config.h b/mooncake-common/include/default_config.h index 2fe0f563..f08291ba 100644 --- a/mooncake-common/include/default_config.h +++ b/mooncake-common/include/default_config.h @@ -1,6 +1,10 @@ #pragma once -#include +#if __has_include() +#include // Ubuntu +#else +#include // CentOS +#endif #include #include diff --git a/mooncake-common/src/default_config.cpp b/mooncake-common/src/default_config.cpp index ac34377f..a1046cdf 100644 --- a/mooncake-common/src/default_config.cpp +++ b/mooncake-common/src/default_config.cpp @@ -1,7 +1,13 @@ #include "default_config.h" +#if __has_include() #include -#include +#include // Ubuntu +#else +#include +#include // CentOS +#endif + #include #include diff --git a/mooncake-transfer-engine/CMakeLists.txt b/mooncake-transfer-engine/CMakeLists.txt index 83682a88..0e7b152f 100644 --- a/mooncake-transfer-engine/CMakeLists.txt +++ b/mooncake-transfer-engine/CMakeLists.txt @@ -11,7 +11,7 @@ if (NOT GLOBAL_CONFIG) endif() # GLOBAL_CONFIG if (USE_ASCEND) - find_package(MPI REQUIRED) + include(../mooncake-common/FindMpi.cmake) file(GLOB ASCEND_TOOLKIT_ROOT "/usr/local/Ascend/ascend-toolkit/latest/*-linux") set(ASCEND_INCLUDE_DIR "${ASCEND_TOOLKIT_ROOT}/include") @@ -25,7 +25,7 @@ if (USE_ASCEND) ${ASCEND_INCLUDE_DIR}/experiment/metadef/common/util/error_manager ${ASCEND_INCLUDE_DIR}/experiment/runtime ${ASCEND_INCLUDE_DIR}/experiment/msprof - ${MPI_CXX_INCLUDE_DIRS} + ${MPI_INCLUDE_DIR} /usr/local/Ascend/ascend-toolkit/latest/tools/hccl_test/common/src/ ) endif() diff --git a/mooncake-transfer-engine/src/CMakeLists.txt b/mooncake-transfer-engine/src/CMakeLists.txt index 5c6493f7..8fd118c8 100644 --- a/mooncake-transfer-engine/src/CMakeLists.txt +++ b/mooncake-transfer-engine/src/CMakeLists.txt @@ -48,9 +48,8 @@ if (USE_CUDA) endif() if (USE_ASCEND) - find_package(MPI REQUIRED) file(GLOB ASCEND_TOOLKIT_ROOT "/usr/local/Ascend/ascend-toolkit/latest/*-linux") set(ASCEND_LIB_DIR "${ASCEND_TOOLKIT_ROOT}/lib64") link_directories(${ASCEND_LIB_DIR}) - target_link_libraries(transfer_engine PUBLIC ascendcl hccl ascend_transport ${MPI_CXX_LIBRARIES} ${MPI_C_LIBRARIES}) + target_link_libraries(transfer_engine PUBLIC ascendcl hccl ascend_transport MPI::MPI) endif() \ No newline at end of file diff --git a/mooncake-transfer-engine/src/transport/ascend_transport/hccl_transport/ascend_transport_c/CMakeLists.txt b/mooncake-transfer-engine/src/transport/ascend_transport/hccl_transport/ascend_transport_c/CMakeLists.txt index 58589f4a..9218a4fc 100644 --- a/mooncake-transfer-engine/src/transport/ascend_transport/hccl_transport/ascend_transport_c/CMakeLists.txt +++ b/mooncake-transfer-engine/src/transport/ascend_transport/hccl_transport/ascend_transport_c/CMakeLists.txt @@ -5,32 +5,12 @@ set(CMAKE_POSITION_INDEPENDENT_CODE OFF) string(REPLACE "-O2" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") string(REPLACE "-O2" "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}") -file(GLOB ASCEND_TOOLKIT_ROOT "/usr/local/Ascend/ascend-toolkit/latest/*-linux") -set(ASCEND_LIB_DIR "${ASCEND_TOOLKIT_ROOT}/lib64") -set(ASCEND_INCLUDE_DIR "${ASCEND_TOOLKIT_ROOT}/include") - -find_package(MPI REQUIRED) - -link_directories(${ASCEND_LIB_DIR}) -include_directories(/usr/local/include - /usr/include - ${ASCEND_INCLUDE_DIR} - ${ASCEND_INCLUDE_DIR}/hccl - ${ASCEND_INCLUDE_DIR}/experiment - ${ASCEND_INCLUDE_DIR}/experiment/hccl - ${ASCEND_INCLUDE_DIR}/experiment/slog/toolchain - ${ASCEND_INCLUDE_DIR}/experiment/metadef/common/util/error_manager - ${ASCEND_INCLUDE_DIR}/experiment/runtime - ${ASCEND_INCLUDE_DIR}/experiment/msprof - ${MPI_CXX_INCLUDE_DIRS} - /usr/local/Ascend/ascend-toolkit/latest/tools/hccl_test/common/src/ - ) add_library(ascend_transport_mem SHARED ${ASCEND_TRANSPORT_MEM_SOURCES}) set_target_properties(ascend_transport_mem PROPERTIES POSITION_INDEPENDENT_CODE ON) target_link_libraries(ascend_transport_mem PRIVATE hccl ascendcl glog gflags) -target_compile_options(ascend_transport_mem BEFORE PRIVATE "-std=c++11" "-fstack-protector-strong" "-O2" "-s" "-Wl,-z,relro" "-Wl,-z,now" "-Wl,-z,noexecstack" "-Wl,--copy-dt-needed-entries" "-DOPEN_BUILD_PROJECT" "-DLOG_CPP" "-D_GLIBCXX_USE_CXX11_ABI=0") -target_link_options(ascend_transport_mem BEFORE PRIVATE "-std=c++11" "-fstack-protector-strong" "-O2" "-s" "-Wl,-z,relro" "-Wl,-z,now" "-Wl,-z,noexecstack" "-Wl,--copy-dt-needed-entries" "-DOPEN_BUILD_PROJECT" "-DLOG_CPP" "-D_GLIBCXX_USE_CXX11_ABI=0") +target_compile_options(ascend_transport_mem BEFORE PRIVATE "-std=c++11" "-fstack-protector-strong" "-s" "-Wl,-z,relro" "-Wl,-z,now" "-Wl,-z,noexecstack" "-Wl,--copy-dt-needed-entries" "-DOPEN_BUILD_PROJECT" "-DLOG_CPP" "-D_GLIBCXX_USE_CXX11_ABI=0") +target_link_options(ascend_transport_mem BEFORE PRIVATE "-std=c++11" "-fstack-protector-strong" "-s" "-Wl,-z,relro" "-Wl,-z,now" "-Wl,-z,noexecstack" "-Wl,--copy-dt-needed-entries" "-DOPEN_BUILD_PROJECT" "-DLOG_CPP" "-D_GLIBCXX_USE_CXX11_ABI=0") diff --git a/mooncake-transfer-engine/src/transport/ascend_transport/hccl_transport/ascend_transport_c/hccl_transport_mem_c.cpp b/mooncake-transfer-engine/src/transport/ascend_transport/hccl_transport/ascend_transport_c/hccl_transport_mem_c.cpp index 0548fb4b..3fe281d2 100644 --- a/mooncake-transfer-engine/src/transport/ascend_transport/hccl_transport/ascend_transport_c/hccl_transport_mem_c.cpp +++ b/mooncake-transfer-engine/src/transport/ascend_transport/hccl_transport/ascend_transport_c/hccl_transport_mem_c.cpp @@ -292,7 +292,12 @@ static int connectToTarget(std::string target_ip, int target_port) { } int connected = 0; - for (int i = 0; i < RETRY_TIMES; ++i) { + + const char* tcp_timeout_str = std::getenv("Ascend_TCP_TIMEOUT"); + int ascend_tcp_timeout = tcp_timeout_str ? std::atoi(tcp_timeout_str) : 30; + int connect_retry_times = ascend_tcp_timeout * 100; + + for (int i = 0; i < connect_retry_times; ++i) { if (connect(client_socket, (struct sockaddr*)&server_addr, sizeof(server_addr)) == 0) { LOG(INFO) << "Connect to host server " << target_ip << ":" << ntohs(server_addr.sin_port) << " successful"; connected = 1; @@ -301,13 +306,13 @@ static int connectToTarget(std::string target_ip, int target_port) { LOG(INFO) << "Connect attempt " << i << " failed: " << strerror(errno) << ", retry once"; - std::this_thread::sleep_for(std::chrono::milliseconds(1)); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); } if (!connected) { - LOG(ERROR) << "Failed to connect to server after " << RETRY_TIMES << " retries"; + LOG(ERROR) << "Failed to connect to server after " << connect_retry_times << " retries"; close(client_socket); - return -1; + return HCCL_E_TIMEOUT; } return client_socket; @@ -424,10 +429,23 @@ int createClientSocket(std::shared_ptr &hccl_socket, RankInfo return ret; } LOG(INFO) << "hccl_socket begin to connect, local devicePhyId: " << local_rank_info->devicePhyId << ", target devicePhyId: " << remote_rank_info->devicePhyId; + hccl::HcclSocketStatus status; + struct timespec start, end; + const char* hccl_socket_timeout_str = std::getenv("Ascend_HCCL_SOCKET_TIMEOUT"); + int hccl_socket_timeout = hccl_socket_timeout_str ? std::atoi(hccl_socket_timeout_str) : 30; + long long hccl_socket_timeout_ns = static_cast(hccl_socket_timeout) * 1000000000LL; + clock_gettime(CLOCK_MONOTONIC, &start); do { status = hccl_socket->GetStatus(); + clock_gettime(CLOCK_MONOTONIC, &end); + long long elapsed_time = (end.tv_sec - start.tv_sec) * 1000000000LL + (end.tv_nsec - start.tv_nsec); + if (elapsed_time > hccl_socket_timeout_ns) { // Exceeds 20 seconds,TimeOut + LOG(ERROR) << "hccl_socket connect timeout, local devicePhyId: " << local_rank_info->devicePhyId << ", target devicePhyId: " << remote_rank_info->devicePhyId; + return HCCL_E_TIMEOUT; + } } while (status != hccl::HcclSocketStatus::SOCKET_OK); + LOG(INFO) << "hccl_socket connect success, local devicePhyId: " << local_rank_info->devicePhyId << ", target devicePhyId: " << remote_rank_info->devicePhyId; return 0; @@ -495,7 +513,9 @@ int createTransportMem(RankInfo *local_rank_info, RankInfo *remote_rank_info, st << ", ret: " << ret; return ret; } - ret = transport_mem->Connect(120); + const char* transport_mem_timeout_str = std::getenv("Ascend_TRANSPORT_MEM_TIMEOUT"); + int transport_mem_timeout = transport_mem_timeout_str ? std::atoi(transport_mem_timeout_str) : 120; + ret = transport_mem->Connect(transport_mem_timeout); if (ret) { char deviceIp[64]; inet_ntop(AF_INET, &remote_rank_info->deviceIp, deviceIp, sizeof(deviceIp)); @@ -821,7 +841,9 @@ int transportMemAccept(RankInfo *local_rank_info) { << ", ret: " << ret; return ret; } - ret = transport_mem->Connect(120); + const char* transport_mem_timeout_str = std::getenv("Ascend_TRANSPORT_MEM_TIMEOUT"); + int transport_mem_timeout = transport_mem_timeout_str ? std::atoi(transport_mem_timeout_str) : 120; + ret = transport_mem->Connect(transport_mem_timeout); if (ret) { char deviceIp[64]; inet_ntop(AF_INET, &rempoteDevIp, deviceIp, sizeof(deviceIp)); @@ -906,8 +928,7 @@ int transportMemAccept(RankInfo *local_rank_info) { return 0; } -int regLocalRmaMem(void *addr, uint64_t length) -{ +int regLocalRmaMem(void *addr, uint64_t length) { g_localMergeMem.push_back(MergeMem{addr, length}); return 0; } diff --git a/scripts/ascend/dependencies_ascend.sh b/scripts/ascend/dependencies_ascend.sh index 785b8121..dc5b9c38 100644 --- a/scripts/ascend/dependencies_ascend.sh +++ b/scripts/ascend/dependencies_ascend.sh @@ -46,7 +46,6 @@ set +e if command -v apt-get &> /dev/null; then echo "Detected apt-get. Using Debian-based package manager." apt-get update - apt purge -y openmpi-bin libopenmpi-dev || true apt-get install -y build-essential \ cmake \ git \ @@ -63,6 +62,7 @@ if command -v apt-get &> /dev/null; then libgrpc-dev \ libgrpc++-dev \ libprotobuf-dev \ + libyaml-cpp-dev \ protobuf-compiler-grpc \ libcurl4-openssl-dev \ libhiredis-dev \ @@ -70,6 +70,7 @@ if command -v apt-get &> /dev/null; then patchelf \ mpich \ libmpich-dev + apt purge -y openmpi-bin libopenmpi-dev || true elif command -v yum &> /dev/null; then echo "Detected yum. Using Red Hat-based package manager." yum makecache @@ -87,6 +88,16 @@ elif command -v yum &> /dev/null; then jsoncpp-devel \ mpich \ mpich-devel + # Install yaml-cpp + cd "$TARGET_DIR" + clone_repo_if_not_exists "yaml-cpp" https://github.com/jbeder/yaml-cpp.git + cd yaml-cpp || exit + rm -rf build + mkdir -p build && cd build + cmake .. + make -j$(nproc) + make install + cd ../.. else echo "Unsupported package manager. Please install the dependencies manually." exit 1 @@ -118,8 +129,12 @@ cd Mooncake || exit if ! git submodule update --init --recursive; then if [ ! -d "extern/pybind11" ] || [ -z "$(ls -A 'extern/pybind11' 2>/dev/null)" ]; then echo "git submodule update failed, try to cp pybind11..." - cp -r ../pybind11 extern/ - check_success "Failed to update submodule pybind11" + if [ -d "../pybind11" ]; then + cp -r ../pybind11 extern/ + else + echo "Error: ../pybind11 does not exist. Cannot copy pybind11." + exit 1 + fi else echo "Detected that extern/pybind11 already exists, continuing execution...." fi @@ -136,7 +151,8 @@ echo -e "Mooncake installed successfully." # Add the so package to the environment variables export LD_LIBRARY_PATH=/usr/local/Ascend/ascend-toolkit/latest/python/site-packages:$LD_LIBRARY_PATH -cp build/mooncake-transfer-engine/src/transport/ascend_transport/hccl_transport/ascend_transport_c/libascend_transport_mem.so /usr/local/Ascend/ascend-toolkit/latest/python/site-packages +cp build/mooncake-transfer-engine/src/transport/ascend_transport/hccl_transport/ascend_transport_c/libascend_transport_mem.so build/ +cp build/libascend_transport_mem.so /usr/local/Ascend/ascend-toolkit/latest/python/site-packages cp /usr/local/lib/python*/site-packages/mooncake/*.so /usr/local/Ascend/ascend-toolkit/latest/python/site-packages # Copy the so package to a shared path for others to use diff --git a/scripts/ascend/dependencies_ascend_installation.sh b/scripts/ascend/dependencies_ascend_installation.sh index 81d576b4..5174fb43 100644 --- a/scripts/ascend/dependencies_ascend_installation.sh +++ b/scripts/ascend/dependencies_ascend_installation.sh @@ -42,7 +42,6 @@ set +e if command -v apt-get &> /dev/null; then echo "Detected apt-get. Using Debian-based package manager." apt-get update - apt purge -y openmpi-bin libopenmpi-dev || true apt-get install -y build-essential \ cmake \ git \ @@ -59,6 +58,7 @@ if command -v apt-get &> /dev/null; then libgrpc-dev \ libgrpc++-dev \ libprotobuf-dev \ + libyaml-cpp-dev \ protobuf-compiler-grpc \ libcurl4-openssl-dev \ libhiredis-dev \ @@ -66,6 +66,7 @@ if command -v apt-get &> /dev/null; then patchelf \ mpich \ libmpich-dev + apt purge -y openmpi-bin libopenmpi-dev || true elif command -v yum &> /dev/null; then echo "Detected yum. Using Red Hat-based package manager." yum makecache @@ -83,6 +84,16 @@ elif command -v yum &> /dev/null; then jsoncpp-devel \ mpich \ mpich-devel + # Install yaml-cpp + cd "$TARGET_DIR" + clone_repo_if_not_exists "yaml-cpp" https://github.com/jbeder/yaml-cpp.git + cd yaml-cpp || exit + rm -rf build + mkdir -p build && cd build + cmake .. + make -j$(nproc) + make install + cd ../.. else echo "Unsupported package manager. Please install the dependencies manually." exit 1