diff --git a/mooncake-store/src/transfer_task.cpp b/mooncake-store/src/transfer_task.cpp index 0299bbbe..2d3c7b6b 100644 --- a/mooncake-store/src/transfer_task.cpp +++ b/mooncake-store/src/transfer_task.cpp @@ -244,9 +244,11 @@ void TransferEngineOperationState::check_task_status() { case TransferStatusEnum::FAILED: case TransferStatusEnum::CANCELED: case TransferStatusEnum::INVALID: +#ifndef USE_ASCEND_DIRECT LOG(ERROR) << "Transfer failed for batch " << batch_id_ << " task " << i << " with status " << static_cast(status.s); +#endif has_failure = true; break; default: diff --git a/mooncake-transfer-engine/include/transport/ascend_transport/ascend_direct_transport/ascend_direct_transport.h b/mooncake-transfer-engine/include/transport/ascend_transport/ascend_direct_transport/ascend_direct_transport.h index fb60f3ee..ec3b4069 100644 --- a/mooncake-transfer-engine/include/transport/ascend_transport/ascend_direct_transport/ascend_direct_transport.h +++ b/mooncake-transfer-engine/include/transport/ascend_transport/ascend_direct_transport/ascend_direct_transport.h @@ -124,8 +124,8 @@ class AscendDirectTransport : public Transport { int32_t device_logic_id_{}; aclrtContext rt_context_{nullptr}; - int32_t connect_timeout_ = 3000; - int32_t transfer_timeout_ = 3000; + int32_t connect_timeout_ = 10000; + int32_t transfer_timeout_ = 10000; std::string local_adxl_engine_name_{}; aclrtStream stream_{}; bool use_buffer_pool_{false}; diff --git a/mooncake-transfer-engine/src/transport/ascend_transport/ascend_direct_transport/ascend_direct_transport.cpp b/mooncake-transfer-engine/src/transport/ascend_transport/ascend_direct_transport/ascend_direct_transport.cpp index 95157f7b..dec4e147 100644 --- a/mooncake-transfer-engine/src/transport/ascend_transport/ascend_direct_transport/ascend_direct_transport.cpp +++ b/mooncake-transfer-engine/src/transport/ascend_transport/ascend_direct_transport/ascend_direct_transport.cpp @@ -573,8 +573,14 @@ void AscendDirectTransport::processSliceList( return; } if (target_adxl_engine_name == local_adxl_engine_name_) { - VLOG(1) << "Target is local, use memory copy."; - return localCopy(slice_list[0]->opcode, slice_list); + auto start = std::chrono::steady_clock::now(); + localCopy(slice_list[0]->opcode, slice_list); + LOG(INFO) << "Local copy time: " + << std::chrono::duration_cast( + std::chrono::steady_clock::now() - start) + .count() + << "us"; + return; } int ret = checkAndConnect(target_adxl_engine_name); if (ret != 0) { @@ -608,7 +614,14 @@ void AscendDirectTransport::processSliceList( .count() << " us"; } else { - LOG(ERROR) << "Transfer slice failed with status: " << status; + if (status == adxl::TIMEOUT) { + LOG(ERROR) << "Transfer timeout to: " << target_adxl_engine_name + << ", you can increase the timeout duration to reduce " + "the failure rate by configuring " + "the ASCEND_TRANSFER_TIMEOUT environment variable."; + } else { + LOG(ERROR) << "Transfer slice failed with status: " << status; + } for (auto &slice : slice_list) { slice->markFailed(); } @@ -816,7 +829,11 @@ int AscendDirectTransport::checkAndConnect( } auto status = adxl_->Connect(target_adxl_engine_name.c_str(), connect_timeout_); - if (status != adxl::SUCCESS) { + if (status == adxl::TIMEOUT) { + LOG(ERROR) << "Connect timeout to: " << target_adxl_engine_name + << ", you can increase the timeout duration to reduce " + "the ASCEND_CONNECT_TIMEOUT environment variable."; + } else if (status != adxl::SUCCESS) { LOG(ERROR) << "Failed to connect to target: " << target_adxl_engine_name << ", status: " << status; return -1;