From 49328ae86d33eba1a825049926ddb26966148864 Mon Sep 17 00:00:00 2001 From: caifubi Date: Fri, 31 Dec 2021 10:55:10 +0800 Subject: [PATCH] Modify Hccl Error Log --- .../device/ascend/ascend_kernel_runtime.cc | 5 +- .../hardware/device_context_manager.cc | 4 +- .../runtime/hccl_adapter/hccl_adapter.cc | 60 +++++++++++++++++-- .../ccsrc/runtime/hccl_adapter/hccl_adapter.h | 9 ++- tests/ut/cpp/stub/ge/ge_task_launch_stub.cc | 2 +- 5 files changed, 67 insertions(+), 13 deletions(-) diff --git a/mindspore/ccsrc/runtime/device/ascend/ascend_kernel_runtime.cc b/mindspore/ccsrc/runtime/device/ascend/ascend_kernel_runtime.cc index f65f32c1cd0..debaed67a1c 100644 --- a/mindspore/ccsrc/runtime/device/ascend/ascend_kernel_runtime.cc +++ b/mindspore/ccsrc/runtime/device/ascend/ascend_kernel_runtime.cc @@ -1173,8 +1173,9 @@ bool AscendKernelRuntime::HcclInit() { return false; } MS_LOG(INFO) << "MINDSPORE_HCCL_CONFIG_PATH : " << full_path << ", RANK_ID: " << rank_id_str; - bool ret = hccl::HcclAdapter::GetInstance().InitHccl(context_ptr->get_param(MS_CTX_DEVICE_ID), rank_id_str, - full_path, mode == kGraphMode); + bool ret = hccl::HcclAdapter::GetInstance().InitHccl( + context_ptr->get_param(MS_CTX_DEVICE_ID), rank_id_str, full_path, + mode == kGraphMode ? hccl::HcclMode::kGraph : hccl::HcclMode::kPynative); free(full_path); if (!ret) { MS_LOG(ERROR) << "Hcom init failed."; diff --git a/mindspore/ccsrc/runtime/hardware/device_context_manager.cc b/mindspore/ccsrc/runtime/hardware/device_context_manager.cc index 1c6b9d7fef5..b05a85108fa 100644 --- a/mindspore/ccsrc/runtime/hardware/device_context_manager.cc +++ b/mindspore/ccsrc/runtime/hardware/device_context_manager.cc @@ -48,8 +48,8 @@ DeviceContext *DeviceContextManager::GetOrCreateDeviceContext(const DeviceContex MS_EXCEPTION_IF_NULL(device_context); device_contexts_[device_context_key_str] = device_context; } else { - MS_LOG(EXCEPTION) << "There is no device context creator for " << device_context_key.device_name_ - << " with device id " << device_context_key.device_id_; + MS_LOG(EXCEPTION) << "Create device context failed, please make sure target device:" + << device_context_key.device_name_ << " is available."; } return device_context.get(); } diff --git a/mindspore/ccsrc/runtime/hccl_adapter/hccl_adapter.cc b/mindspore/ccsrc/runtime/hccl_adapter/hccl_adapter.cc index e82d34070d1..5253631de3b 100644 --- a/mindspore/ccsrc/runtime/hccl_adapter/hccl_adapter.cc +++ b/mindspore/ccsrc/runtime/hccl_adapter/hccl_adapter.cc @@ -40,6 +40,17 @@ static constexpr const char *kHcclAlgoOption = "HCCL_algorithm"; return HcclResult::HCCL_E_RESERVED; \ } +#define CHECK_EXCUTION_MODE() \ + do { \ + auto hccl_mode = GetCurrentHcclMode(); \ + if (hccl_mode != hccl_mode_) { \ + MS_LOG(EXCEPTION) << "HCCL is initialized in " << GetHcclModeString(hccl_mode_) \ + << " but current execution mode is " << GetHcclModeString(hccl_mode) \ + << ". Please set the execution mode before HCCL init(), and then do not " \ + "change it in the subsequent script"; \ + } \ + } while (0) + static std::map GenHcclOptions(uint32_t device_id, std::string_view rank_id, std::string_view rank_file) { auto env_deploy_mode = mindspore::common::GetEnv(kHcclDeployModeEnv); @@ -134,6 +145,28 @@ void HcclAdapter::FinalizePlugin() { plugin_handle_ = nullptr; } +HcclMode HcclAdapter::GetCurrentHcclMode() const { + auto context = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context); + bool is_graph_mode = context->get_param(MS_CTX_EXECUTION_MODE) == kGraphMode; + bool is_task_sink = context->get_param(MS_CTX_ENABLE_TASK_SINK); + if (!is_graph_mode) { + return HcclMode::kPynative; + } else if (is_task_sink) { + return HcclMode::kGraph; + } else { + return HcclMode::kKernelByKernel; + } +} + +std::string HcclAdapter::GetHcclModeString(HcclMode hccl_mode) { + static std::map kHcclModeString = { + {HcclMode::kGraph, "GRAPH_MODE"}, + {HcclMode::kPynative, "PYNATIVE_MODE"}, + {HcclMode::kKernelByKernel, "GRAPH_MODE disable TASK_SINK"}}; + return kHcclModeString.at(hccl_mode); +} + bool HcclAdapter::InitHccl() { MS_LOG(INFO) << "Start init hccl adapter."; std::lock_guard lock(init_mutex_); @@ -143,21 +176,23 @@ bool HcclAdapter::InitHccl() { } InitPlugin(); init_flag_ = true; + hccl_mode_ = HcclMode::kKernelByKernel; MS_LOG(INFO) << "Init hccl adapter success."; return true; } bool HcclAdapter::InitHccl(uint32_t device_id, std::string_view rank_id, std::string_view rank_file, - bool is_graph_mode) { - MS_LOG(INFO) << "Start init hccl adapter for " << (is_graph_mode ? "graph mode." : "pynative mode."); + HcclMode hccl_mode) { + MS_LOG(INFO) << "Start init hccl adapter for " << GetHcclModeString(hccl_mode); std::lock_guard lock(init_mutex_); if (init_flag_) { MS_LOG(INFO) << "Hccl has been inited, skip."; return true; } - is_graph_mode_ = is_graph_mode; + + hccl_mode_ = hccl_mode; InitPlugin(); - if (is_graph_mode_) { + if (hccl_mode_ == HcclMode::kGraph) { bool ret = InitKernelInfoStore(device_id, rank_id, rank_file); if (!ret) { return false; @@ -181,13 +216,13 @@ bool HcclAdapter::InitHccl(uint32_t device_id, std::string_view rank_id, std::st bool HcclAdapter::FinalizeHccl() { std::lock_guard lock(init_mutex_); - MS_LOG(INFO) << "Start destroy hccl adapter for " << (is_graph_mode_ ? "graph mode." : "pynative mode."); + MS_LOG(INFO) << "Start destroy hccl adapter for " << GetHcclModeString(hccl_mode_); if (!init_flag_) { MS_LOG(INFO) << "Hccl has never been inited, skip."; return true; } - if (is_graph_mode_) { + if (hccl_mode_ == HcclMode::kGraph) { (void)FinalizeHcclExec(); (void)FinalizeKernelInfoStore(); } else { @@ -272,12 +307,14 @@ std::string HcclAdapter::GetHcclType(const AnfNodePtr &node) { HcclResult HcclAdapter::HcclBroadcast(void *buf, uint64_t count, HcclDataType dataType, uint32_t root, aclrtStream stream) const { + CHECK_EXCUTION_MODE(); CHECK_SYMBOL_NULL(launch_hccl_broadcast_); return launch_hccl_broadcast_(buf, count, dataType, root, hccl_comm_, stream); } HcclResult HcclAdapter::HcclAllReduce(void *send_buf, void *recv_buf, uint64_t count, HcclDataType dataType, HcclReduceOp op, aclrtStream stream, const std::string &group) const { + CHECK_EXCUTION_MODE(); CHECK_SYMBOL_NULL(launch_hccl_all_reduce_); auto hccl_comm = GetHcomm(group); MS_EXCEPTION_IF_NULL(hccl_comm); @@ -286,6 +323,7 @@ HcclResult HcclAdapter::HcclAllReduce(void *send_buf, void *recv_buf, uint64_t c HcclResult HcclAdapter::HcclReduceScatter(void *send_buf, void *recv_buf, uint64_t count, HcclDataType dataType, HcclReduceOp op, aclrtStream stream, const std::string &group) const { + CHECK_EXCUTION_MODE(); CHECK_SYMBOL_NULL(launch_hccl_reduce_scatter_); auto hccl_comm = GetHcomm(group); MS_EXCEPTION_IF_NULL(hccl_comm); @@ -294,6 +332,7 @@ HcclResult HcclAdapter::HcclReduceScatter(void *send_buf, void *recv_buf, uint64 HcclResult HcclAdapter::HcclAllGather(void *send_buf, void *recv_buf, uint64_t count, HcclDataType dataType, aclrtStream stream, const std::string &group) const { + CHECK_EXCUTION_MODE(); CHECK_SYMBOL_NULL(launch_hccl_all_gather_); auto hccl_comm = GetHcomm(group); MS_EXCEPTION_IF_NULL(hccl_comm); @@ -302,6 +341,7 @@ HcclResult HcclAdapter::HcclAllGather(void *send_buf, void *recv_buf, uint64_t c HcclResult HcclAdapter::HcclSend(void *send_buf, uint64_t count, HcclDataType dataType, uint32_t destRank, aclrtStream stream, const std::string &group) const { + CHECK_EXCUTION_MODE(); CHECK_SYMBOL_NULL(launch_hccl_send_); auto hccl_comm = GetHcomm(group); MS_EXCEPTION_IF_NULL(hccl_comm); @@ -310,6 +350,7 @@ HcclResult HcclAdapter::HcclSend(void *send_buf, uint64_t count, HcclDataType da HcclResult HcclAdapter::HcclRecv(void *recv_buf, uint64_t count, HcclDataType dataType, uint32_t srcRank, aclrtStream stream, const std::string &group) const { + CHECK_EXCUTION_MODE(); CHECK_SYMBOL_NULL(launch_hccl_recv_); auto hccl_comm = GetHcomm(group); MS_EXCEPTION_IF_NULL(hccl_comm); @@ -432,6 +473,7 @@ bool HcclAdapter::FinalizeHcclComm() { } HcclResult HcclAdapter::HcclCreateGroup(const std::string &group, uint32_t rank_num, uint32_t *rank_ids) const { + CHECK_EXCUTION_MODE(); CHECK_SYMBOL_NULL(hccl_create_group_); return hccl_create_group_(group.c_str(), rank_num, rank_ids); } @@ -442,21 +484,25 @@ HcclResult HcclAdapter::HcclDestroyGroup(const std::string &group) const { } HcclResult HcclAdapter::HcclGetRankId(uint32_t *rank_id) const { + CHECK_EXCUTION_MODE(); CHECK_SYMBOL_NULL(single_op_hccl_get_rank_id_); return single_op_hccl_get_rank_id_(hccl_comm_, rank_id); } HcclResult HcclAdapter::HcclGetRankSize(uint32_t *rank_size) const { + CHECK_EXCUTION_MODE(); CHECK_SYMBOL_NULL(single_op_hccl_get_rank_size_); return single_op_hccl_get_rank_size_(hccl_comm_, rank_size); } HcclResult HcclAdapter::HcclGetRankId(const std::string &group, uint32_t *rank_id) const { + CHECK_EXCUTION_MODE(); CHECK_SYMBOL_NULL(hccl_get_rank_id_); return hccl_get_rank_id_(group.c_str(), rank_id); } HcclResult HcclAdapter::HcclGetRankSize(const std::string &group, uint32_t *rank_size) const { + CHECK_EXCUTION_MODE(); CHECK_SYMBOL_NULL(hccl_get_rank_size_); return hccl_get_rank_size_(group.c_str(), rank_size); } @@ -490,11 +536,13 @@ bool HcclAdapter::FinalizeHcclExec() { } HcclResult HcclAdapter::HcclExecEnqueueOp(const ::HcomOperation &op_info, const HExecCallBack &callback) const { + CHECK_EXCUTION_MODE(); CHECK_SYMBOL_NULL(hccl_exec_enqueue_op_); return hccl_exec_enqueue_op_(op_info, callback); } HcclResult HcclAdapter::HcclExecAllToAllv(const ::HcomAllToAllVParams ¶ms, const HExecCallBack &callback) const { + CHECK_EXCUTION_MODE(); CHECK_SYMBOL_NULL(hccl_exec_enqueue_all_to_all_v_); return hccl_exec_enqueue_all_to_all_v_(params, callback); } diff --git a/mindspore/ccsrc/runtime/hccl_adapter/hccl_adapter.h b/mindspore/ccsrc/runtime/hccl_adapter/hccl_adapter.h index 3b0225f1dc5..85451c281cd 100644 --- a/mindspore/ccsrc/runtime/hccl_adapter/hccl_adapter.h +++ b/mindspore/ccsrc/runtime/hccl_adapter/hccl_adapter.h @@ -39,12 +39,14 @@ struct HcclTaskInfo { int64_t stream_num; }; +enum HcclMode { kGraph, kPynative, kKernelByKernel }; + class HcclAdapter { public: static HcclAdapter &GetInstance(); // common - bool InitHccl(uint32_t device_id, std::string_view rank_id, std::string_view rank_file, bool is_graph_mode); + bool InitHccl(uint32_t device_id, std::string_view rank_id, std::string_view rank_file, HcclMode hccl_mode); bool InitHccl(); bool FinalizeHccl(); const bool Inited() const { return init_flag_; } @@ -103,6 +105,9 @@ class HcclAdapter { bool InitHcclExec(); bool FinalizeHcclExec(); + HcclMode GetCurrentHcclMode() const; + static std::string GetHcclModeString(HcclMode hccl_mode); + void *plugin_handle_ = nullptr; InitHcomGraphAdapterFunObj init_hcom_graph_adapter_ = nullptr; @@ -137,7 +142,7 @@ class HcclAdapter { std::shared_ptr<::ge::OpsKernelBuilder> ops_kernel_builder_ = nullptr; bool init_flag_ = false; - bool is_graph_mode_ = false; + HcclMode hccl_mode_ = HcclMode::kGraph; std::mutex init_mutex_; }; } // namespace mindspore::hccl diff --git a/tests/ut/cpp/stub/ge/ge_task_launch_stub.cc b/tests/ut/cpp/stub/ge/ge_task_launch_stub.cc index 9dcd67640c1..0ea58ce3b07 100644 --- a/tests/ut/cpp/stub/ge/ge_task_launch_stub.cc +++ b/tests/ut/cpp/stub/ge/ge_task_launch_stub.cc @@ -23,7 +23,7 @@ HcclAdapter &HcclAdapter::GetInstance() { return instance; } bool HcclAdapter::InitHccl() { return true; } -bool HcclAdapter::InitHccl(uint32_t, std::string_view, std::string_view, bool) { return true; } +bool HcclAdapter::InitHccl(uint32_t, std::string_view, std::string_view, HcclMode) { return true; } bool HcclAdapter::FinalizeHccl() { return true; } HcclResult HcclAdapter::HcclCreateGroup(const std::string &, uint32_t, uint32_t *) const { return HCCL_SUCCESS; } HcclResult HcclAdapter::HcclDestroyGroup(const std::string &) const { return HCCL_SUCCESS; }