From 00b309d10f62a39281d441d077a387fd795e00e2 Mon Sep 17 00:00:00 2001 From: zhangzhaoju Date: Tue, 8 Jun 2021 14:35:38 +0800 Subject: [PATCH] solve alexnet coredump problem --- .../minddata/dataset/engine/tdt/tdt_handle.cc | 16 +++++++++------- .../minddata/dataset/engine/tdt/tdt_handle.h | 5 +++-- .../minddata/dataset/engine/tdt/tdt_plugin.cc | 2 +- .../minddata/dataset/engine/tdt/tdt_plugin.h | 1 + mindspore/core/gvar/logging_level.cc | 2 +- mindspore/core/utils/log_adapter.h | 5 +++-- mindspore/core/utils/ms_context.cc | 7 +++---- 7 files changed, 21 insertions(+), 17 deletions(-) diff --git a/mindspore/ccsrc/minddata/dataset/engine/tdt/tdt_handle.cc b/mindspore/ccsrc/minddata/dataset/engine/tdt/tdt_handle.cc index f5233877353..d6577584990 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/tdt/tdt_handle.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/tdt/tdt_handle.cc @@ -16,26 +16,28 @@ #include "minddata/dataset/engine/tdt/tdt_handle.h" namespace mindspore { -extern std::set acl_handle_set; +extern std::map acl_handle_map; namespace dataset { - -void TdtHandle::AddHandle(acltdtChannelHandle **handle) { +void TdtHandle::AddHandle(acltdtChannelHandle **handle, std::thread *use_thread) { if (*handle != nullptr) { - acl_handle_set.insert(reinterpret_cast(handle)); + acl_handle_map.insert({reinterpret_cast(handle), use_thread}); } } void TdtHandle::DelHandle(acltdtChannelHandle **handle) { void **void_handle = reinterpret_cast(handle); - acl_handle_set.erase(void_handle); + acl_handle_map.erase(void_handle); } bool TdtHandle::DestroyHandle() { bool destroy_all = true; - for (auto it = acl_handle_set.begin(); it != acl_handle_set.end(); it++) { - acltdtChannelHandle **handle = reinterpret_cast(*it); + for (auto &item : acl_handle_map) { + acltdtChannelHandle **handle = reinterpret_cast(item.first); if (*handle != nullptr) { acltdtStopChannel(*handle); + if (item.second != nullptr && item.second->joinable()) { + item.second->join(); + } if (acltdtDestroyChannel(*handle) != ACL_SUCCESS) { destroy_all = false; } else { diff --git a/mindspore/ccsrc/minddata/dataset/engine/tdt/tdt_handle.h b/mindspore/ccsrc/minddata/dataset/engine/tdt/tdt_handle.h index 2477f2c48a6..c0eb7759ba0 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/tdt/tdt_handle.h +++ b/mindspore/ccsrc/minddata/dataset/engine/tdt/tdt_handle.h @@ -17,14 +17,15 @@ #define MINDSPORE_CCSRC_MINDDATA_DATASET_ENGINE_TDT_TDT_HANDLE_H_ #include -#include +#include +#include #include "acl/acl_tdt.h" namespace mindspore { namespace dataset { class TdtHandle { public: - static void AddHandle(acltdtChannelHandle **handle); + static void AddHandle(acltdtChannelHandle **handle, std::thread *use_thread); static bool DestroyHandle(); diff --git a/mindspore/ccsrc/minddata/dataset/engine/tdt/tdt_plugin.cc b/mindspore/ccsrc/minddata/dataset/engine/tdt/tdt_plugin.cc index a6135c3ff3b..55587d551e8 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/tdt/tdt_plugin.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/tdt/tdt_plugin.cc @@ -29,7 +29,7 @@ TdtPlugin::TdtPlugin(const std::string &channel_name, int32_t device_id) { if (acl_handle_ == nullptr) { MS_LOG(ERROR) << "Failed to create channel for tdt queue."; } - TdtHandle::AddHandle(&acl_handle_); + TdtHandle::AddHandle(&acl_handle_, nullptr); } TdtPlugin::~TdtPlugin() { diff --git a/mindspore/ccsrc/minddata/dataset/engine/tdt/tdt_plugin.h b/mindspore/ccsrc/minddata/dataset/engine/tdt/tdt_plugin.h index 0d2202d51bc..69445a335c2 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/tdt/tdt_plugin.h +++ b/mindspore/ccsrc/minddata/dataset/engine/tdt/tdt_plugin.h @@ -22,6 +22,7 @@ #include #include #include +#include #include "acl/acl_tdt.h" #include "minddata/dataset/engine/tdt/tdt_handle.h" diff --git a/mindspore/core/gvar/logging_level.cc b/mindspore/core/gvar/logging_level.cc index 5c435454a7d..217736b9263 100644 --- a/mindspore/core/gvar/logging_level.cc +++ b/mindspore/core/gvar/logging_level.cc @@ -17,7 +17,7 @@ #include "utils/log_adapter.h" namespace mindspore { -std::set acl_handle_set = std::set(); +std::map acl_handle_map; // set default log level to WARNING for all sub modules int g_ms_submodule_log_levels[NUM_SUBMODUES] = {WARNING}; } // namespace mindspore diff --git a/mindspore/core/utils/log_adapter.h b/mindspore/core/utils/log_adapter.h index 7e4d5907a9f..8d48211145c 100644 --- a/mindspore/core/utils/log_adapter.h +++ b/mindspore/core/utils/log_adapter.h @@ -22,7 +22,8 @@ #include #include #include -#include +#include +#include #include #include "utils/overload.h" #include "./securec.h" @@ -42,7 +43,7 @@ static constexpr size_t GetRelPathPos() noexcept { } namespace mindspore { -extern std::set acl_handle_set __attribute__((visibility("default"))); +extern std::map acl_handle_map __attribute__((visibility("default"))); #define FILE_NAME \ (sizeof(__FILE__) > GetRelPathPos() ? static_cast(__FILE__) + GetRelPathPos() \ : static_cast(__FILE__)) diff --git a/mindspore/core/utils/ms_context.cc b/mindspore/core/utils/ms_context.cc index f0a1f787eeb..d109c6fbd96 100644 --- a/mindspore/core/utils/ms_context.cc +++ b/mindspore/core/utils/ms_context.cc @@ -116,14 +116,13 @@ void MsContext::CreateTensorPrintThread(PrintThreadCrt ctr) { std::string kReceivePrefix = "TF_RECEIVE_"; std::string channel_name = "_npu_log"; acl_handle_ = acltdtCreateChannel(device_id, (kReceivePrefix + channel_name).c_str()); - if (acl_handle_ != nullptr) { - MS_LOG(INFO) << "Success to create acltdt handle, tsd reference = " << get_param(MS_CTX_TSD_REF) << "."; - TdtHandle::AddHandle(&acl_handle_); - } else { + if (acl_handle_ == nullptr) { MS_LOG(EXCEPTION) << "Get acltdt handle failed"; } + MS_LOG(INFO) << "Success to create acltdt handle, tsd reference = " << get_param(MS_CTX_TSD_REF) << "."; std::string print_file_path = get_param(MS_CTX_PRINT_FILE_PATH); acl_tdt_print_ = ctr(print_file_path, acl_handle_); + TdtHandle::AddHandle(&acl_handle_, &acl_tdt_print_); } static void JoinAclPrintThread(std::thread *thread) {