diff --git a/mindspore/ccsrc/backend/session/ascend_session.cc b/mindspore/ccsrc/backend/session/ascend_session.cc index 2060cf2798d..2ab0b1b3337 100644 --- a/mindspore/ccsrc/backend/session/ascend_session.cc +++ b/mindspore/ccsrc/backend/session/ascend_session.cc @@ -1213,7 +1213,7 @@ void AscendSession::SelectKernel(const KernelGraph &kernel_graph) const { void DumpInit(uint32_t device_id) { auto &json_parser = DumpJsonParser::GetInstance(); json_parser.Parse(); - json_parser.CopyJsonToDir(device_id); + json_parser.CopyDumpJsonToDir(device_id); json_parser.CopyHcclJsonToDir(device_id); json_parser.CopyMSCfgJsonToDir(device_id); if (json_parser.async_dump_enabled()) { @@ -1555,7 +1555,7 @@ void AscendSession::Execute(const std::shared_ptr &kernel_graph, bo void AscendSession::DumpSetup(const std::shared_ptr &kernel_graph) const { MS_LOG(DEBUG) << "Start!"; MS_EXCEPTION_IF_NULL(kernel_graph); - E2eDump::DumpSetup(kernel_graph.get(), rank_id_); + E2eDump::DumpSetup(kernel_graph.get()); MS_LOG(DEBUG) << "Finish!"; } diff --git a/mindspore/ccsrc/backend/session/gpu_session.cc b/mindspore/ccsrc/backend/session/gpu_session.cc index b6f7bc06332..04b3a1e3401 100644 --- a/mindspore/ccsrc/backend/session/gpu_session.cc +++ b/mindspore/ccsrc/backend/session/gpu_session.cc @@ -126,7 +126,7 @@ void GPUSession::Init(uint32_t device_id) { #ifndef ENABLE_SECURITY auto &json_parser = DumpJsonParser::GetInstance(); // Dump json config file if dump is enabled - json_parser.CopyJsonToDir(rank_id_); + json_parser.CopyDumpJsonToDir(rank_id_); json_parser.CopyMSCfgJsonToDir(rank_id_); #endif MS_LOG(INFO) << "Set device id " << device_id << " for gpu session."; @@ -705,7 +705,7 @@ void GPUSession::RunOpImpl(const GraphInfo &graph_info, OpRunInfo *op_run_info, void GPUSession::DumpSetup(const std::shared_ptr &kernel_graph) const { MS_LOG(INFO) << "Start!"; MS_EXCEPTION_IF_NULL(kernel_graph); - E2eDump::DumpSetup(kernel_graph.get(), rank_id_); + E2eDump::DumpSetup(kernel_graph.get()); MS_LOG(INFO) << "Finish!"; } diff --git a/mindspore/ccsrc/debug/CMakeLists.txt b/mindspore/ccsrc/debug/CMakeLists.txt index b405b5093fb..a134d2650dd 100644 --- a/mindspore/ccsrc/debug/CMakeLists.txt +++ b/mindspore/ccsrc/debug/CMakeLists.txt @@ -14,8 +14,8 @@ set(_DEBUG_SRC_LIST set(_OFFLINE_SRC_LIST "${CMAKE_CURRENT_SOURCE_DIR}/debug_services.cc" "${CMAKE_CURRENT_SOURCE_DIR}/debugger/tensor_summary.cc" - "${CMAKE_CURRENT_SOURCE_DIR}/debugger/offline_debug/offline_logger.cc" "${CMAKE_CURRENT_SOURCE_DIR}/debugger/offline_debug/dbg_services.cc" + "${CMAKE_SOURCE_DIR}/mindspore/core/utils/log_adapter.cc" "${CMAKE_CURRENT_SOURCE_DIR}/debugger/offline_debug/mi_pybind_register.cc" ) @@ -66,6 +66,7 @@ if(ENABLE_DEBUGGER) set_property(SOURCE ${_OFFLINE_SRC_LIST} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_OFFLINE_DEBUG) add_library(_mindspore_offline_debug SHARED ${_OFFLINE_SRC_LIST}) + target_link_libraries(_mindspore_offline_debug PRIVATE mindspore::glog mindspore_gvar) set_target_properties(_mindspore_offline_debug PROPERTIES PREFIX "${PYTHON_MODULE_PREFIX}" SUFFIX "${PYTHON_MODULE_EXTENSION}" diff --git a/mindspore/ccsrc/debug/data_dump/dump_json_parser.cc b/mindspore/ccsrc/debug/data_dump/dump_json_parser.cc index 422796f63e6..407f76f221c 100644 --- a/mindspore/ccsrc/debug/data_dump/dump_json_parser.cc +++ b/mindspore/ccsrc/debug/data_dump/dump_json_parser.cc @@ -134,7 +134,7 @@ void WriteJsonFile(const std::string &file_path, const std::ifstream &json_file) ChangeFileMode(file_path, S_IRUSR); } -void DumpJsonParser::CopyJsonToDir(uint32_t rank_id) { +void DumpJsonParser::CopyDumpJsonToDir(uint32_t rank_id) { this->Parse(); if (!IsDumpEnabled()) { return; @@ -148,7 +148,7 @@ void DumpJsonParser::CopyJsonToDir(uint32_t rank_id) { auto realpath = Common::CreatePrefixPath(path_ + "/rank_" + std::to_string(rank_id) + "/.dump_metadata/data_dump.json"); if (!realpath.has_value()) { - MS_LOG(ERROR) << "Get real path failed in CopyJsonDir."; + MS_LOG(ERROR) << "Get real path failed in CopyDumpJsonToDir."; } else { WriteJsonFile(realpath.value(), json_file); } @@ -374,50 +374,49 @@ void DumpJsonParser::ParseIteration(const nlohmann::json &content) { } } +bool IsIterInRange(uint32_t iteration, const std::string &range) { + if (range.empty()) { + return false; + } + const std::string dash = "-"; + std::size_t range_idx = range.find(dash); + // no dash in range, compare the value directly + if (range_idx == std::string::npos) { + return iteration == std::stoul(range); + } + // make sure there is only one dash in range + if (range.find(dash, range_idx + 1) != std::string::npos) { + return false; + } + auto low_range_str = range.substr(0, range_idx); + auto high_range_str = range.substr(range_idx + 1); + if (low_range_str.empty() || high_range_str.empty()) { + return false; + } + uint32_t low_range = std::stoul(low_range_str); + uint32_t high_range = std::stoul(high_range_str); + return (low_range <= iteration) && (iteration <= high_range); +} + bool DumpJsonParser::IsDumpIter(uint32_t iteration) const { // bool DumpJsonParser::IsDumpIter(uint32_t iteration) --> checks if iteration should be dumped or not. if (iteration_ == "all") { return true; } const std::string vertical_bar = "|"; - const std::string dash = "-"; - int start = 0; - int end = iteration_.find(vertical_bar); - while (end != -1) { - std::string temp = iteration_.substr(IntToSize(start), IntToSize(end - start)); - int range_idx = temp.find(dash); - if (range_idx != -1) { - uint32_t low_range = static_cast(std::stoul(temp.substr(0, IntToSize(range_idx)))); - uint32_t high_range = static_cast(std::stoul(temp.substr(IntToSize(range_idx + 1), -1))); - if ((low_range <= iteration) && (iteration <= high_range)) { - return true; - } - } else if (iteration == std::stoul(temp)) { + std::size_t start = 0; + std::size_t end = iteration_.find(vertical_bar); + while (end != std::string::npos) { + std::string temp = iteration_.substr(start, end - start); + auto found = IsIterInRange(iteration, temp); + if (found) { return true; } start = end + 1; - end = static_cast(iteration_.find(vertical_bar, start)); + end = iteration_.find(vertical_bar, start); } - std::string temp = iteration_.substr(IntToSize(start), IntToSize(end - start)); - int range_idx = temp.find(dash); - if (range_idx != -1) { - uint32_t low_range = static_cast(std::stoul(temp.substr(0, IntToSize(range_idx)))); - uint32_t high_range = static_cast(std::stoul(temp.substr(IntToSize(range_idx + 1), -1))); - if ((low_range <= iteration) && (iteration <= high_range)) { - return true; - } - } else if (iteration == std::stoul(temp)) { - return true; - } - return false; -} - -bool DumpJsonParser::IsSingleIter() { - // bool DumpJsonParser::IsSingleIter() --> checks if iteration in json dump file is single or not. - if (iteration_ != "all" && iteration_.find("-") == std::string::npos && iteration_.find("|") == std::string::npos) { - return true; - } - return false; + std::string temp = iteration_.substr(start); + return IsIterInRange(iteration, temp); } void DumpJsonParser::ParseInputOutput(const nlohmann::json &content) { diff --git a/mindspore/ccsrc/debug/data_dump/dump_json_parser.h b/mindspore/ccsrc/debug/data_dump/dump_json_parser.h index 76d89983165..62450b03c60 100644 --- a/mindspore/ccsrc/debug/data_dump/dump_json_parser.h +++ b/mindspore/ccsrc/debug/data_dump/dump_json_parser.h @@ -36,15 +36,14 @@ class DumpJsonParser { void Parse(); static bool DumpToFile(const std::string &filename, const void *data, size_t len, const ShapeVector &shape, TypeId type); - void CopyJsonToDir(uint32_t device_id); - void CopyHcclJsonToDir(uint32_t device_id); - void CopyMSCfgJsonToDir(uint32_t device_id); + void CopyDumpJsonToDir(uint32_t rank_id); + void CopyHcclJsonToDir(uint32_t rank_id); + void CopyMSCfgJsonToDir(uint32_t rank_id); bool NeedDump(const std::string &op_full_name) const; void MatchKernel(const std::string &kernel_name); void PrintUnusedKernel(); bool IsDumpIter(uint32_t iteration) const; bool DumpAllIter(); - bool IsSingleIter(); bool async_dump_enabled() const { return async_dump_enabled_; } bool e2e_dump_enabled() const { return e2e_dump_enabled_; } diff --git a/mindspore/ccsrc/debug/data_dump/e2e_dump.cc b/mindspore/ccsrc/debug/data_dump/e2e_dump.cc index 5be11b3c7ce..68d5af02bc0 100644 --- a/mindspore/ccsrc/debug/data_dump/e2e_dump.cc +++ b/mindspore/ccsrc/debug/data_dump/e2e_dump.cc @@ -299,7 +299,7 @@ void E2eDump::UpdateIterDumpSetup(const session::KernelGraph *graph, bool sink_m } } -void E2eDump::DumpSetup(const session::KernelGraph *graph, uint32_t rank_id) { +void E2eDump::DumpSetup(const session::KernelGraph *graph) { auto &dump_json_parser = DumpJsonParser::GetInstance(); bool sink_mode = (ConfigManager::GetInstance().dataset_mode() || E2eDump::isDatasetGraph(graph)); diff --git a/mindspore/ccsrc/debug/data_dump/e2e_dump.h b/mindspore/ccsrc/debug/data_dump/e2e_dump.h index 05a1ae56228..b99a6b2e284 100644 --- a/mindspore/ccsrc/debug/data_dump/e2e_dump.h +++ b/mindspore/ccsrc/debug/data_dump/e2e_dump.h @@ -35,7 +35,7 @@ class E2eDump { public: E2eDump() = default; ~E2eDump() = default; - static void DumpSetup(const session::KernelGraph *graph, uint32_t rank_id); + static void DumpSetup(const session::KernelGraph *graph); static void UpdateIterGPUDump(); diff --git a/mindspore/ccsrc/debug/debug_services.cc b/mindspore/ccsrc/debug/debug_services.cc index 308dc0b5e6d..1e1aba714fe 100644 --- a/mindspore/ccsrc/debug/debug_services.cc +++ b/mindspore/ccsrc/debug/debug_services.cc @@ -85,7 +85,7 @@ void DebugServices::RemoveWatchpoint(unsigned int id) { } std::unique_ptr GetSummaryPtr(const std::shared_ptr &tensor, - void *const previous_tensor_ptr, uint32_t num_elements, + const void *const previous_tensor_ptr, uint32_t num_elements, uint32_t prev_num_elements, int tensor_dtype) { switch (tensor_dtype) { case DbgDataType::DT_UINT8: { @@ -170,9 +170,9 @@ DebugServices::TensorStat DebugServices::GetTensorStatistics(const std::shared_p return tensor_stat_data; } #ifdef OFFLINE_DBG_MODE -void *DebugServices::GetPrevTensor(const std::shared_ptr &tensor, bool previous_iter_tensor_needed, - uint32_t *prev_num_elements) { - void *previous_tensor_ptr = nullptr; +const void *DebugServices::GetPrevTensor(const std::shared_ptr &tensor, bool previous_iter_tensor_needed, + uint32_t *prev_num_elements) { + const void *previous_tensor_ptr = nullptr; std::shared_ptr tensor_prev; if (previous_iter_tensor_needed && tensor->GetIteration() >= 1) { // read data in offline mode @@ -369,7 +369,7 @@ void DebugServices::CheckWatchpointsForTensor( int tensor_dtype = tensor->GetType(); uint32_t num_elements = tensor->GetNumElements(); uint32_t prev_num_elements = 0; - void *previous_tensor_ptr = nullptr; + const void *previous_tensor_ptr = nullptr; #ifdef OFFLINE_DBG_MODE previous_tensor_ptr = GetPrevTensor(tensor, previous_iter_tensor_needed, &prev_num_elements); #else @@ -949,15 +949,36 @@ void DebugServices::ReadDumpedTensor(std::vector backend_name, std: } } +void DebugServices::ReadFileAndAddToTensor(const bool found, const std::vector &matched_paths, + const std::string &backend_name, const unsigned int device_id, + const unsigned int root_graph_id, const bool &is_output, size_t slot, + bool *no_mem_to_read, unsigned int iteration, + std::vector> *result_list) { + std::string time_stamp; + std::string type_name = ""; + uint64_t data_size = 0; + std::vector shape; + std::vector *buffer = nullptr; + if (found) { + std::string result_path = GetNewestFilePath(matched_paths); + time_stamp = GetTimeStampStr(result_path); + std::string key_name_in_cache = backend_name + ":" + std::to_string(device_id) + ":" + + std::to_string(root_graph_id) + ":" + std::to_string(is_output) + ":" + + std::to_string(slot); + ReadTensorFromNpy(key_name_in_cache, result_path, &type_name, &data_size, &shape, &buffer, no_mem_to_read); + AddToTensorData(backend_name, time_stamp, slot, iteration, device_id, root_graph_id, is_output, data_size, + type_name, shape, buffer, result_list); + } else { + AddToTensorData(backend_name, time_stamp, slot, iteration, device_id, root_graph_id, is_output, 0, type_name, shape, + buffer, result_list); + MS_LOG(INFO) << "Target tensor has not been found."; + } +} + void DebugServices::ReadDumpedTensorSync(const std::string &prefix_dump_file_name, const std::string &specific_dump_dir, - const std::string &backend_name, size_t slot, unsigned int device_id, + const std::string &backend_name, size_t slot, const unsigned int device_id, unsigned int iteration, unsigned int root_graph_id, const bool &is_output, std::vector> *result_list, bool *no_mem_to_read) { - std::vector *buffer = nullptr; - std::string type_name = ""; - std::vector shape; - uint64_t data_size = 0; - std::string time_stamp; std::string abspath = RealPath(specific_dump_dir); DIR *d = opendir(abspath.c_str()); bool found_file = false; @@ -984,22 +1005,8 @@ void DebugServices::ReadDumpedTensorSync(const std::string &prefix_dump_file_nam } (void)closedir(d); } - - if (found_file) { - shape.clear(); - std::string result_path = GetNewestFilePath(matched_paths); - time_stamp = GetTimeStampStr(result_path); - std::string key_name_in_cache = backend_name + ":" + std::to_string(device_id) + ":" + - std::to_string(root_graph_id) + ":" + std::to_string(is_output) + ":" + - std::to_string(slot); - ReadTensorFromNpy(key_name_in_cache, result_path, &type_name, &data_size, &shape, &buffer, no_mem_to_read); - AddToTensorData(backend_name, time_stamp, slot, iteration, device_id, root_graph_id, is_output, data_size, - type_name, shape, buffer, result_list); - } else { - AddToTensorData(backend_name, time_stamp, slot, iteration, device_id, root_graph_id, is_output, 0, type_name, shape, - buffer, result_list); - MS_LOG(INFO) << "Target tensor has not been found."; - } + ReadFileAndAddToTensor(found_file, matched_paths, backend_name, device_id, root_graph_id, is_output, slot, + no_mem_to_read, iteration, result_list); } void DebugServices::ReadDumpedTensorAsync(const std::string &specific_dump_dir, const std::string &prefix_dump_to_check, @@ -1008,11 +1015,6 @@ void DebugServices::ReadDumpedTensorAsync(const std::string &specific_dump_dir, unsigned int root_graph_id, const bool &is_output, const std::vector &async_file_pool, std::vector> *result_list, bool *no_mem_to_read) { - std::vector *buffer = nullptr; - std::string type_name = ""; - std::vector shape; - uint64_t data_size = 0; - std::string time_stamp; bool found = false; std::vector matched_paths; // if async mode @@ -1024,22 +1026,8 @@ void DebugServices::ReadDumpedTensorAsync(const std::string &specific_dump_dir, found = true; } } - if (found) { - shape.clear(); - std::string result_path = GetNewestFilePath(matched_paths); - time_stamp = GetTimeStampStr(result_path); - std::string key_name_in_cache = backend_name + ":" + std::to_string(device_id) + ":" + - std::to_string(root_graph_id) + ":" + std::to_string(is_output) + ":" + - std::to_string(slot); - ReadTensorFromNpy(key_name_in_cache, result_path, &type_name, &data_size, &shape, &buffer, no_mem_to_read); - AddToTensorData(backend_name, time_stamp, slot, iteration, device_id, root_graph_id, is_output, data_size, - type_name, shape, buffer, result_list); - } else { - // If no npy file is found, add empty tensor data. - AddToTensorData(backend_name, time_stamp, slot, iteration, device_id, root_graph_id, is_output, 0, type_name, shape, - buffer, result_list); - MS_LOG(INFO) << "Target tensor has not been found."; - } + ReadFileAndAddToTensor(found, matched_paths, backend_name, device_id, root_graph_id, is_output, slot, no_mem_to_read, + iteration, result_list); } std::string DebugServices::GetStrippedFilename(const std::string &file_name) { @@ -1187,7 +1175,7 @@ std::string DebugServices::IterationString(unsigned int iteration) { #endif void DebugServices::ReadNodesTensors(const std::vector &name, std::vector *const ret_name, - std::vector *const data_ptr, std::vector *const data_size, + std::vector *const data_ptr, std::vector *const data_size, std::vector *const dtype, std::vector> *const shape) { std::vector>> result_list; @@ -1198,7 +1186,7 @@ void DebugServices::ReadNodesTensors(const std::vector &name, std:: continue; } (void)ret_name->emplace_back(std::get<0>(result)); - (void)data_ptr->emplace_back(reinterpret_cast(std::get<1>(result)->GetDataPtr())); + (void)data_ptr->emplace_back(reinterpret_cast(std::get<1>(result)->GetDataPtr())); (void)data_size->emplace_back(std::get<1>(result)->GetByteSize()); (void)dtype->emplace_back(std::get<1>(result)->GetType()); (void)shape->emplace_back(std::get<1>(result)->GetShape()); @@ -1351,9 +1339,6 @@ bool DebugServices::CheckOpOverflow(std::string node_name_to_find, unsigned int infile.open(file_path.c_str(), std::ios::ate | std::ios::binary | std::ios::in); if (!infile.is_open()) { MS_LOG(ERROR) << "Failed to open overflow bin file " << file_name << " Errno:" << errno; - const int kMaxFilenameLength = 128; - char err_info[kMaxFilenameLength]; - MS_LOG(ERROR) << " ErrInfo:" << strerror_r(errno, err_info, sizeof(err_info)); continue; } diff --git a/mindspore/ccsrc/debug/debug_services.h b/mindspore/ccsrc/debug/debug_services.h index 943f57823a8..3a49da33c22 100644 --- a/mindspore/ccsrc/debug/debug_services.h +++ b/mindspore/ccsrc/debug/debug_services.h @@ -22,7 +22,6 @@ #ifdef OFFLINE_DBG_MODE #include "base/float16.h" -#include "debugger/offline_debug/offline_logger.h" #endif #include @@ -330,6 +329,12 @@ class DebugServices { unsigned int device_id, unsigned int root_graph_id, std::vector> *const tensor_list); + void ReadFileAndAddToTensor(const bool found, const std::vector &matched_paths, + const std::string &backend_name, const unsigned int device_id, + const unsigned int root_graph_id, const bool &is_output, size_t slot, + bool *no_mem_to_read, unsigned int iteration, + std::vector> *result_list); + void ReadDumpedTensorSync(const std::string &prefix_dump_file_name, const std::string &specific_dump_dir, const std::string &backend_name, size_t slot, unsigned int device_id, unsigned int iteration, unsigned int root_graph_id, const bool &is_output, @@ -344,8 +349,8 @@ class DebugServices { std::vector> ReadNeededDumpedTensors(unsigned int iteration, std::vector *const async_file_pool); - void *GetPrevTensor(const std::shared_ptr &tensor, bool previous_iter_tensor_needed, - uint32_t *prev_num_elements); + const void *GetPrevTensor(const std::shared_ptr &tensor, bool previous_iter_tensor_needed, + uint32_t *prev_num_elements); void ReadTensorFromNpy(const std::string &tensor_name, const std::string &file_name, std::string *const tensor_type, std::size_t *const size, std::vector *const shape, @@ -380,7 +385,7 @@ class DebugServices { std::string IterationString(unsigned int iteration); #endif void ReadNodesTensors(const std::vector &name, std::vector *ret_name, - std::vector *data_ptr, std::vector *data_size, + std::vector *data_ptr, std::vector *data_size, std::vector *dtype, std::vector> *const shape); void SearchNodesTensors(const std::vector &name, diff --git a/mindspore/ccsrc/debug/debugger/debugger.cc b/mindspore/ccsrc/debug/debugger/debugger.cc index 0a9e518df54..3ce4860b038 100644 --- a/mindspore/ccsrc/debug/debugger/debugger.cc +++ b/mindspore/ccsrc/debug/debugger/debugger.cc @@ -407,9 +407,8 @@ void Debugger::DumpSingleNode(const CNodePtr &node, uint32_t graph_id) { void Debugger::DumpSetup(const KernelGraphPtr &kernel_graph) const { MS_LOG(INFO) << "Start!"; - uint32_t rank_id = GetRankID(); MS_EXCEPTION_IF_NULL(kernel_graph); - E2eDump::DumpSetup(kernel_graph.get(), rank_id); + E2eDump::DumpSetup(kernel_graph.get()); MS_LOG(INFO) << "Finish!"; } @@ -985,7 +984,7 @@ void Debugger::RemoveWatchpoint(const int32_t id) { debug_services_->RemoveWatch std::list Debugger::LoadTensors(const ProtoVector &tensors) const { std::vector name; std::vector ret_name; - std::vector data_ptr; + std::vector data_ptr; std::vector data_size; std::vector dtype; std::vector> shape; diff --git a/mindspore/ccsrc/debug/debugger/offline_debug/dbg_services.cc b/mindspore/ccsrc/debug/debugger/offline_debug/dbg_services.cc index 49de5afb055..6d142cca892 100644 --- a/mindspore/ccsrc/debug/debugger/offline_debug/dbg_services.cc +++ b/mindspore/ccsrc/debug/debugger/offline_debug/dbg_services.cc @@ -18,14 +18,7 @@ #include #include -DbgServices::DbgServices(bool verbose) { - DbgLogger::verbose = verbose; - std::string dbg_log_path = common::GetEnv("OFFLINE_DBG_LOG"); - if (!dbg_log_path.empty()) { - DbgLogger::verbose = true; - } - debug_services_ = std::make_shared(); -} +DbgServices::DbgServices() { debug_services_ = std::make_shared(); } DbgServices::DbgServices(const DbgServices &other) { MS_LOG(INFO) << "cpp DbgServices object is created via copy"; diff --git a/mindspore/ccsrc/debug/debugger/offline_debug/dbg_services.h b/mindspore/ccsrc/debug/debugger/offline_debug/dbg_services.h index 639baa6ae87..a5fc61442df 100644 --- a/mindspore/ccsrc/debug/debugger/offline_debug/dbg_services.h +++ b/mindspore/ccsrc/debug/debugger/offline_debug/dbg_services.h @@ -101,7 +101,7 @@ struct tensor_info_t { }; struct tensor_data_t { - tensor_data_t(char *data_ptr, uint64_t data_size, int dtype, const std::vector &shape) + tensor_data_t(const char *data_ptr, uint64_t data_size, int dtype, const std::vector &shape) : data_size(data_size), dtype(dtype), shape(shape) { if (data_ptr != nullptr) { this->data_ptr = py::bytes(data_ptr, data_size); @@ -183,7 +183,7 @@ struct TensorStatData { class DbgServices { public: - explicit DbgServices(bool verbose = false); + DbgServices(); DbgServices(const DbgServices &other); diff --git a/mindspore/ccsrc/debug/debugger/offline_debug/mi_pybind_register.cc b/mindspore/ccsrc/debug/debugger/offline_debug/mi_pybind_register.cc index db07ecff077..667374939f3 100644 --- a/mindspore/ccsrc/debug/debugger/offline_debug/mi_pybind_register.cc +++ b/mindspore/ccsrc/debug/debugger/offline_debug/mi_pybind_register.cc @@ -21,7 +21,7 @@ PYBIND11_MODULE(_mindspore_offline_debug, m) { m.doc() = "pybind11 debug services api"; (void)py::class_(m, "DbgServices") - .def(py::init()) + .def(py::init()) .def("Initialize", &DbgServices::Initialize) .def("AddWatchpoint", &DbgServices::AddWatchpoint) .def("RemoveWatchpoint", &DbgServices::RemoveWatchpoint) diff --git a/mindspore/ccsrc/debug/debugger/offline_debug/offline_logger.cc b/mindspore/ccsrc/debug/debugger/offline_debug/offline_logger.cc deleted file mode 100644 index bb6ebbe509a..00000000000 --- a/mindspore/ccsrc/debug/debugger/offline_debug/offline_logger.cc +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Copyright 2021 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "debugger/offline_debug/offline_logger.h" - -bool DbgLogger::verbose = false; diff --git a/mindspore/ccsrc/debug/debugger/offline_debug/offline_logger.h b/mindspore/ccsrc/debug/debugger/offline_debug/offline_logger.h deleted file mode 100644 index 8ca0f55bee3..00000000000 --- a/mindspore/ccsrc/debug/debugger/offline_debug/offline_logger.h +++ /dev/null @@ -1,63 +0,0 @@ -/** - * Copyright 2021 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef OFFLINE_LOGGER_H_ -#define OFFLINE_LOGGER_H_ - -#include - -#define PATH_MAX 4096 - -#define MS_LOG(level) MS_LOG_##level - -#define MS_LOG_INFO static_cast(0), !(DbgLogger::verbose) ? void(0) : DbgLogger(DbgLoggerLvl::INFO) < std::cout - -#define MS_LOG_ERROR MS_LOG_INFO - -#define MS_LOG_DEBUG MS_LOG_INFO - -#define MS_LOG_WARNING MS_LOG_INFO - -#define MS_LOG_EXCEPTION static_cast(0), DbgLogger(DbgLoggerLvl::EXCEPTION) < std::cout - -enum DbgLoggerLvl : int { DEBUG = 0, INFO, WARNING, ERROR, EXCEPTION }; - -class DbgLogger { - public: - explicit DbgLogger(DbgLoggerLvl lvl) : lvl_(lvl) {} - ~DbgLogger() = default; - void operator<(std::ostream &os) const { - char *dbg_log_path = std::getenv("OFFLINE_DBG_LOG"); - if (dbg_log_path != nullptr) { - char abspath[PATH_MAX]; - if (sizeof(dbg_log_path) > PATH_MAX || NULL == realpath(dbg_log_path, abspath)) { - std::cout << "ERROR: DbgLogger could not create real path"; - } - FILE *fp = freopen(abspath, "a", stdout); - if (fp == nullptr) { - std::cout << "ERROR: DbgLogger could not redirect all stdout to a file"; - } - } - os << std::endl; - if (lvl_ == DbgLoggerLvl::EXCEPTION) { - throw lvl_; - } - } - static bool verbose; - - private: - DbgLoggerLvl lvl_; -}; -#endif // OFFLINE_LOGGER_H_ diff --git a/mindspore/ccsrc/debug/debugger/proto_exporter.cc b/mindspore/ccsrc/debug/debugger/proto_exporter.cc index ffa995fa89f..b6cc686606f 100644 --- a/mindspore/ccsrc/debug/debugger/proto_exporter.cc +++ b/mindspore/ccsrc/debug/debugger/proto_exporter.cc @@ -167,7 +167,7 @@ void DebuggerProtoExporter::SetValueToProto(const ValuePtr &val, debugger::Value TypePtr elem_type = dyn_cast(val)->element(); type_proto->mutable_tensor_type()->set_elem_type(GetDebuggerNumberDataType(elem_type)); } else { - MS_LOG(WARNING) << "Unsupported type " << val->type_name(); + MS_LOG(INFO) << "Unsupported type " << val->type_name(); } } diff --git a/mindspore/ccsrc/debug/debugger/tensor_summary.cc b/mindspore/ccsrc/debug/debugger/tensor_summary.cc index d3fdc0c4cb4..38c5cb6d152 100644 --- a/mindspore/ccsrc/debug/debugger/tensor_summary.cc +++ b/mindspore/ccsrc/debug/debugger/tensor_summary.cc @@ -25,7 +25,6 @@ #ifdef OFFLINE_DBG_MODE #include "base/float16.h" -#include "offline_debug/offline_logger.h" #endif #ifdef ONLINE_DBG_MODE @@ -91,10 +90,10 @@ double VarianceAndMeanCalculator::GetVariance() const { double VarianceAndMeanCalculator::GetStandardDeviation() { return sqrt(GetVariance()); } template -TensorSummary::TensorSummary(void *current_tensor_ptr, void *const previous_tensor_ptr, uint32_t num_elements, - uint32_t prev_num_elements) - : current_tensor_ptr_(reinterpret_cast(current_tensor_ptr)), - prev_tensor_ptr_(reinterpret_cast(previous_tensor_ptr)), +TensorSummary::TensorSummary(const void *current_tensor_ptr, const void *const previous_tensor_ptr, + uint32_t num_elements, uint32_t prev_num_elements) + : current_tensor_ptr_(reinterpret_cast(current_tensor_ptr)), + prev_tensor_ptr_(reinterpret_cast(previous_tensor_ptr)), num_elements_(num_elements), prev_num_elements_(prev_num_elements), min_(std::numeric_limits::max()), diff --git a/mindspore/ccsrc/debug/debugger/tensor_summary.h b/mindspore/ccsrc/debug/debugger/tensor_summary.h index 0cc34e87596..6b9794d434b 100644 --- a/mindspore/ccsrc/debug/debugger/tensor_summary.h +++ b/mindspore/ccsrc/debug/debugger/tensor_summary.h @@ -111,7 +111,7 @@ class TensorSummary : public ITensorSummary { public: TensorSummary() = default; ~TensorSummary() override = default; - TensorSummary(void *, void *, uint32_t, uint32_t); + TensorSummary(const void *, const void *, uint32_t, uint32_t); void SummarizeTensor(const std::vector &) override; // returns hit, error_code, parameter_list std::tuple> IsWatchpointHit(DebugServices::watchpoint_t) override; @@ -129,8 +129,8 @@ class TensorSummary : public ITensorSummary { const int zero_count() const override { return zero_count_; } private: - T *current_tensor_ptr_; - T *prev_tensor_ptr_; + const T *current_tensor_ptr_; + const T *prev_tensor_ptr_; uint32_t num_elements_; uint32_t prev_num_elements_; double min_; diff --git a/mindspore/ccsrc/debug/tensor_data.h b/mindspore/ccsrc/debug/tensor_data.h index af5f0483053..e8cfe33503c 100644 --- a/mindspore/ccsrc/debug/tensor_data.h +++ b/mindspore/ccsrc/debug/tensor_data.h @@ -21,11 +21,9 @@ #include #include #include -#ifdef OFFLINE_DBG_MODE -#include "debugger/offline_debug/offline_logger.h" -#else -#include "ir/tensor.h" #include "mindspore/core/utils/log_adapter.h" +#ifdef ONLINE_DBG_MODE +#include "ir/tensor.h" #endif #ifdef ONLINE_DBG_MODE @@ -213,7 +211,7 @@ class TensorData { void SetSlot(size_t slot) { this->slot_ = slot; } - char *GetDataPtr() const { return this->data_ptr_; } + const char *GetDataPtr() const { return this->data_ptr_; } void SetDataPtr(char *data_ptr) { this->data_ptr_ = data_ptr; } diff --git a/mindspore/ccsrc/debug/tensor_load.h b/mindspore/ccsrc/debug/tensor_load.h index 99d914a934c..31619d474e4 100644 --- a/mindspore/ccsrc/debug/tensor_load.h +++ b/mindspore/ccsrc/debug/tensor_load.h @@ -25,9 +25,6 @@ #include #include #include -#ifdef OFFLINE_DBG_MODE -#include "debugger/offline_debug/offline_logger.h" -#endif #include "debug/tensor_data.h" #ifdef ONLINE_DBG_MODE #include "debug/data_dump/dump_json_parser.h" diff --git a/mindspore/ccsrc/runtime/hardware/cpu/cpu_device_context.cc b/mindspore/ccsrc/runtime/hardware/cpu/cpu_device_context.cc index 49fd32f4a77..74c7112f206 100644 --- a/mindspore/ccsrc/runtime/hardware/cpu/cpu_device_context.cc +++ b/mindspore/ccsrc/runtime/hardware/cpu/cpu_device_context.cc @@ -60,7 +60,7 @@ void CPUDeviceContext::Initialize() { auto rank_id = GetRankID(); auto &json_parser = DumpJsonParser::GetInstance(); json_parser.Parse(); - json_parser.CopyJsonToDir(rank_id); + json_parser.CopyDumpJsonToDir(rank_id); json_parser.CopyMSCfgJsonToDir(rank_id); #endif diff --git a/mindspore/ccsrc/runtime/hardware/gpu/gpu_device_context.cc b/mindspore/ccsrc/runtime/hardware/gpu/gpu_device_context.cc index 243a45fe78f..10904b8ac81 100644 --- a/mindspore/ccsrc/runtime/hardware/gpu/gpu_device_context.cc +++ b/mindspore/ccsrc/runtime/hardware/gpu/gpu_device_context.cc @@ -102,7 +102,7 @@ void GPUDeviceContext::Initialize() { auto rank_id = GetRankID(); auto &json_parser = DumpJsonParser::GetInstance(); json_parser.Parse(); - json_parser.CopyJsonToDir(rank_id); + json_parser.CopyDumpJsonToDir(rank_id); json_parser.CopyMSCfgJsonToDir(rank_id); #endif initialized_ = true; diff --git a/mindspore/offline_debug/dbg_services.py b/mindspore/offline_debug/dbg_services.py index 7041b926b95..c94089e8a80 100644 --- a/mindspore/offline_debug/dbg_services.py +++ b/mindspore/offline_debug/dbg_services.py @@ -22,6 +22,7 @@ from mindspore.offline_debug.mi_validators import check_init, check_initialize, check_tensor_info_init, check_tensor_data_init, check_tensor_base_data_init, check_tensor_stat_data_init,\ check_watchpoint_hit_init, check_parameter_init from mindspore.offline_debug.mi_validator_helpers import replace_minus_one +from mindspore import log as logger if not security.enable_security(): import mindspore._mindspore_offline_debug as cds @@ -40,29 +41,7 @@ def get_version(): if security.enable_security(): raise ValueError("Offline debugger is not supported in security mode. " "Please recompile mindspore without `-s on`.") - return cds.DbgServices(False).GetVersion() - - -class DbgLogger: - """ - Offline Debug Services Logger - - Args: - verbose (bool): Whether to print logs. - - Examples: - >>> from mindspore.ccsrc.debug.debugger.offline_debug import dbg_services - >>> version = dbg_services.DbgLogger(verbose=False) - """ - def __init__(self, verbose): - self.verbose = verbose - - def __call__(self, *logs): - if self.verbose: - print(logs) - - -log = DbgLogger(False) + return cds.DbgServices().GetVersion() class DbgServices: @@ -71,25 +50,21 @@ class DbgServices: Args: dump_file_path (str): Directory where the dump files are saved. - verbose (bool): Whether to print logs. Default: False. Examples: >>> from mindspore.ccsrc.debug.debugger.offline_debug import dbg_services - >>> d = dbg_services.DbgServices(dump_file_path="dump_file_path", - ... verbose=True) + >>> d = dbg_services.DbgServices(dump_file_path="dump_file_path") """ @check_init - def __init__(self, dump_file_path, verbose=False): + def __init__(self, dump_file_path): if security.enable_security(): raise ValueError("Offline debugger is not supported in security mode. " "Please recompile mindspore without `-s on`.") - log.verbose = verbose - log("in Python __init__, file path is ", dump_file_path) + logger.info("in Python __init__, file path is %s", dump_file_path) self.dump_file_path = dump_file_path - self.dbg_instance = cds.DbgServices(verbose) + self.dbg_instance = cds.DbgServices() self.version = self.dbg_instance.GetVersion() - self.verbose = verbose self.initialized = False @check_initialize @@ -109,11 +84,10 @@ class DbgServices: Examples: >>> from mindspore.ccsrc.debug.debugger.offline_debug import dbg_services - >>> d = dbg_services.DbgServices(dump_file_path="dump_file_path", - ... verbose=True) + >>> d = dbg_services.DbgServices(dump_file_path="dump_file_path") >>> d_init = d.initialize(net_name="network name", is_sync_mode=True, max_mem_usage=4096) """ - log("in Python Initialize dump_file_path ", self.dump_file_path) + logger.info("in Python Initialize dump_file_path %s", self.dump_file_path) self.initialized = True return self.dbg_instance.Initialize(net_name, self.dump_file_path, is_sync_mode, max_mem_usage) @@ -134,8 +108,7 @@ class DbgServices: Examples: >>> from mindspore.offline_debug import dbg_services - >>> d = dbg_services.DbgServices(dump_file_path="dump_file_path", - >>> verbose=True) + >>> d = dbg_services.DbgServices(dump_file_path="dump_file_path") >>> d_init = d.initialize(is_sync_mode=True) >>> d_wp = d_init.transform_check_node_list(info_name="rank_id", >>> info_param=[0], @@ -171,8 +144,7 @@ class DbgServices: Examples: >>> from mindspore.ccsrc.debug.debugger.offline_debug import dbg_services - >>> d = dbg_services.DbgServices(dump_file_path="dump_file_path", - ... verbose=True) + >>> d = dbg_services.DbgServices(dump_file_path="dump_file_path") >>> d_init = d.initialize(is_sync_mode=True) >>> d_wp = d_init.add_watchpoint(watchpoint_id=1, ... watch_condition=6, @@ -184,7 +156,7 @@ class DbgServices: ... hit=False, ... actual_value=0.0)]) """ - log("in Python AddWatchpoint") + logger.info("in Python AddWatchpoint") for node_name, node_info in check_node_list.items(): for info_name, info_param in node_info.items(): check_node_list = self.transform_check_node_list(info_name, info_param, node_name, check_node_list) @@ -207,8 +179,7 @@ class DbgServices: Examples: >>> from mindspore.ccsrc.debug.debugger.offline_debug import dbg_services - >>> d = dbg_services.DbgServices(dump_file_path="dump_file_path", - ... verbose=True) + >>> d = dbg_services.DbgServices(dump_file_path="dump_file_path") >>> d_init = d.initialize(is_sync_mode=True) >>> d_wp = d_init.add_watchpoint(watchpoint_id=1, ... watch_condition=6, @@ -221,7 +192,7 @@ class DbgServices: ... actual_value=0.0)]) >>> d_wp = d_wp.remove_watchpoint(watchpoint_id=1) """ - log("in Python Remove Watchpoint id ", watchpoint_id) + logger.info("in Python Remove Watchpoint id %d", watchpoint_id) return self.dbg_instance.RemoveWatchpoint(watchpoint_id) @check_initialize_done @@ -238,8 +209,7 @@ class DbgServices: Examples: >>> from mindspore.ccsrc.debug.debugger.offline_debug import dbg_services - >>> d = dbg_services.DbgServices(dump_file_path="dump_file_path", - ... verbose=True) + >>> d = dbg_services.DbgServices(dump_file_path="dump_file_path") >>> d_init = d.initialize(is_sync_mode=True) >>> d_wp = d_init.add_watchpoint(id=1, ... watch_condition=6, @@ -252,7 +222,7 @@ class DbgServices: ... actual_value=0.0)]) >>> watchpoints = d_wp.check_watchpoints(iteration=8) """ - log("in Python CheckWatchpoints iteration ", iteration) + logger.info("in Python CheckWatchpoints iteration %d", iteration) iteration = replace_minus_one(iteration) watchpoint_list = self.dbg_instance.CheckWatchpoints(iteration) watchpoint_hit_list = [] @@ -291,8 +261,7 @@ class DbgServices: Examples: >>> from mindspore.ccsrc.debug.debugger.offline_debug import dbg_services - >>> d = dbg_services.DbgServices(dump_file_path="dump_file_path", - ... verbose=True) + >>> d = dbg_services.DbgServices(dump_file_path="dump_file_path") >>> d_init = d.initialize(is_sync_mode=True) >>> tensor_data_list = d_init.read_tensors([dbg_services.TensorInfo(node_name="conv2.bias", ... slot=0, @@ -301,10 +270,12 @@ class DbgServices: ... root_graph_id=0, ... is_output=True)]) """ - log("in Python ReadTensors info ", info) + logger.info("in Python ReadTensors info:") + logger.info(info) info_list_inst = [] for elem in info: - log("in Python ReadTensors info ", info) + logger.info("in Python ReadTensors info:") + logger.info(info) info_list_inst.append(elem.instance) tensor_data_list = self.dbg_instance.ReadTensors(info_list_inst) tensor_data_list_ret = [] @@ -330,8 +301,7 @@ class DbgServices: Examples: >>> from mindspore.ccsrc.debug.debugger.offline_debug import dbg_services - >>> d = dbg_services.DbgServices(dump_file_path="dump_file_path", - ... verbose=True) + >>> d = dbg_services.DbgServices(dump_file_path="dump_file_path") >>> d_init = d.initialize(is_sync_mode=True) >>> tensor_base_data_list = d_init.read_tensor_base([dbg_services.TensorInfo(node_name="conv2.bias", ... slot=0, @@ -340,10 +310,12 @@ class DbgServices: ... root_graph_id=0, ... is_output=True)]) """ - log("in Python ReadTensorsBase info ", info) + logger.info("in Python ReadTensorsBase info:") + logger.info(info) info_list_inst = [] for elem in info: - log("in Python ReadTensorsBase info ", info) + logger.info("in Python ReadTensorsBase info:") + logger.info(info) info_list_inst.append(elem.instance) tensor_base_data_list = self.dbg_instance.ReadTensorsBase(info_list_inst) tensor_base_data_list_ret = [] @@ -366,8 +338,7 @@ class DbgServices: Examples: >>> from mindspore.ccsrc.debug.debugger.offline_debug import dbg_services - >>> d = dbg_services.DbgServices(dump_file_path="dump_file_path", - ... verbose=True) + >>> d = dbg_services.DbgServices(dump_file_path="dump_file_path") >>> d_init = d.initialize(is_sync_mode=True) >>> tensor_stat_data_list = d_init.read_tensor_stats([dbg_services.TensorInfo(node_name="conv2.bias", ... slot=0, @@ -376,10 +347,12 @@ class DbgServices: ... root_graph_id=0, ... is_output=True)]) """ - log("in Python ReadTensorsStat info ", info) + logger.info("in Python ReadTensorsStat info:") + logger.info(info) info_list_inst = [] for elem in info: - log("in Python ReadTensorsStat info ", info) + logger.info("in Python ReadTensorsStat info:") + logger.info(info) info_list_inst.append(elem.instance) tensor_stat_data_list = self.dbg_instance.ReadTensorsStat(info_list_inst) tensor_stat_data_list_ret = [] diff --git a/mindspore/offline_debug/mi_validator_helpers.py b/mindspore/offline_debug/mi_validator_helpers.py index d88707f9074..45fb7cf4ab7 100644 --- a/mindspore/offline_debug/mi_validator_helpers.py +++ b/mindspore/offline_debug/mi_validator_helpers.py @@ -131,3 +131,21 @@ def type_check_list(args, types, arg_names): def replace_minus_one(value): """ replace -1 with a default value """ return value if value != -1 else UINT32_MAX + +def check_param_id(info_param, info_name): + """ + Check the type of info_param. + + Args: + info_param (Union[list[int], str]): Info parameters of check_node_list that is either list of ints or *. + info_name (str): Info name of check_node_list. + + Raises: + ValueError: When the type of info_param is not correct, otherwise nothing. + """ + if isinstance(info_param, str): + if info_param not in ["*"]: + raise ValueError("Node parameter {} only accepts '*' as string.".format(info_name)) + else: + for param in info_param: + check_uint32(param, info_name) diff --git a/mindspore/offline_debug/mi_validators.py b/mindspore/offline_debug/mi_validators.py index f0eff829489..a3cc929049e 100644 --- a/mindspore/offline_debug/mi_validators.py +++ b/mindspore/offline_debug/mi_validators.py @@ -19,7 +19,7 @@ from functools import wraps import mindspore.offline_debug.dbg_services as cds from mindspore.offline_debug.mi_validator_helpers import parse_user_args, type_check, \ - type_check_list, check_dir, check_uint32, check_uint64, check_iteration + type_check_list, check_dir, check_uint32, check_uint64, check_iteration, check_param_id def check_init(method): @@ -27,10 +27,9 @@ def check_init(method): @wraps(method) def new_method(self, *args, **kwargs): - [dump_file_path, verbose], _ = parse_user_args(method, *args, **kwargs) + [dump_file_path], _ = parse_user_args(method, *args, **kwargs) type_check(dump_file_path, (str,), "dump_file_path") - type_check(verbose, (bool,), "verbose") check_dir(dump_file_path) return method(self, *args, **kwargs) @@ -70,19 +69,9 @@ def check_add_watchpoint(method): for info_name, info_param in node_info.items(): type_check(info_name, (str,), "node parameter name") if info_name in ["rank_id"]: - if isinstance(info_param, str): - if info_param not in ["*"]: - raise ValueError("Node parameter {} only accepts '*' as string.".format(info_name)) - else: - for param in info_param: - check_uint32(param, "rank_id") + check_param_id(info_param, info_name="rank_id") elif info_name in ["root_graph_id"]: - if isinstance(info_param, str): - if info_param not in ["*"]: - raise ValueError("Node parameter {} only accepts '*' as string.".format(info_name)) - else: - for param in info_param: - check_uint32(param, "root_graph_id") + check_param_id(info_param, info_name="root_graph_id") elif info_name in ["is_output"]: type_check(info_param, (bool,), "is_output") else: diff --git a/tests/ut/python/debugger/gpu_tests/test_sync_read_tensors_base_stat.py b/tests/ut/python/debugger/gpu_tests/test_sync_read_tensors_base_stat.py index df9b2292736..2e0ecd9edb2 100644 --- a/tests/ut/python/debugger/gpu_tests/test_sync_read_tensors_base_stat.py +++ b/tests/ut/python/debugger/gpu_tests/test_sync_read_tensors_base_stat.py @@ -64,7 +64,7 @@ class TestOfflineReadTensorBaseStat: cls.test_path = build_dump_structure([name1, name2, name3, name4], [value_tensor, inf_tensor, nan_tensor, invalid_tensor], "Test", cls.tensor_info) - cls.debugger_backend = d.DbgServices(dump_file_path=cls.test_path, verbose=True) + cls.debugger_backend = d.DbgServices(dump_file_path=cls.test_path) _ = cls.debugger_backend.initialize(net_name="Test", is_sync_mode=True) @classmethod