nuoya
/
mindspore2022
forked from huawei/mindspore2022
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

use ms logging instead of custom debugger logging 

add const to the data_ptr definition

Fix pclint for debugger

fix bugs

fix duplicated code issue

Fix the depth of method over 4 issue

remove verbose option from DbgServices

fix comments

fix CI errors

remove redundancy arguments
This commit is contained in:
John Tzanakakis 2021-09-29 23:13:55 -04:00 committed by maning202007
parent e0253ea896
commit 8b44dd1793
26 changed files with 165 additions and 292 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
mindspore/ccsrc/backend/session/ascend_session.cc
View File

@ -1213,7 +1213,7 @@ void AscendSession::SelectKernel(const KernelGraph &kernel_graph) const {
void DumpInit(uint32_t device_id) {
auto &json_parser = DumpJsonParser::GetInstance();
json_parser.Parse();
json_parser.CopyJsonToDir(device_id);
json_parser.CopyDumpJsonToDir(device_id);
json_parser.CopyHcclJsonToDir(device_id);
json_parser.CopyMSCfgJsonToDir(device_id);
if (json_parser.async_dump_enabled()) {
@ -1555,7 +1555,7 @@ void AscendSession::Execute(const std::shared_ptr<KernelGraph> &kernel_graph, bo
void AscendSession::DumpSetup(const std::shared_ptr<KernelGraph> &kernel_graph) const {
MS_LOG(DEBUG) << "Start!";
MS_EXCEPTION_IF_NULL(kernel_graph);
E2eDump::DumpSetup(kernel_graph.get(), rank_id_);
E2eDump::DumpSetup(kernel_graph.get());
MS_LOG(DEBUG) << "Finish!";
}

4
mindspore/ccsrc/backend/session/gpu_session.cc
View File

@ -126,7 +126,7 @@ void GPUSession::Init(uint32_t device_id) {
#ifndef ENABLE_SECURITY
auto &json_parser = DumpJsonParser::GetInstance();
// Dump json config file if dump is enabled
json_parser.CopyJsonToDir(rank_id_);
json_parser.CopyDumpJsonToDir(rank_id_);
json_parser.CopyMSCfgJsonToDir(rank_id_);
#endif
MS_LOG(INFO) << "Set device id " << device_id << " for gpu session.";
@ -705,7 +705,7 @@ void GPUSession::RunOpImpl(const GraphInfo &graph_info, OpRunInfo *op_run_info,
void GPUSession::DumpSetup(const std::shared_ptr<KernelGraph> &kernel_graph) const {
MS_LOG(INFO) << "Start!";
MS_EXCEPTION_IF_NULL(kernel_graph);
E2eDump::DumpSetup(kernel_graph.get(), rank_id_);
E2eDump::DumpSetup(kernel_graph.get());
MS_LOG(INFO) << "Finish!";
}

3
mindspore/ccsrc/debug/CMakeLists.txt
View File

@ -14,8 +14,8 @@ set(_DEBUG_SRC_LIST
set(_OFFLINE_SRC_LIST
"${CMAKE_CURRENT_SOURCE_DIR}/debug_services.cc"
"${CMAKE_CURRENT_SOURCE_DIR}/debugger/tensor_summary.cc"
"${CMAKE_CURRENT_SOURCE_DIR}/debugger/offline_debug/offline_logger.cc"
"${CMAKE_CURRENT_SOURCE_DIR}/debugger/offline_debug/dbg_services.cc"
"${CMAKE_SOURCE_DIR}/mindspore/core/utils/log_adapter.cc"
"${CMAKE_CURRENT_SOURCE_DIR}/debugger/offline_debug/mi_pybind_register.cc"
)
@ -66,6 +66,7 @@ if(ENABLE_DEBUGGER)
set_property(SOURCE ${_OFFLINE_SRC_LIST} PROPERTY COMPILE_DEFINITIONS
SUBMODULE_ID=mindspore::SubModuleId::SM_OFFLINE_DEBUG)
add_library(_mindspore_offline_debug SHARED ${_OFFLINE_SRC_LIST})
target_link_libraries(_mindspore_offline_debug PRIVATE mindspore::glog mindspore_gvar)
set_target_properties(_mindspore_offline_debug PROPERTIES
PREFIX "${PYTHON_MODULE_PREFIX}"
SUFFIX "${PYTHON_MODULE_EXTENSION}"

71
mindspore/ccsrc/debug/data_dump/dump_json_parser.cc
View File

@ -134,7 +134,7 @@ void WriteJsonFile(const std::string &file_path, const std::ifstream &json_file)
ChangeFileMode(file_path, S_IRUSR);
}
void DumpJsonParser::CopyJsonToDir(uint32_t rank_id) {
void DumpJsonParser::CopyDumpJsonToDir(uint32_t rank_id) {
this->Parse();
if (!IsDumpEnabled()) {
return;
@ -148,7 +148,7 @@ void DumpJsonParser::CopyJsonToDir(uint32_t rank_id) {
auto realpath =
Common::CreatePrefixPath(path_ + "/rank_" + std::to_string(rank_id) + "/.dump_metadata/data_dump.json");
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed in CopyJsonDir.";
MS_LOG(ERROR) << "Get real path failed in CopyDumpJsonToDir.";
} else {
WriteJsonFile(realpath.value(), json_file);
}
@ -374,50 +374,49 @@ void DumpJsonParser::ParseIteration(const nlohmann::json &content) {
}
}
bool IsIterInRange(uint32_t iteration, const std::string &range) {
if (range.empty()) {
return false;
}
const std::string dash = "-";
std::size_t range_idx = range.find(dash);
// no dash in range, compare the value directly
if (range_idx == std::string::npos) {
return iteration == std::stoul(range);
}
// make sure there is only one dash in range
if (range.find(dash, range_idx + 1) != std::string::npos) {
return false;
}
auto low_range_str = range.substr(0, range_idx);
auto high_range_str = range.substr(range_idx + 1);
if (low_range_str.empty() || high_range_str.empty()) {
return false;
}
uint32_t low_range = std::stoul(low_range_str);
uint32_t high_range = std::stoul(high_range_str);
return (low_range <= iteration) && (iteration <= high_range);
}
bool DumpJsonParser::IsDumpIter(uint32_t iteration) const {
// bool DumpJsonParser::IsDumpIter(uint32_t iteration) --> checks if iteration should be dumped or not.
if (iteration_ == "all") {
return true;
}
const std::string vertical_bar = "|";
const std::string dash = "-";
int start = 0;
int end = iteration_.find(vertical_bar);
while (end != -1) {
std::string temp = iteration_.substr(IntToSize(start), IntToSize(end - start));
int range_idx = temp.find(dash);
if (range_idx != -1) {
uint32_t low_range = static_cast<uint32_t>(std::stoul(temp.substr(0, IntToSize(range_idx))));
uint32_t high_range = static_cast<uint32_t>(std::stoul(temp.substr(IntToSize(range_idx + 1), -1)));
if ((low_range <= iteration) && (iteration <= high_range)) {
return true;
}
} else if (iteration == std::stoul(temp)) {
std::size_t start = 0;
std::size_t end = iteration_.find(vertical_bar);
while (end != std::string::npos) {
std::string temp = iteration_.substr(start, end - start);
auto found = IsIterInRange(iteration, temp);
if (found) {
return true;
}
start = end + 1;
end = static_cast<int>(iteration_.find(vertical_bar, start));
end = iteration_.find(vertical_bar, start);
}
std::string temp = iteration_.substr(IntToSize(start), IntToSize(end - start));
int range_idx = temp.find(dash);
if (range_idx != -1) {
uint32_t low_range = static_cast<uint32_t>(std::stoul(temp.substr(0, IntToSize(range_idx))));
uint32_t high_range = static_cast<uint32_t>(std::stoul(temp.substr(IntToSize(range_idx + 1), -1)));
if ((low_range <= iteration) && (iteration <= high_range)) {
return true;
}
} else if (iteration == std::stoul(temp)) {
return true;
}
return false;
}
bool DumpJsonParser::IsSingleIter() {
// bool DumpJsonParser::IsSingleIter() --> checks if iteration in json dump file is single or not.
if (iteration_ != "all" && iteration_.find("-") == std::string::npos && iteration_.find("|") == std::string::npos) {
return true;
}
return false;
std::string temp = iteration_.substr(start);
return IsIterInRange(iteration, temp);
}
void DumpJsonParser::ParseInputOutput(const nlohmann::json &content) {

7
mindspore/ccsrc/debug/data_dump/dump_json_parser.h
View File

@ -36,15 +36,14 @@ class DumpJsonParser {
void Parse();
static bool DumpToFile(const std::string &filename, const void *data, size_t len, const ShapeVector &shape,
TypeId type);
void CopyJsonToDir(uint32_t device_id);
void CopyHcclJsonToDir(uint32_t device_id);
void CopyMSCfgJsonToDir(uint32_t device_id);
void CopyDumpJsonToDir(uint32_t rank_id);
void CopyHcclJsonToDir(uint32_t rank_id);
void CopyMSCfgJsonToDir(uint32_t rank_id);
bool NeedDump(const std::string &op_full_name) const;
void MatchKernel(const std::string &kernel_name);
void PrintUnusedKernel();
bool IsDumpIter(uint32_t iteration) const;
bool DumpAllIter();
bool IsSingleIter();
bool async_dump_enabled() const { return async_dump_enabled_; }
bool e2e_dump_enabled() const { return e2e_dump_enabled_; }

2
mindspore/ccsrc/debug/data_dump/e2e_dump.cc
View File

@ -299,7 +299,7 @@ void E2eDump::UpdateIterDumpSetup(const session::KernelGraph *graph, bool sink_m
}
}
void E2eDump::DumpSetup(const session::KernelGraph *graph, uint32_t rank_id) {
void E2eDump::DumpSetup(const session::KernelGraph *graph) {
auto &dump_json_parser = DumpJsonParser::GetInstance();
bool sink_mode = (ConfigManager::GetInstance().dataset_mode() || E2eDump::isDatasetGraph(graph));

2
mindspore/ccsrc/debug/data_dump/e2e_dump.h
View File

@ -35,7 +35,7 @@ class E2eDump {
public:
E2eDump() = default;
~E2eDump() = default;
static void DumpSetup(const session::KernelGraph *graph, uint32_t rank_id);
static void DumpSetup(const session::KernelGraph *graph);
static void UpdateIterGPUDump();

91
mindspore/ccsrc/debug/debug_services.cc
View File

@ -85,7 +85,7 @@ void DebugServices::RemoveWatchpoint(unsigned int id) {
}
std::unique_ptr<ITensorSummary> GetSummaryPtr(const std::shared_ptr<TensorData> &tensor,
void *const previous_tensor_ptr, uint32_t num_elements,
const void *const previous_tensor_ptr, uint32_t num_elements,
uint32_t prev_num_elements, int tensor_dtype) {
switch (tensor_dtype) {
case DbgDataType::DT_UINT8: {
@ -170,9 +170,9 @@ DebugServices::TensorStat DebugServices::GetTensorStatistics(const std::shared_p
return tensor_stat_data;
}
#ifdef OFFLINE_DBG_MODE
void *DebugServices::GetPrevTensor(const std::shared_ptr<TensorData> &tensor, bool previous_iter_tensor_needed,
uint32_t *prev_num_elements) {
void *previous_tensor_ptr = nullptr;
const void *DebugServices::GetPrevTensor(const std::shared_ptr<TensorData> &tensor, bool previous_iter_tensor_needed,
uint32_t *prev_num_elements) {
const void *previous_tensor_ptr = nullptr;
std::shared_ptr<TensorData> tensor_prev;
if (previous_iter_tensor_needed && tensor->GetIteration() >= 1) {
// read data in offline mode
@ -369,7 +369,7 @@ void DebugServices::CheckWatchpointsForTensor(
int tensor_dtype = tensor->GetType();
uint32_t num_elements = tensor->GetNumElements();
uint32_t prev_num_elements = 0;
void *previous_tensor_ptr = nullptr;
const void *previous_tensor_ptr = nullptr;
#ifdef OFFLINE_DBG_MODE
previous_tensor_ptr = GetPrevTensor(tensor, previous_iter_tensor_needed, &prev_num_elements);
#else
@ -949,15 +949,36 @@ void DebugServices::ReadDumpedTensor(std::vector<std::string> backend_name, std:
}
}
void DebugServices::ReadFileAndAddToTensor(const bool found, const std::vector<std::string> &matched_paths,
const std::string &backend_name, const unsigned int device_id,
const unsigned int root_graph_id, const bool &is_output, size_t slot,
bool *no_mem_to_read, unsigned int iteration,
std::vector<std::shared_ptr<TensorData>> *result_list) {
std::string time_stamp;
std::string type_name = "";
uint64_t data_size = 0;
std::vector<int64_t> shape;
std::vector<char> *buffer = nullptr;
if (found) {
std::string result_path = GetNewestFilePath(matched_paths);
time_stamp = GetTimeStampStr(result_path);
std::string key_name_in_cache = backend_name + ":" + std::to_string(device_id) + ":" +
std::to_string(root_graph_id) + ":" + std::to_string(is_output) + ":" +
std::to_string(slot);
ReadTensorFromNpy(key_name_in_cache, result_path, &type_name, &data_size, &shape, &buffer, no_mem_to_read);
AddToTensorData(backend_name, time_stamp, slot, iteration, device_id, root_graph_id, is_output, data_size,
type_name, shape, buffer, result_list);
} else {
AddToTensorData(backend_name, time_stamp, slot, iteration, device_id, root_graph_id, is_output, 0, type_name, shape,
buffer, result_list);
MS_LOG(INFO) << "Target tensor has not been found.";
}
}
void DebugServices::ReadDumpedTensorSync(const std::string &prefix_dump_file_name, const std::string &specific_dump_dir,
const std::string &backend_name, size_t slot, unsigned int device_id,
const std::string &backend_name, size_t slot, const unsigned int device_id,
unsigned int iteration, unsigned int root_graph_id, const bool &is_output,
std::vector<std::shared_ptr<TensorData>> *result_list, bool *no_mem_to_read) {
std::vector<char> *buffer = nullptr;
std::string type_name = "";
std::vector<int64_t> shape;
uint64_t data_size = 0;
std::string time_stamp;
std::string abspath = RealPath(specific_dump_dir);
DIR *d = opendir(abspath.c_str());
bool found_file = false;
@ -984,22 +1005,8 @@ void DebugServices::ReadDumpedTensorSync(const std::string &prefix_dump_file_nam
}
(void)closedir(d);
}
if (found_file) {
shape.clear();
std::string result_path = GetNewestFilePath(matched_paths);
time_stamp = GetTimeStampStr(result_path);
std::string key_name_in_cache = backend_name + ":" + std::to_string(device_id) + ":" +
std::to_string(root_graph_id) + ":" + std::to_string(is_output) + ":" +
std::to_string(slot);
ReadTensorFromNpy(key_name_in_cache, result_path, &type_name, &data_size, &shape, &buffer, no_mem_to_read);
AddToTensorData(backend_name, time_stamp, slot, iteration, device_id, root_graph_id, is_output, data_size,
type_name, shape, buffer, result_list);
} else {
AddToTensorData(backend_name, time_stamp, slot, iteration, device_id, root_graph_id, is_output, 0, type_name, shape,
buffer, result_list);
MS_LOG(INFO) << "Target tensor has not been found.";
}
ReadFileAndAddToTensor(found_file, matched_paths, backend_name, device_id, root_graph_id, is_output, slot,
no_mem_to_read, iteration, result_list);
}
void DebugServices::ReadDumpedTensorAsync(const std::string &specific_dump_dir, const std::string &prefix_dump_to_check,
@ -1008,11 +1015,6 @@ void DebugServices::ReadDumpedTensorAsync(const std::string &specific_dump_dir,
unsigned int root_graph_id, const bool &is_output,
const std::vector<std::string> &async_file_pool,
std::vector<std::shared_ptr<TensorData>> *result_list, bool *no_mem_to_read) {
std::vector<char> *buffer = nullptr;
std::string type_name = "";
std::vector<int64_t> shape;
uint64_t data_size = 0;
std::string time_stamp;
bool found = false;
std::vector<std::string> matched_paths;
// if async mode
@ -1024,22 +1026,8 @@ void DebugServices::ReadDumpedTensorAsync(const std::string &specific_dump_dir,
found = true;
}
}
if (found) {
shape.clear();
std::string result_path = GetNewestFilePath(matched_paths);
time_stamp = GetTimeStampStr(result_path);
std::string key_name_in_cache = backend_name + ":" + std::to_string(device_id) + ":" +
std::to_string(root_graph_id) + ":" + std::to_string(is_output) + ":" +
std::to_string(slot);
ReadTensorFromNpy(key_name_in_cache, result_path, &type_name, &data_size, &shape, &buffer, no_mem_to_read);
AddToTensorData(backend_name, time_stamp, slot, iteration, device_id, root_graph_id, is_output, data_size,
type_name, shape, buffer, result_list);
} else {
// If no npy file is found, add empty tensor data.
AddToTensorData(backend_name, time_stamp, slot, iteration, device_id, root_graph_id, is_output, 0, type_name, shape,
buffer, result_list);
MS_LOG(INFO) << "Target tensor has not been found.";
}
ReadFileAndAddToTensor(found, matched_paths, backend_name, device_id, root_graph_id, is_output, slot, no_mem_to_read,
iteration, result_list);
}
std::string DebugServices::GetStrippedFilename(const std::string &file_name) {
@ -1187,7 +1175,7 @@ std::string DebugServices::IterationString(unsigned int iteration) {
#endif
void DebugServices::ReadNodesTensors(const std::vector<std::string> &name, std::vector<std::string> *const ret_name,
std::vector<char *> *const data_ptr, std::vector<ssize_t> *const data_size,
std::vector<const char *> *const data_ptr, std::vector<ssize_t> *const data_size,
std::vector<unsigned int> *const dtype,
std::vector<std::vector<int64_t>> *const shape) {
std::vector<std::tuple<std::string, std::shared_ptr<TensorData>>> result_list;
@ -1198,7 +1186,7 @@ void DebugServices::ReadNodesTensors(const std::vector<std::string> &name, std::
continue;
}
(void)ret_name->emplace_back(std::get<0>(result));
(void)data_ptr->emplace_back(reinterpret_cast<char *>(std::get<1>(result)->GetDataPtr()));
(void)data_ptr->emplace_back(reinterpret_cast<const char *>(std::get<1>(result)->GetDataPtr()));
(void)data_size->emplace_back(std::get<1>(result)->GetByteSize());
(void)dtype->emplace_back(std::get<1>(result)->GetType());
(void)shape->emplace_back(std::get<1>(result)->GetShape());
@ -1351,9 +1339,6 @@ bool DebugServices::CheckOpOverflow(std::string node_name_to_find, unsigned int
infile.open(file_path.c_str(), std::ios::ate | std::ios::binary | std::ios::in);
if (!infile.is_open()) {
MS_LOG(ERROR) << "Failed to open overflow bin file " << file_name << " Errno:" << errno;
const int kMaxFilenameLength = 128;
char err_info[kMaxFilenameLength];
MS_LOG(ERROR) << " ErrInfo:" << strerror_r(errno, err_info, sizeof(err_info));
continue;
}

13
mindspore/ccsrc/debug/debug_services.h
View File

@ -22,7 +22,6 @@
#ifdef OFFLINE_DBG_MODE
#include "base/float16.h"
#include "debugger/offline_debug/offline_logger.h"
#endif
#include <math.h>
@ -330,6 +329,12 @@ class DebugServices {
unsigned int device_id, unsigned int root_graph_id,
std::vector<std::shared_ptr<TensorData>> *const tensor_list);
void ReadFileAndAddToTensor(const bool found, const std::vector<std::string> &matched_paths,
const std::string &backend_name, const unsigned int device_id,
const unsigned int root_graph_id, const bool &is_output, size_t slot,
bool *no_mem_to_read, unsigned int iteration,
std::vector<std::shared_ptr<TensorData>> *result_list);
void ReadDumpedTensorSync(const std::string &prefix_dump_file_name, const std::string &specific_dump_dir,
const std::string &backend_name, size_t slot, unsigned int device_id,
unsigned int iteration, unsigned int root_graph_id, const bool &is_output,
@ -344,8 +349,8 @@ class DebugServices {
std::vector<std::shared_ptr<TensorData>> ReadNeededDumpedTensors(unsigned int iteration,
std::vector<std::string> *const async_file_pool);
void *GetPrevTensor(const std::shared_ptr<TensorData> &tensor, bool previous_iter_tensor_needed,
uint32_t *prev_num_elements);
const void *GetPrevTensor(const std::shared_ptr<TensorData> &tensor, bool previous_iter_tensor_needed,
uint32_t *prev_num_elements);
void ReadTensorFromNpy(const std::string &tensor_name, const std::string &file_name, std::string *const tensor_type,
std::size_t *const size, std::vector<int64_t> *const shape,
@ -380,7 +385,7 @@ class DebugServices {
std::string IterationString(unsigned int iteration);
#endif
void ReadNodesTensors(const std::vector<std::string> &name, std::vector<std::string> *ret_name,
std::vector<char *> *data_ptr, std::vector<ssize_t> *data_size,
std::vector<const char *> *data_ptr, std::vector<ssize_t> *data_size,
std::vector<unsigned int> *dtype, std::vector<std::vector<int64_t>> *const shape);
void SearchNodesTensors(const std::vector<std::string> &name,

5
mindspore/ccsrc/debug/debugger/debugger.cc
View File

@ -407,9 +407,8 @@ void Debugger::DumpSingleNode(const CNodePtr &node, uint32_t graph_id) {
void Debugger::DumpSetup(const KernelGraphPtr &kernel_graph) const {
MS_LOG(INFO) << "Start!";
uint32_t rank_id = GetRankID();
MS_EXCEPTION_IF_NULL(kernel_graph);
E2eDump::DumpSetup(kernel_graph.get(), rank_id);
E2eDump::DumpSetup(kernel_graph.get());
MS_LOG(INFO) << "Finish!";
}
@ -985,7 +984,7 @@ void Debugger::RemoveWatchpoint(const int32_t id) { debug_services_->RemoveWatch
std::list<TensorProto> Debugger::LoadTensors(const ProtoVector<TensorProto> &tensors) const {
std::vector<std::string> name;
std::vector<std::string> ret_name;
std::vector<char *> data_ptr;
std::vector<const char *> data_ptr;
std::vector<ssize_t> data_size;
std::vector<unsigned int> dtype;
std::vector<std::vector<int64_t>> shape;

9
mindspore/ccsrc/debug/debugger/offline_debug/dbg_services.cc
View File

@ -18,14 +18,7 @@
#include <algorithm>
#include <chrono>
DbgServices::DbgServices(bool verbose) {
DbgLogger::verbose = verbose;
std::string dbg_log_path = common::GetEnv("OFFLINE_DBG_LOG");
if (!dbg_log_path.empty()) {
DbgLogger::verbose = true;
}
debug_services_ = std::make_shared<DebugServices>();
}
DbgServices::DbgServices() { debug_services_ = std::make_shared<DebugServices>(); }
DbgServices::DbgServices(const DbgServices &other) {
MS_LOG(INFO) << "cpp DbgServices object is created via copy";

4
mindspore/ccsrc/debug/debugger/offline_debug/dbg_services.h
View File

@ -101,7 +101,7 @@ struct tensor_info_t {
};
struct tensor_data_t {
tensor_data_t(char *data_ptr, uint64_t data_size, int dtype, const std::vector<int64_t> &shape)
tensor_data_t(const char *data_ptr, uint64_t data_size, int dtype, const std::vector<int64_t> &shape)
: data_size(data_size), dtype(dtype), shape(shape) {
if (data_ptr != nullptr) {
this->data_ptr = py::bytes(data_ptr, data_size);
@ -183,7 +183,7 @@ struct TensorStatData {
class DbgServices {
public:
explicit DbgServices(bool verbose = false);
DbgServices();
DbgServices(const DbgServices &other);

2
mindspore/ccsrc/debug/debugger/offline_debug/mi_pybind_register.cc
View File

@ -21,7 +21,7 @@
PYBIND11_MODULE(_mindspore_offline_debug, m) {
m.doc() = "pybind11 debug services api";
(void)py::class_<DbgServices>(m, "DbgServices")
.def(py::init<bool>())
.def(py::init())
.def("Initialize", &DbgServices::Initialize)
.def("AddWatchpoint", &DbgServices::AddWatchpoint)
.def("RemoveWatchpoint", &DbgServices::RemoveWatchpoint)

19
mindspore/ccsrc/debug/debugger/offline_debug/offline_logger.cc
View File

@ -1,19 +0,0 @@
/**
* Copyright 2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "debugger/offline_debug/offline_logger.h"
bool DbgLogger::verbose = false;

63
mindspore/ccsrc/debug/debugger/offline_debug/offline_logger.h
View File

@ -1,63 +0,0 @@
/**
* Copyright 2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef OFFLINE_LOGGER_H_
#define OFFLINE_LOGGER_H_
#include <iostream>
#define PATH_MAX 4096
#define MS_LOG(level) MS_LOG_##level
#define MS_LOG_INFO static_cast<void>(0), !(DbgLogger::verbose) ? void(0) : DbgLogger(DbgLoggerLvl::INFO) < std::cout
#define MS_LOG_ERROR MS_LOG_INFO
#define MS_LOG_DEBUG MS_LOG_INFO
#define MS_LOG_WARNING MS_LOG_INFO
#define MS_LOG_EXCEPTION static_cast<void>(0), DbgLogger(DbgLoggerLvl::EXCEPTION) < std::cout
enum DbgLoggerLvl : int { DEBUG = 0, INFO, WARNING, ERROR, EXCEPTION };
class DbgLogger {
public:
explicit DbgLogger(DbgLoggerLvl lvl) : lvl_(lvl) {}
~DbgLogger() = default;
void operator<(std::ostream &os) const {
char *dbg_log_path = std::getenv("OFFLINE_DBG_LOG");
if (dbg_log_path != nullptr) {
char abspath[PATH_MAX];
if (sizeof(dbg_log_path) > PATH_MAX || NULL == realpath(dbg_log_path, abspath)) {
std::cout << "ERROR: DbgLogger could not create real path";
}
FILE *fp = freopen(abspath, "a", stdout);
if (fp == nullptr) {
std::cout << "ERROR: DbgLogger could not redirect all stdout to a file";
}
}
os << std::endl;
if (lvl_ == DbgLoggerLvl::EXCEPTION) {
throw lvl_;
}
}
static bool verbose;
private:
DbgLoggerLvl lvl_;
};
#endif // OFFLINE_LOGGER_H_

2
mindspore/ccsrc/debug/debugger/proto_exporter.cc
View File

@ -167,7 +167,7 @@ void DebuggerProtoExporter::SetValueToProto(const ValuePtr &val, debugger::Value
TypePtr elem_type = dyn_cast<TensorType>(val)->element();
type_proto->mutable_tensor_type()->set_elem_type(GetDebuggerNumberDataType(elem_type));
} else {
MS_LOG(WARNING) << "Unsupported type " << val->type_name();
MS_LOG(INFO) << "Unsupported type " << val->type_name();
}
}

9
mindspore/ccsrc/debug/debugger/tensor_summary.cc
View File

@ -25,7 +25,6 @@
#ifdef OFFLINE_DBG_MODE
#include "base/float16.h"
#include "offline_debug/offline_logger.h"
#endif
#ifdef ONLINE_DBG_MODE
@ -91,10 +90,10 @@ double VarianceAndMeanCalculator::GetVariance() const {
double VarianceAndMeanCalculator::GetStandardDeviation() { return sqrt(GetVariance()); }
template <typename T>
TensorSummary<T>::TensorSummary(void *current_tensor_ptr, void *const previous_tensor_ptr, uint32_t num_elements,
uint32_t prev_num_elements)
: current_tensor_ptr_(reinterpret_cast<T *>(current_tensor_ptr)),
prev_tensor_ptr_(reinterpret_cast<T *>(previous_tensor_ptr)),
TensorSummary<T>::TensorSummary(const void *current_tensor_ptr, const void *const previous_tensor_ptr,
uint32_t num_elements, uint32_t prev_num_elements)
: current_tensor_ptr_(reinterpret_cast<const T *>(current_tensor_ptr)),
prev_tensor_ptr_(reinterpret_cast<const T *>(previous_tensor_ptr)),
num_elements_(num_elements),
prev_num_elements_(prev_num_elements),
min_(std::numeric_limits<double>::max()),

6
mindspore/ccsrc/debug/debugger/tensor_summary.h
View File

@ -111,7 +111,7 @@ class TensorSummary : public ITensorSummary {
public:
TensorSummary() = default;
~TensorSummary() override = default;
TensorSummary(void *, void *, uint32_t, uint32_t);
TensorSummary(const void *, const void *, uint32_t, uint32_t);
void SummarizeTensor(const std::vector<DebugServices::watchpoint_t> &) override;
// returns hit, error_code, parameter_list
std::tuple<bool, int, std::vector<DebugServices::parameter_t>> IsWatchpointHit(DebugServices::watchpoint_t) override;
@ -129,8 +129,8 @@ class TensorSummary : public ITensorSummary {
const int zero_count() const override { return zero_count_; }
private:
T *current_tensor_ptr_;
T *prev_tensor_ptr_;
const T *current_tensor_ptr_;
const T *prev_tensor_ptr_;
uint32_t num_elements_;
uint32_t prev_num_elements_;
double min_;

8
mindspore/ccsrc/debug/tensor_data.h
View File

@ -21,11 +21,9 @@
#include <string>
#include <cstring>
#include <iostream>
#ifdef OFFLINE_DBG_MODE
#include "debugger/offline_debug/offline_logger.h"
#else
#include "ir/tensor.h"
#include "mindspore/core/utils/log_adapter.h"
#ifdef ONLINE_DBG_MODE
#include "ir/tensor.h"
#endif
#ifdef ONLINE_DBG_MODE
@ -213,7 +211,7 @@ class TensorData {
void SetSlot(size_t slot) { this->slot_ = slot; }
char *GetDataPtr() const { return this->data_ptr_; }
const char *GetDataPtr() const { return this->data_ptr_; }
void SetDataPtr(char *data_ptr) { this->data_ptr_ = data_ptr; }

3
mindspore/ccsrc/debug/tensor_load.h
View File

@ -25,9 +25,6 @@
#include <utility>
#include <deque>
#include <algorithm>
#ifdef OFFLINE_DBG_MODE
#include "debugger/offline_debug/offline_logger.h"
#endif
#include "debug/tensor_data.h"
#ifdef ONLINE_DBG_MODE
#include "debug/data_dump/dump_json_parser.h"

2
mindspore/ccsrc/runtime/hardware/cpu/cpu_device_context.cc
View File

@ -60,7 +60,7 @@ void CPUDeviceContext::Initialize() {
auto rank_id = GetRankID();
auto &json_parser = DumpJsonParser::GetInstance();
json_parser.Parse();
json_parser.CopyJsonToDir(rank_id);
json_parser.CopyDumpJsonToDir(rank_id);
json_parser.CopyMSCfgJsonToDir(rank_id);
#endif

2
mindspore/ccsrc/runtime/hardware/gpu/gpu_device_context.cc
View File

@ -102,7 +102,7 @@ void GPUDeviceContext::Initialize() {
auto rank_id = GetRankID();
auto &json_parser = DumpJsonParser::GetInstance();
json_parser.Parse();
json_parser.CopyJsonToDir(rank_id);
json_parser.CopyDumpJsonToDir(rank_id);
json_parser.CopyMSCfgJsonToDir(rank_id);
#endif
initialized_ = true;

87
mindspore/offline_debug/dbg_services.py
View File

@ -22,6 +22,7 @@ from mindspore.offline_debug.mi_validators import check_init, check_initialize,
check_tensor_info_init, check_tensor_data_init, check_tensor_base_data_init, check_tensor_stat_data_init,\
check_watchpoint_hit_init, check_parameter_init
from mindspore.offline_debug.mi_validator_helpers import replace_minus_one
from mindspore import log as logger
if not security.enable_security():
import mindspore._mindspore_offline_debug as cds
@ -40,29 +41,7 @@ def get_version():
if security.enable_security():
raise ValueError("Offline debugger is not supported in security mode. "
"Please recompile mindspore without `-s on`.")
return cds.DbgServices(False).GetVersion()
class DbgLogger:
"""
Offline Debug Services Logger
Args:
verbose (bool): Whether to print logs.
Examples:
>>> from mindspore.ccsrc.debug.debugger.offline_debug import dbg_services
>>> version = dbg_services.DbgLogger(verbose=False)
"""
def __init__(self, verbose):
self.verbose = verbose
def __call__(self, *logs):
if self.verbose:
print(logs)
log = DbgLogger(False)
return cds.DbgServices().GetVersion()
class DbgServices:
@ -71,25 +50,21 @@ class DbgServices:
Args:
dump_file_path (str): Directory where the dump files are saved.
verbose (bool): Whether to print logs. Default: False.
Examples:
>>> from mindspore.ccsrc.debug.debugger.offline_debug import dbg_services
>>> d = dbg_services.DbgServices(dump_file_path="dump_file_path",
... verbose=True)
>>> d = dbg_services.DbgServices(dump_file_path="dump_file_path")
"""
@check_init
def __init__(self, dump_file_path, verbose=False):
def __init__(self, dump_file_path):
if security.enable_security():
raise ValueError("Offline debugger is not supported in security mode. "
"Please recompile mindspore without `-s on`.")
log.verbose = verbose
log("in Python __init__, file path is ", dump_file_path)
logger.info("in Python __init__, file path is %s", dump_file_path)
self.dump_file_path = dump_file_path
self.dbg_instance = cds.DbgServices(verbose)
self.dbg_instance = cds.DbgServices()
self.version = self.dbg_instance.GetVersion()
self.verbose = verbose
self.initialized = False
@check_initialize
@ -109,11 +84,10 @@ class DbgServices:
Examples:
>>> from mindspore.ccsrc.debug.debugger.offline_debug import dbg_services
>>> d = dbg_services.DbgServices(dump_file_path="dump_file_path",
... verbose=True)
>>> d = dbg_services.DbgServices(dump_file_path="dump_file_path")
>>> d_init = d.initialize(net_name="network name", is_sync_mode=True, max_mem_usage=4096)
"""
log("in Python Initialize dump_file_path ", self.dump_file_path)
logger.info("in Python Initialize dump_file_path %s", self.dump_file_path)
self.initialized = True
return self.dbg_instance.Initialize(net_name, self.dump_file_path, is_sync_mode, max_mem_usage)
@ -134,8 +108,7 @@ class DbgServices:
Examples:
>>> from mindspore.offline_debug import dbg_services
>>> d = dbg_services.DbgServices(dump_file_path="dump_file_path",
>>> verbose=True)
>>> d = dbg_services.DbgServices(dump_file_path="dump_file_path")
>>> d_init = d.initialize(is_sync_mode=True)
>>> d_wp = d_init.transform_check_node_list(info_name="rank_id",
>>> info_param=[0],
@ -171,8 +144,7 @@ class DbgServices:
Examples:
>>> from mindspore.ccsrc.debug.debugger.offline_debug import dbg_services
>>> d = dbg_services.DbgServices(dump_file_path="dump_file_path",
... verbose=True)
>>> d = dbg_services.DbgServices(dump_file_path="dump_file_path")
>>> d_init = d.initialize(is_sync_mode=True)
>>> d_wp = d_init.add_watchpoint(watchpoint_id=1,
... watch_condition=6,
@ -184,7 +156,7 @@ class DbgServices:
... hit=False,
... actual_value=0.0)])
"""
log("in Python AddWatchpoint")
logger.info("in Python AddWatchpoint")
for node_name, node_info in check_node_list.items():
for info_name, info_param in node_info.items():
check_node_list = self.transform_check_node_list(info_name, info_param, node_name, check_node_list)
@ -207,8 +179,7 @@ class DbgServices:
Examples:
>>> from mindspore.ccsrc.debug.debugger.offline_debug import dbg_services
>>> d = dbg_services.DbgServices(dump_file_path="dump_file_path",
... verbose=True)
>>> d = dbg_services.DbgServices(dump_file_path="dump_file_path")
>>> d_init = d.initialize(is_sync_mode=True)
>>> d_wp = d_init.add_watchpoint(watchpoint_id=1,
... watch_condition=6,
@ -221,7 +192,7 @@ class DbgServices:
... actual_value=0.0)])
>>> d_wp = d_wp.remove_watchpoint(watchpoint_id=1)
"""
log("in Python Remove Watchpoint id ", watchpoint_id)
logger.info("in Python Remove Watchpoint id %d", watchpoint_id)
return self.dbg_instance.RemoveWatchpoint(watchpoint_id)
@check_initialize_done
@ -238,8 +209,7 @@ class DbgServices:
Examples:
>>> from mindspore.ccsrc.debug.debugger.offline_debug import dbg_services
>>> d = dbg_services.DbgServices(dump_file_path="dump_file_path",
... verbose=True)
>>> d = dbg_services.DbgServices(dump_file_path="dump_file_path")
>>> d_init = d.initialize(is_sync_mode=True)
>>> d_wp = d_init.add_watchpoint(id=1,
... watch_condition=6,
@ -252,7 +222,7 @@ class DbgServices:
... actual_value=0.0)])
>>> watchpoints = d_wp.check_watchpoints(iteration=8)
"""
log("in Python CheckWatchpoints iteration ", iteration)
logger.info("in Python CheckWatchpoints iteration %d", iteration)
iteration = replace_minus_one(iteration)
watchpoint_list = self.dbg_instance.CheckWatchpoints(iteration)
watchpoint_hit_list = []
@ -291,8 +261,7 @@ class DbgServices:
Examples:
>>> from mindspore.ccsrc.debug.debugger.offline_debug import dbg_services
>>> d = dbg_services.DbgServices(dump_file_path="dump_file_path",
... verbose=True)
>>> d = dbg_services.DbgServices(dump_file_path="dump_file_path")
>>> d_init = d.initialize(is_sync_mode=True)
>>> tensor_data_list = d_init.read_tensors([dbg_services.TensorInfo(node_name="conv2.bias",
... slot=0,
@ -301,10 +270,12 @@ class DbgServices:
... root_graph_id=0,
... is_output=True)])
"""
log("in Python ReadTensors info ", info)
logger.info("in Python ReadTensors info:")
logger.info(info)
info_list_inst = []
for elem in info:
log("in Python ReadTensors info ", info)
logger.info("in Python ReadTensors info:")
logger.info(info)
info_list_inst.append(elem.instance)
tensor_data_list = self.dbg_instance.ReadTensors(info_list_inst)
tensor_data_list_ret = []
@ -330,8 +301,7 @@ class DbgServices:
Examples:
>>> from mindspore.ccsrc.debug.debugger.offline_debug import dbg_services
>>> d = dbg_services.DbgServices(dump_file_path="dump_file_path",
... verbose=True)
>>> d = dbg_services.DbgServices(dump_file_path="dump_file_path")
>>> d_init = d.initialize(is_sync_mode=True)
>>> tensor_base_data_list = d_init.read_tensor_base([dbg_services.TensorInfo(node_name="conv2.bias",
... slot=0,
@ -340,10 +310,12 @@ class DbgServices:
... root_graph_id=0,
... is_output=True)])
"""
log("in Python ReadTensorsBase info ", info)
logger.info("in Python ReadTensorsBase info:")
logger.info(info)
info_list_inst = []
for elem in info:
log("in Python ReadTensorsBase info ", info)
logger.info("in Python ReadTensorsBase info:")
logger.info(info)
info_list_inst.append(elem.instance)
tensor_base_data_list = self.dbg_instance.ReadTensorsBase(info_list_inst)
tensor_base_data_list_ret = []
@ -366,8 +338,7 @@ class DbgServices:
Examples:
>>> from mindspore.ccsrc.debug.debugger.offline_debug import dbg_services
>>> d = dbg_services.DbgServices(dump_file_path="dump_file_path",
... verbose=True)
>>> d = dbg_services.DbgServices(dump_file_path="dump_file_path")
>>> d_init = d.initialize(is_sync_mode=True)
>>> tensor_stat_data_list = d_init.read_tensor_stats([dbg_services.TensorInfo(node_name="conv2.bias",
... slot=0,
@ -376,10 +347,12 @@ class DbgServices:
... root_graph_id=0,
... is_output=True)])
"""
log("in Python ReadTensorsStat info ", info)
logger.info("in Python ReadTensorsStat info:")
logger.info(info)
info_list_inst = []
for elem in info:
log("in Python ReadTensorsStat info ", info)
logger.info("in Python ReadTensorsStat info:")
logger.info(info)
info_list_inst.append(elem.instance)
tensor_stat_data_list = self.dbg_instance.ReadTensorsStat(info_list_inst)
tensor_stat_data_list_ret = []

18
mindspore/offline_debug/mi_validator_helpers.py
View File

@ -131,3 +131,21 @@ def type_check_list(args, types, arg_names):
def replace_minus_one(value):
""" replace -1 with a default value """
return value if value != -1 else UINT32_MAX
def check_param_id(info_param, info_name):
"""
Check the type of info_param.
Args:
info_param (Union[list[int], str]): Info parameters of check_node_list that is either list of ints or *.
info_name (str): Info name of check_node_list.
Raises:
ValueError: When the type of info_param is not correct, otherwise nothing.
"""
if isinstance(info_param, str):
if info_param not in ["*"]:
raise ValueError("Node parameter {} only accepts '*' as string.".format(info_name))
else:
for param in info_param:
check_uint32(param, info_name)

19
mindspore/offline_debug/mi_validators.py
View File

@ -19,7 +19,7 @@ from functools import wraps
import mindspore.offline_debug.dbg_services as cds
from mindspore.offline_debug.mi_validator_helpers import parse_user_args, type_check, \
type_check_list, check_dir, check_uint32, check_uint64, check_iteration
type_check_list, check_dir, check_uint32, check_uint64, check_iteration, check_param_id
def check_init(method):
@ -27,10 +27,9 @@ def check_init(method):
@wraps(method)
def new_method(self, *args, **kwargs):
[dump_file_path, verbose], _ = parse_user_args(method, *args, **kwargs)
[dump_file_path], _ = parse_user_args(method, *args, **kwargs)
type_check(dump_file_path, (str,), "dump_file_path")
type_check(verbose, (bool,), "verbose")
check_dir(dump_file_path)
return method(self, *args, **kwargs)
@ -70,19 +69,9 @@ def check_add_watchpoint(method):
for info_name, info_param in node_info.items():
type_check(info_name, (str,), "node parameter name")
if info_name in ["rank_id"]:
if isinstance(info_param, str):
if info_param not in ["*"]:
raise ValueError("Node parameter {} only accepts '*' as string.".format(info_name))
else:
for param in info_param:
check_uint32(param, "rank_id")
check_param_id(info_param, info_name="rank_id")
elif info_name in ["root_graph_id"]:
if isinstance(info_param, str):
if info_param not in ["*"]:
raise ValueError("Node parameter {} only accepts '*' as string.".format(info_name))
else:
for param in info_param:
check_uint32(param, "root_graph_id")
check_param_id(info_param, info_name="root_graph_id")
elif info_name in ["is_output"]:
type_check(info_param, (bool,), "is_output")
else:

2
tests/ut/python/debugger/gpu_tests/test_sync_read_tensors_base_stat.py
View File

@ -64,7 +64,7 @@ class TestOfflineReadTensorBaseStat:
cls.test_path = build_dump_structure([name1, name2, name3, name4],
[value_tensor, inf_tensor, nan_tensor, invalid_tensor],
"Test", cls.tensor_info)
cls.debugger_backend = d.DbgServices(dump_file_path=cls.test_path, verbose=True)
cls.debugger_backend = d.DbgServices(dump_file_path=cls.test_path)
_ = cls.debugger_backend.initialize(net_name="Test", is_sync_mode=True)
@classmethod