Col_lin
/
mindspore2022
forked from huawei/mindspore2022
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

sync commercial branch self check fix to master

This commit is contained in:
sabrinasun 2021-10-13 23:04:49 -04:00
parent ac9f5c5ede
commit 41d737553a
9 changed files with 315 additions and 239 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
mindspore/ccsrc/debug/data_dump/dump_json_parser.cc
View File

@ -364,6 +364,7 @@ void DumpJsonParser::ParseNetName(const nlohmann::json &content) {
void DumpJsonParser::ParseIteration(const nlohmann::json &content) {
CheckJsonStringType(content, kIteration);
auto context = MsContext::GetInstance();
MS_EXCEPTION_IF_NULL(context);
if (e2e_dump_enabled_ || async_dump_enabled_) {
iteration_ = content;
if (iteration_.empty() || (!std::all_of(iteration_.begin(), iteration_.end(), [](char c) {
@ -383,11 +384,13 @@ bool DumpJsonParser::IsDumpIter(uint32_t iteration) const {
if (iteration_ == "all") {
return true;
}
const std::string vertical_bar = "|";
const std::string dash = "-";
int start = 0;
int end = iteration_.find("|");
int end = iteration_.find(vertical_bar);
while (end != -1) {
std::string temp = iteration_.substr(IntToSize(start), IntToSize(end - start));
int range_idx = temp.find("-");
int range_idx = temp.find(dash);
if (range_idx != -1) {
uint32_t low_range = static_cast<uint32_t>(std::stoul(temp.substr(0, IntToSize(range_idx))));
uint32_t high_range = static_cast<uint32_t>(std::stoul(temp.substr(IntToSize(range_idx + 1), -1)));
@ -398,10 +401,10 @@ bool DumpJsonParser::IsDumpIter(uint32_t iteration) const {
return true;
}
start = end + 1;
end = static_cast<int>(iteration_.find("|", start));
end = static_cast<int>(iteration_.find(vertical_bar, start));
}
std::string temp = iteration_.substr(IntToSize(start), IntToSize(end - start));
int range_idx = temp.find("-");
int range_idx = temp.find(dash);
if (range_idx != -1) {
uint32_t low_range = static_cast<uint32_t>(std::stoul(temp.substr(0, IntToSize(range_idx))));
uint32_t high_range = static_cast<uint32_t>(std::stoul(temp.substr(IntToSize(range_idx + 1), -1)));

11
mindspore/ccsrc/debug/data_dump/e2e_dump.cc
View File

@ -175,16 +175,13 @@ void E2eDump::DumpInputImpl(const CNodePtr &node, bool trans_flag, const std::st
auto addr = AnfAlgo::GetOutputAddr(input, index);
MS_EXCEPTION_IF_NULL(addr);
std::string tensor_name;
size_t slot;
std::string tensor_name = GetKernelNodeName(node);
size_t slot = j;
if (IsDeviceTargetGPU()) {
auto input_kernel = node->input(j + 1);
std::string input_kernel_name = GetKernelNodeName(input_kernel);
tensor_name = input_kernel_name;
slot = 0;
} else {
tensor_name = GetKernelNodeName(node);
slot = j;
}
ShapeVector int_shapes;
GetDumpIntShape(input, index, NOT_NULL(&int_shapes), trans_flag);
@ -395,7 +392,7 @@ bool E2eDump::MoveDumpFiles(const std::string &first_dir, const std::string &sec
DIR *d_handle = opendir(first_dir.c_str());
struct dirent *next_file;
while ((next_file = readdir(d_handle)) != NULL) {
while ((next_file = readdir(d_handle)) != nullptr) {
if (next_file->d_type != DT_REG) {
continue;
}
@ -420,7 +417,7 @@ bool E2eDump::DeleteDirContents(const std::string &dir_path) {
DIR *d_handle = opendir(dir_path.c_str());
struct dirent *next_file;
while ((next_file = readdir(d_handle)) != NULL) {
while ((next_file = readdir(d_handle)) != nullptr) {
if (next_file->d_type != DT_REG) {
continue;
}

399
mindspore/ccsrc/debug/debug_services.cc
View File

@ -87,6 +87,7 @@ void DebugServices::RemoveWatchpoint(unsigned int id) {
std::unique_ptr<ITensorSummary> GetSummaryPtr(const std::shared_ptr<TensorData> &tensor,
void *const previous_tensor_ptr, uint32_t num_elements,
uint32_t prev_num_elements, int tensor_dtype) {
MS_EXCEPTION_IF_NULL(tensor);
switch (tensor_dtype) {
case DbgDataType::DT_UINT8: {
return std::make_unique<TensorSummary<uint8_t>>(tensor->GetDataPtr(), previous_tensor_ptr, num_elements,
@ -172,6 +173,7 @@ DebugServices::TensorStat DebugServices::GetTensorStatistics(const std::shared_p
#ifdef OFFLINE_DBG_MODE
void *DebugServices::GetPrevTensor(const std::shared_ptr<TensorData> &tensor, bool previous_iter_tensor_needed,
uint32_t *prev_num_elements) {
MS_EXCEPTION_IF_NULL(tensor);
void *previous_tensor_ptr = nullptr;
std::shared_ptr<TensorData> tensor_prev;
if (previous_iter_tensor_needed && tensor->GetIteration() >= 1) {
@ -271,22 +273,48 @@ void DebugServices::SetCheckWatchpointsResult(
const std::string time_stamp, const std::string &qualified_tensor_name, const std::string &tensor_slot,
const watchpoint_t &wp, const unsigned int device_id_val, const unsigned int root_graph_id_val,
const std::vector<parameter_t> &parameter_list, const int32_t error_code) {
(*chunk_exec_orders)[chunk_id].push_back(exec_order);
(*chunk_names)[chunk_id].push_back(qualified_tensor_name);
(*chunk_slots)[chunk_id].push_back(tensor_slot);
(*chunk_conditions)[chunk_id].push_back(wp.condition.type);
(*chunk_watchpoint_id)[chunk_id].push_back(wp.id);
(void)(*chunk_exec_orders)[chunk_id].emplace_back(exec_order);
(void)(*chunk_names)[chunk_id].emplace_back(qualified_tensor_name);
(void)(*chunk_slots)[chunk_id].emplace_back(tensor_slot);
(void)(*chunk_conditions)[chunk_id].emplace_back(wp.condition.type);
(void)(*chunk_watchpoint_id)[chunk_id].emplace_back(wp.id);
if (device_id != nullptr) {
(*chunk_device_id)[chunk_id].push_back(device_id_val);
(void)(*chunk_device_id)[chunk_id].emplace_back(device_id_val);
}
if (root_graph_id != nullptr) {
(*chunk_root_graph_id)[chunk_id].push_back(root_graph_id_val);
(void)(*chunk_root_graph_id)[chunk_id].emplace_back(root_graph_id_val);
}
(*chunk_parameters)[chunk_id].push_back(parameter_list);
(*chunk_error_codes)[chunk_id].push_back(error_code);
(*chunk_time_stamp)[chunk_id].push_back(time_stamp);
(void)(*chunk_parameters)[chunk_id].emplace_back(parameter_list);
(void)(*chunk_error_codes)[chunk_id].emplace_back(error_code);
(void)(*chunk_time_stamp)[chunk_id].emplace_back(time_stamp);
}
#ifdef OFFLINE_DBG_MODE
void DebugServices::ProcessCheckpointsOutofMemory(
const bool no_mem_to_read, const std::vector<watchpoint_t> watchpoints_to_check, int chunk_id,
partitioned_names *const chunk_names, partitioned_names *const chunk_slots,
partitioned_numbers *const chunk_conditions, partitioned_id *const chunk_watchpoint_id,
partitioned_parameters *const chunk_parameters, partitioned_error_code *const chunk_error_codes,
partitioned_numbers *const chunk_exec_orders, partitioned_names *const chunk_time_stamp,
partitioned_id *const chunk_device_id, partitioned_id *const chunk_root_graph_id,
std::vector<unsigned int> *const device_id, std::vector<unsigned int> *const root_graph_id, const int exec_order,
const std::string time_stamp, const std::string &qualified_tensor_name, const std::string &tensor_slot,
const unsigned int device_id_val, const unsigned int root_graph_id_val,
const std::vector<parameter_t> &parameter_list) {
if (no_mem_to_read) {
// bit 3 denotes failed to load tensor because tensor is oversized and no enough memory to fit in
int32_t oversize_error_code = 8;
for (auto &wp : watchpoints_to_check) {
SetCheckWatchpointsResult(chunk_id, chunk_names, chunk_slots, chunk_conditions, chunk_watchpoint_id,
chunk_parameters, chunk_error_codes, chunk_exec_orders, chunk_time_stamp,
chunk_device_id, chunk_root_graph_id, device_id, root_graph_id, exec_order, time_stamp,
qualified_tensor_name, tensor_slot, wp, device_id_val, root_graph_id_val,
parameter_list, oversize_error_code);
}
}
}
#endif
void DebugServices::CheckWatchpointsForTensor(
partitioned_names *const chunk_names, partitioned_names *const chunk_slots,
partitioned_numbers *const chunk_conditions, partitioned_id *const chunk_watchpoint_id,
@ -297,6 +325,10 @@ void DebugServices::CheckWatchpointsForTensor(
partitioned_id *const chunk_device_id, partitioned_id *const chunk_root_graph_id,
std::vector<uint64_t> *const chunk_tensor_byte_size, partitioned_names *const chunk_time_stamp,
std::vector<unsigned int> *const device_id, std::vector<unsigned int> *const root_graph_id) {
int list_size = tensor_list->size();
if (end > list_size) {
end = list_size;
}
for (int i = begin; i < end; i++) {
auto &tensor = (*tensor_list)[i];
const auto tensor_name = tensor->GetName();
@ -305,8 +337,6 @@ void DebugServices::CheckWatchpointsForTensor(
std::vector<watchpoint_t> watchpoints_to_check;
std::string qualified_tensor_name;
bool previous_iter_tensor_needed = false;
// Add do nothing line in case offline debug is off, prevent unused var warning
(void)previous_iter_tensor_needed;
AddWatchPointsToCheck(init_dbg_suspend, step_end, recheck, tensor, &previous_iter_tensor_needed,
&qualified_tensor_name, &watchpoints_to_check);
// no wp set on current tensor
@ -324,17 +354,11 @@ void DebugServices::CheckWatchpointsForTensor(
async_file_pool, &result_list, &no_mem_to_read);
tensor = result_list[0];
if (!tensor->GetByteSize()) {
if (no_mem_to_read) {
// bit 3 denotes failed to load tensor because tensor is oversized and no enough memory to fit in
int32_t oversize_error_code = 8;
for (auto &wp : watchpoints_to_check) {
SetCheckWatchpointsResult(
chunk_id, chunk_names, chunk_slots, chunk_conditions, chunk_watchpoint_id, chunk_parameters,
chunk_error_codes, chunk_exec_orders, chunk_time_stamp, chunk_device_id, chunk_root_graph_id, device_id,
root_graph_id, tensor->GetExecutionOrder(), tensor->GetTimeStamp(), qualified_tensor_name, tensor_slot, wp,
tensor->GetDeviceId(), tensor->GetRootGraphId(), std::vector<parameter_t>(), oversize_error_code);
}
}
ProcessCheckpointsOutofMemory(
no_mem_to_read, watchpoints_to_check, chunk_id, chunk_names, chunk_slots, chunk_conditions, chunk_watchpoint_id,
chunk_parameters, chunk_error_codes, chunk_exec_orders, chunk_time_stamp, chunk_device_id, chunk_root_graph_id,
device_id, root_graph_id, tensor->GetExecutionOrder(), tensor->GetTimeStamp(), qualified_tensor_name,
tensor_slot, tensor->GetDeviceId(), tensor->GetRootGraphId(), std::vector<parameter_t>());
tensor.reset();
continue;
}
@ -420,7 +444,7 @@ void DebugServices::CheckWatchpoints(
int tensor_list_size = tensor_list->size();
uint64_t tensor_list_byte_size = 0;
MS_LOG(INFO) << "tensor list size: " << tensor_list_size;
if (tensor_list_size == 0) {
if (tensor_list_size <= 0) {
return;
}
// default value for number of threads
@ -453,7 +477,7 @@ void DebugServices::CheckWatchpoints(
end++;
remainder--;
}
tensor_future_vec.push_back(std::async(
(void)tensor_future_vec.emplace_back(std::async(
std::launch::async, &DebugServices::CheckWatchpointsForTensor, this, &chunk_names, &chunk_slots,
&chunk_conditions, &chunk_watchpoint_id, &chunk_parameters, &chunk_error_codes, op_overflows, async_file_pool,
&chunk_exec_orders, tensor_list, begin, end, i, init_dbg_suspend, step_end, recheck, &chunk_device_id,
@ -494,26 +518,26 @@ void DebugServices::SortWatchpointsInfo(
std::vector<int>::iterator iter =
std::lower_bound(exec_order->begin(), exec_order->end(), (*chunk_exec_orders)[i][j]);
int position = iter - exec_order->begin();
(void)exec_order->insert(iter, (*chunk_exec_orders)[i][j]);
(void)exec_order->emplace(iter, (*chunk_exec_orders)[i][j]);
#endif
#ifdef OFFLINE_DBG_MODE
std::vector<std::string>::iterator iter =
std::lower_bound(time_stamps->begin(), time_stamps->end(), (*chunk_time_stamp)[i][j]);
int position = iter - time_stamps->begin();
(void)time_stamps->insert(iter, (*chunk_time_stamp)[i][j]);
(void)time_stamps->emplace(iter, (*chunk_time_stamp)[i][j]);
#endif
(void)name->insert(name->begin() + position, (*chunk_names)[i][j]);
(void)slot->insert(slot->begin() + position, (*chunk_slots)[i][j]);
(void)condition->insert(condition->begin() + position, (*chunk_conditions)[i][j]);
(void)watchpoint_id->insert(watchpoint_id->begin() + position, (*chunk_watchpoint_id)[i][j]);
(void)name->emplace(name->begin() + position, (*chunk_names)[i][j]);
(void)slot->emplace(slot->begin() + position, (*chunk_slots)[i][j]);
(void)condition->emplace(condition->begin() + position, (*chunk_conditions)[i][j]);
(void)watchpoint_id->emplace(watchpoint_id->begin() + position, (*chunk_watchpoint_id)[i][j]);
if (device_id != nullptr) {
(void)device_id->insert(device_id->begin() + position, (*chunk_device_id)[i][j]);
(void)device_id->emplace(device_id->begin() + position, (*chunk_device_id)[i][j]);
}
if (root_graph_id != nullptr) {
(void)root_graph_id->insert(root_graph_id->begin() + position, (*chunk_root_graph_id)[i][j]);
(void)root_graph_id->emplace(root_graph_id->begin() + position, (*chunk_root_graph_id)[i][j]);
}
(void)parameters->insert(parameters->begin() + position, (*chunk_parameters)[i][j]);
(void)error_codes->insert(error_codes->begin() + position, (*chunk_error_codes)[i][j]);
(void)parameters->emplace(parameters->begin() + position, (*chunk_parameters)[i][j]);
(void)error_codes->emplace(error_codes->begin() + position, (*chunk_error_codes)[i][j]);
}
// free the memory for used vectors
std::vector<int>().swap((*chunk_exec_orders)[i]);
@ -532,8 +556,9 @@ void DebugServices::SortWatchpointsInfo(
#ifdef OFFLINE_DBG_MODE
void DebugServices::ReadTensorFromNpy(const std::string &tensor_name, const std::string &file_name,
std::string *tensor_type, std::size_t *size, std::vector<int64_t> *shape,
std::vector<char> **data_buffer, bool *no_mem_to_read) {
std::string *const tensor_type, std::size_t *const size,
std::vector<int64_t> *const shape, std::vector<char> **const data_buffer,
bool *no_mem_to_read) {
std::ifstream infile;
std::string file_path = file_name;
MS_LOG(INFO) << "Reading in file: " << file_path;
@ -597,7 +622,7 @@ void DebugServices::ReadTensorFromNpy(const std::string &tensor_name, const std:
} else {
(void)infile.seekg(header_len + type_offset);
*data_buffer = new std::vector<char>(data_size);
if (data_buffer == nullptr || !infile.read((*data_buffer)->data(), data_size)) {
if ((*data_buffer) == nullptr || !infile.read((*data_buffer)->data(), data_size)) {
MS_LOG(ERROR) << "Unable to get tensor data from npy";
}
*size = data_size;
@ -605,7 +630,7 @@ void DebugServices::ReadTensorFromNpy(const std::string &tensor_name, const std:
}
void DebugServices::ConvertToHostFormat(const std::map<std::string, std::vector<std::string>> &dir_to_files_map,
std::vector<std::string> *result_list) {
std::vector<std::string> *const result_list) {
std::string file_format = "npy";
for (auto const &d : dir_to_files_map) {
std::vector<std::string> files_to_convert_in_dir;
@ -624,11 +649,12 @@ void DebugServices::ConvertToHostFormat(const std::map<std::string, std::vector<
for (std::string &file_found : *result_list) {
if (file_found.find(file_name_without_scope) != std::string::npos) {
already_converted = true;
break;
}
}
if (!already_converted) {
files_to_convert_in_dir.push_back(dump_key + "/" + file_name);
files_after_convert_in_dir.push_back(dump_key + "/" + file_name_without_scope);
(void)files_to_convert_in_dir.emplace_back(dump_key + "/" + file_name);
(void)files_after_convert_in_dir.emplace_back(dump_key + "/" + file_name_without_scope);
}
}
MS_LOG(INFO) << "Number of files to convert: " << files_to_convert_in_dir.size();
@ -642,46 +668,54 @@ void DebugServices::ConvertToHostFormat(const std::map<std::string, std::vector<
} catch (pybind11::error_already_set &e) {
MS_LOG(EXCEPTION) << "Failed to convert async dump data: " << e.what();
}
std::string abspath = RealPath(dump_key);
DIR *d_handle = opendir(abspath.c_str());
if (d_handle == nullptr) {
MS_LOG(ERROR) << "Directory does not exit in ConvertToHostFormat.";
return;
}
struct dirent *dir = nullptr;
while ((dir = readdir(d_handle)) != nullptr) {
if (dir->d_type == DT_REG) {
std::string candidate = dir->d_name;
for (const std::string &file_to_find : files_after_convert_in_dir) {
std::string file_n = file_to_find.substr(file_to_find.find_last_of("\\/") + 1);
if (candidate.find(file_n) != std::string::npos && candidate.rfind(file_format) != std::string::npos) {
// we found a converted file for this op
std::string found_file = dump_key + "/" + candidate;
if (std::find(result_list->begin(), result_list->end(), found_file) == result_list->end()) {
result_list->push_back(found_file);
}
}
}
}
}
(void)closedir(d_handle);
ProcessConvertToHostFormat(files_after_convert_in_dir, dump_key, result_list, file_format);
}
}
}
void GetNodeNameWithoutScope(std::string *dump_style_name) {
if (dump_style_name == nullptr) {
void DebugServices::ProcessConvertToHostFormat(const std::vector<std::string> &files_after_convert_in_dir,
const std::string &dump_key, std::vector<std::string> *const result_list,
const std::string &file_format) {
std::string real_dump_iter_dir = RealPath(dump_key);
DIR *d_handle = opendir(real_dump_iter_dir.c_str());
if (d_handle == nullptr) {
MS_LOG(ERROR) << "Directory does not exit in ConvertToHostFormat.";
return;
}
std::string node_name_without_scope = *dump_style_name;
struct dirent *dir = nullptr;
while ((dir = readdir(d_handle)) != nullptr) {
if (dir->d_type == DT_REG) {
std::string candidate = dir->d_name;
for (const std::string &file_to_find : files_after_convert_in_dir) {
std::string file_n = file_to_find;
auto last_slash_pos = file_to_find.find_last_of("\\/");
if (last_slash_pos != std::string::npos) {
file_n = file_to_find.substr(last_slash_pos + 1);
}
if (candidate.find(file_n) != std::string::npos && candidate.rfind(file_format) != std::string::npos) {
// we found a converted file for this op
std::string found_file = dump_key + "/" + candidate;
if (std::find(result_list->begin(), result_list->end(), found_file) == result_list->end()) {
result_list->push_back(found_file);
}
}
}
}
}
(void)closedir(d_handle);
}
std::string GetNodeNameWithoutScope(const std::string &dump_style_name) {
if (dump_style_name.empty()) {
return "";
}
std::size_t last_scope_marker;
std::string delim = "/";
last_scope_marker = node_name_without_scope.rfind(delim);
if (last_scope_marker != std::string::npos) {
node_name_without_scope = node_name_without_scope.substr(last_scope_marker + delim.size());
last_scope_marker = dump_style_name.rfind(delim);
if (last_scope_marker == std::string::npos) {
return dump_style_name;
}
*dump_style_name = node_name_without_scope;
return dump_style_name.substr(last_scope_marker + delim.size());
}
void ReplaceSrcFileName(std::string *dump_style_name) {
@ -702,7 +736,8 @@ void ReplaceSrcFileName(std::string *dump_style_name) {
void DebugServices::ConvertReadTensors(std::vector<std::string> backend_name, std::vector<size_t> slot,
std::vector<unsigned int> device_id, std::vector<unsigned int> iteration,
std::vector<unsigned int> root_graph_id, std::vector<std::string> *result_list) {
std::vector<unsigned int> root_graph_id,
std::vector<std::string> *const result_list) {
std::string file_format = "npy";
std::map<std::string, std::vector<std::string>> dir_to_files_map;
for (unsigned int i = 0; i < backend_name.size(); i++) {
@ -713,8 +748,7 @@ void DebugServices::ConvertReadTensors(std::vector<std::string> backend_name, st
std::size_t found_colon = dump_style_kernel_name.find_last_of(":");
dump_style_kernel_name = dump_style_kernel_name.substr(0, found_colon);
std::string prefix_dump_file_name = dump_style_kernel_name;
GetNodeNameWithoutScope(&prefix_dump_file_name);
std::string prefix_dump_file_name = GetNodeNameWithoutScope(dump_style_kernel_name);
std::string specific_dump_dir = dump_dir_ + "/rank_" + std::to_string(device_id[i]) + "/" + net_name_ + "/" +
std::to_string(root_graph_id[i]) + "/" + IterationString(iteration[i]);
@ -726,26 +760,7 @@ void DebugServices::ConvertReadTensors(std::vector<std::string> backend_name, st
MS_LOG(ERROR) << "Directory does not exist in ConvertReadTensors.";
return;
}
struct dirent *dir = nullptr;
while ((dir = readdir(d)) != nullptr) {
if (dir->d_type == DT_REG) {
std::string file_name = dir->d_name;
std::string file_name_w_o_perfix = file_name.substr(file_name.find('.') + 1);
if (file_name_w_o_perfix.rfind(prefix_dump_file_name) != std::string::npos &&
file_name.rfind(file_format) == std::string::npos) {
// if file matches prefix and is in device format add to candidate files to convert.
dir_to_files_map[specific_dump_dir].push_back(file_name);
} else if (file_name_w_o_perfix.rfind(prefix_dump_file_name) != std::string::npos &&
file_name.rfind(file_format) != std::string::npos) {
// otherwise, if file matches prefix and already has been converted to host format
// add to result of converted files.
std::string found_file = specific_dump_dir + "/" + file_name;
if (std::find(result_list->begin(), result_list->end(), found_file) == result_list->end()) {
result_list->push_back(found_file);
}
}
}
}
ProcessConvertList(prefix_dump_file_name, file_format, specific_dump_dir, &dir_to_files_map, result_list);
(void)closedir(d);
}
ConvertToHostFormat(dir_to_files_map, result_list);
@ -753,7 +768,7 @@ void DebugServices::ConvertReadTensors(std::vector<std::string> backend_name, st
void DebugServices::ConvertWatchPointNodes(const std::vector<std::tuple<std::string, std::string>> &proto_dump,
const std::string &specific_dump_dir,
std::vector<std::string> *result_list) {
std::vector<std::string> *const result_list) {
std::string file_format = "npy";
std::map<std::string, std::vector<std::string>> dir_to_files_map;
for (const auto &node : proto_dump) {
@ -766,35 +781,48 @@ void DebugServices::ConvertWatchPointNodes(const std::vector<std::tuple<std::str
MS_LOG(ERROR) << "Directory " << specific_dump_dir.c_str() << " does not exist in ConvertWatchPointNodes.";
return;
}
struct dirent *dir = nullptr;
while ((dir = readdir(d)) != nullptr) {
if (dir->d_type == DT_REG) {
std::string file_name = dir->d_name;
std::string file_name_w_o_perfix = file_name.substr(file_name.find('.') + 1);
if (file_name_w_o_perfix.rfind(dump_name) != std::string::npos &&
file_name.rfind(file_format) == std::string::npos) {
// if file matches prefix and is in device format add to candidate files to convert.
dir_to_files_map[specific_dump_dir].push_back(file_name);
} else if (file_name_w_o_perfix.rfind(dump_name) != std::string::npos &&
file_name.rfind(file_format) != std::string::npos) {
// otherwise, if file matches prefix and already has been converted to host format
// add to result of converted files.
std::string found_file = specific_dump_dir + "/" + file_name;
if (std::find(result_list->begin(), result_list->end(), found_file) == result_list->end()) {
result_list->push_back(found_file);
}
}
}
}
ProcessConvertList(dump_name, file_format, specific_dump_dir, &dir_to_files_map, result_list);
(void)closedir(d);
}
ConvertToHostFormat(dir_to_files_map, result_list);
}
void DebugServices::ProcessConvertList(const std::string &prefix_dump_file_name, const std::string &file_format,
const std::string &specific_dump_dir,
std::map<std::string, std::vector<std::string>> *dir_to_files_map,
std::vector<std::string> *const result_list) {
MS_EXCEPTION_IF_NULL(dir_to_files_map);
DIR *d = opendir(specific_dump_dir.c_str());
struct dirent *dir = nullptr;
while ((dir = readdir(d)) != nullptr) {
if (dir->d_type != DT_REG) {
continue;
}
std::string file_name = dir->d_name;
std::string file_name_w_o_perfix = file_name;
auto type_pos = file_name.find('.');
if (type_pos == std::string::npos || file_name.find(prefix_dump_file_name, type_pos + 1) == std::string::npos) {
continue;
}
if (file_name.rfind(file_format) == std::string::npos) {
// if file matches prefix and is in device format add to candidate files to convert.
(*dir_to_files_map)[specific_dump_dir].push_back(file_name);
} else {
// otherwise, if file matches prefix and already has been converted to host format
// add to result of converted files.
std::string found_file = specific_dump_dir + "/" + file_name;
if (std::find(result_list->begin(), result_list->end(), found_file) == result_list->end()) {
result_list->push_back(found_file);
}
}
}
(void)closedir(d);
}
void DebugServices::GetTensorDataInfoAsync(const std::vector<std::tuple<std::string, std::string>> &proto_dump,
const std::string &specific_dump_dir, uint32_t iteration, uint32_t device_id,
uint32_t root_graph_id, const std::vector<std::string> &async_file_pool,
std::vector<std::shared_ptr<TensorData>> *tensor_list) {
std::vector<std::shared_ptr<TensorData>> *const tensor_list) {
for (auto &node : proto_dump) {
std::vector<size_t> slot_list;
std::string dump_style_name = std::get<1>(node);
@ -841,7 +869,8 @@ void DebugServices::AddToTensorData(const std::string &backend_name, const std::
const std::size_t slot, const unsigned int iteration, const unsigned int device_id,
const unsigned int root_graph_id, const bool is_output, const std::size_t data_size,
const std::string &type_name, const std::vector<int64_t> &shape,
std::vector<char> *buffer, std::vector<std::shared_ptr<TensorData>> *result_list) {
std::vector<char> *buffer,
std::vector<std::shared_ptr<TensorData>> *const result_list) {
// call LoadNewTensor to store tensor in internal cache
auto tensor_data = std::make_shared<TensorData>();
tensor_data->SetName(backend_name);
@ -868,10 +897,10 @@ void DebugServices::AddToTensorData(const std::string &backend_name, const std::
result_list->push_back(tensor_data);
}
void DebugServices::SetPrefixToCheck(std::string *prefix_dump_file_name, std::string *slot_string_to_check,
std::string *dump_style_kernel_name, size_t slot, bool is_output) {
void DebugServices::SetPrefixToCheck(std::string *const prefix_dump_file_name, std::string *const slot_string_to_check,
std::string *const dump_style_kernel_name, size_t slot, bool is_output) {
std::string dump_style_name_part = *dump_style_kernel_name;
GetNodeNameWithoutScope(&dump_style_name_part);
dump_style_name_part = GetNodeNameWithoutScope(dump_style_name_part);
std::string slot_str;
if (is_output) {
slot_str = ".output." + std::to_string(slot);
@ -885,9 +914,8 @@ void DebugServices::SetPrefixToCheck(std::string *prefix_dump_file_name, std::st
std::string GetNewestFilePath(std::vector<std::string> file_list) {
// get file with the newest timestamp from the list.
std::string newest_file;
if (file_list.empty()) {
return newest_file;
return "";
}
std::sort(file_list.begin(), file_list.end());
return file_list.back();
@ -910,7 +938,8 @@ void DebugServices::ReadDumpedTensor(std::vector<std::string> backend_name, std:
std::vector<unsigned int> device_id, std::vector<unsigned int> iteration,
std::vector<unsigned int> root_graph_id, const std::vector<bool> &is_output,
const std::vector<std::string> &async_file_pool,
std::vector<std::shared_ptr<TensorData>> *result_list, bool *no_mem_to_read) {
std::vector<std::shared_ptr<TensorData>> *const result_list,
bool *no_mem_to_read) {
for (unsigned int i = 0; i < backend_name.size(); i++) {
// form prefix of the tensor file to read from graph pb node name
std::string dump_style_kernel_name = backend_name[i];
@ -922,8 +951,7 @@ void DebugServices::ReadDumpedTensor(std::vector<std::string> backend_name, std:
std::string slot_string_to_check;
std::string prefix_dump_file_name;
SetPrefixToCheck(&prefix_dump_file_name, &slot_string_to_check, &dump_style_kernel_name, slot[i], is_output[i]);
std::string prefix_dump_to_check = dump_style_kernel_name + '.';
GetNodeNameWithoutScope(&prefix_dump_to_check);
std::string prefix_dump_to_check = GetNodeNameWithoutScope(dump_style_kernel_name);
std::string specific_dump_dir = dump_dir_ + "/rank_" + std::to_string(device_id[i]) + "/" + net_name_ + "/" +
std::to_string(root_graph_id[i]) + "/" + IterationString(iteration[i]);
@ -1039,7 +1067,7 @@ std::string DebugServices::GetStrippedFilename(const std::string &file_name) {
size_t seventh_dot = file_name.rfind(".", file_name.rfind(".") - 1);
size_t fifth_dot = file_name.rfind(".", file_name.rfind(".", seventh_dot - 1) - 1);
if (fifth_dot == std::string::npos) {
if (fifth_dot == std::string::npos || fifth_dot <= first_dot) {
return std::string();
}
@ -1050,6 +1078,10 @@ std::string DebugServices::GetStrippedFilename(const std::string &file_name) {
second_dot = file_name.rfind(".", second_dot - 1);
}
if (second_dot == std::string::npos || second_dot <= first_dot) {
return std::string();
}
std::string start_string = file_name.substr(first_dot + 1, second_dot - first_dot - 1);
std::string end_string = file_name.substr(fifth_dot, seventh_dot - fifth_dot);
std::string stripped_file_name = start_string + end_string;
@ -1057,7 +1089,7 @@ std::string DebugServices::GetStrippedFilename(const std::string &file_name) {
}
std::vector<std::shared_ptr<TensorData>> DebugServices::ReadNeededDumpedTensors(
unsigned int iteration, std::vector<std::string> *async_file_pool) {
unsigned int iteration, std::vector<std::string> *const async_file_pool) {
// get a list of nodes and the devices they are on to monitor
std::vector<std::shared_ptr<TensorData>> tensor_list;
std::map<std::tuple<uint32_t, uint32_t>, std::vector<std::tuple<std::string, bool>>> device_and_graph_to_nodes;
@ -1093,9 +1125,8 @@ std::vector<std::shared_ptr<TensorData>> DebugServices::ReadNeededDumpedTensors(
// convert node names to dump style
for (auto node : wp_nodes) {
std::string orig_name = std::get<0>(node);
std::string dump_style_name = orig_name;
// Remove the scope from the fully qualified name to compare for both sync and async case.
GetNodeNameWithoutScope(&dump_style_name);
std::string dump_style_name = GetNodeNameWithoutScope(orig_name);
bool node_is_out = std::get<1>(node);
if (node_is_out) {
@ -1109,46 +1140,14 @@ std::vector<std::shared_ptr<TensorData>> DebugServices::ReadNeededDumpedTensors(
}
}
if (!is_sync_mode_) {
// convert all files in proto_to_dump to npy and add to pool of async file names
ConvertWatchPointNodes(proto_to_dump, specific_dump_dir, async_file_pool);
}
if (is_sync_mode_) {
// search files in dir for the one that meets the filename prefix and read the file into memory
std::string abspath = RealPath(specific_dump_dir);
DIR *d = opendir(abspath.c_str());
if (d == nullptr) {
MS_LOG(ERROR) << "Directory " << specific_dump_dir.c_str() << " does not exist in ReadNeededDumpedTensors.";
} else {
struct dirent *dir = nullptr;
while ((dir = readdir(d)) != nullptr) {
if (dir->d_type == DT_REG) {
std::string file_name = dir->d_name;
for (auto &node : proto_to_dump) {
std::string dump_name = std::get<1>(node);
std::string stripped_file_name = GetStrippedFilename(file_name);
if (stripped_file_name.empty()) {
continue;
}
std::size_t found = stripped_file_name.rfind(dump_name, 0);
if (found == 0) {
size_t slot = std::stoul(stripped_file_name.substr(dump_name.length() + 1));
std::vector<int64_t> shape;
std::string orig_name = std::get<0>(node);
std::string output_str = dump_name.substr(dump_name.rfind(".") + 1);
bool output_flag = (output_str == "output");
AddToTensorData(orig_name, "", slot, iteration, device_id, root_graph_id, output_flag, 0, "", shape,
NULL, &tensor_list);
break;
}
}
}
}
(void)closedir(d);
}
ProcessTensorDataSync(proto_to_dump, abspath, specific_dump_dir, iteration, device_id, root_graph_id,
&tensor_list);
} else {
// convert all files in proto_to_dump to npy and add to pool of async file names
ConvertWatchPointNodes(proto_to_dump, specific_dump_dir, async_file_pool);
GetTensorDataInfoAsync(proto_to_dump, specific_dump_dir, iteration, device_id, root_graph_id, *async_file_pool,
&tensor_list);
}
@ -1157,6 +1156,44 @@ std::vector<std::shared_ptr<TensorData>> DebugServices::ReadNeededDumpedTensors(
return tensor_list;
}
void DebugServices::ProcessTensorDataSync(const std::vector<std::tuple<std::string, std::string>> &proto_to_dump,
const std::string &abspath, const std::string &specific_dump_dir,
unsigned int iteration, unsigned int device_id, unsigned int root_graph_id,
std::vector<std::shared_ptr<TensorData>> *const tensor_list) {
DIR *d = opendir(abspath.c_str());
if (d == nullptr) {
MS_LOG(ERROR) << "Directory " << specific_dump_dir.c_str() << " does not exist in ReadNeededDumpedTensors.";
} else {
struct dirent *dir = nullptr;
while ((dir = readdir(d)) != nullptr) {
if (dir->d_type == DT_REG) {
std::string file_name = dir->d_name;
for (auto &node : proto_to_dump) {
std::string dump_name = std::get<1>(node);
std::string stripped_file_name = GetStrippedFilename(file_name);
if (stripped_file_name.empty() || stripped_file_name.length() <= dump_name.length()) {
continue;
}
std::size_t found = stripped_file_name.rfind(dump_name, 0);
if (found == 0) {
size_t slot = std::stoul(stripped_file_name.substr(dump_name.length() + 1));
std::vector<int64_t> shape;
std::string orig_name = std::get<0>(node);
std::string output_str = dump_name.substr(dump_name.rfind(".") + 1);
bool output_flag = (output_str == "output");
AddToTensorData(orig_name, "", slot, iteration, device_id, root_graph_id, output_flag, 0, "", shape,
nullptr, tensor_list);
break;
}
}
}
}
(void)closedir(d);
}
}
std::string DebugServices::IterationString(unsigned int iteration) {
std::string iteration_string;
bool init_dbg_suspend = (iteration == UINT_MAX);
@ -1180,11 +1217,11 @@ void DebugServices::ReadNodesTensors(const std::vector<std::string> &name, std::
if (std::get<1>(result) == nullptr) {
continue;
}
ret_name->push_back(std::get<0>(result));
data_ptr->push_back(reinterpret_cast<char *>(std::get<1>(result)->GetDataPtr()));
data_size->push_back(std::get<1>(result)->GetByteSize());
dtype->push_back(std::get<1>(result)->GetType());
shape->push_back(std::get<1>(result)->GetShape());
(void)ret_name->emplace_back(std::get<0>(result));
(void)data_ptr->emplace_back(reinterpret_cast<char *>(std::get<1>(result)->GetDataPtr()));
(void)data_size->emplace_back(std::get<1>(result)->GetByteSize());
(void)dtype->emplace_back(std::get<1>(result)->GetType());
(void)shape->emplace_back(std::get<1>(result)->GetShape());
}
}
@ -1328,7 +1365,7 @@ bool DebugServices::CheckOpOverflow(std::string node_name_to_find, unsigned int
// form fully qualified filename
std::string file_path = overflow_bin_path;
std::string file_name = dir->d_name;
file_path.append(file_name);
(void)file_path.append(file_name);
// attempt to read the file
std::ifstream infile;
infile.open(file_path.c_str(), std::ios::ate | std::ios::binary | std::ios::in);
@ -1402,8 +1439,8 @@ bool DebugServices::CheckOpOverflow(std::string node_name_to_find, unsigned int
return false;
}
bool DebugServices::GetAttrsFromAsyncFilename(const std::string &file_name, std::string *node_name, uint64_t *task_id,
uint64_t *stream_id) {
bool DebugServices::GetAttrsFromAsyncFilename(const std::string &file_name, std::string *const node_name,
uint64_t *task_id, uint64_t *stream_id) {
// get the node_name, task_id, and stream_id from async dump filename
// node_type.node_name.task_id.stram_id.timestamp
// WARNING: node_name may have dots in it
@ -1436,8 +1473,11 @@ bool DebugServices::GetAttrsFromAsyncFilename(const std::string &file_name, std:
std::string extracted_task_id = file_name.substr(second_dot + 1, third_dot - second_dot - 1);
try {
*task_id = std::stoull(extracted_task_id);
} catch (...) {
MS_LOG(ERROR) << "stoull failed on extracted_task_id to get task_id.";
} catch (std::invalid_argument &e) {
MS_LOG(ERROR) << "stoull failed on extracted_task_id to get task_id, invalid argument.";
return false;
} catch (std::out_of_range &e) {
MS_LOG(ERROR) << "stoull failed on extracted_task_id to get task_id, out of range.";
return false;
}
} else {
@ -1450,8 +1490,11 @@ bool DebugServices::GetAttrsFromAsyncFilename(const std::string &file_name, std:
std::string extracted_stream_id = file_name.substr(third_dot + 1, fourth_dot - third_dot - 1);
try {
*stream_id = std::stoull(extracted_stream_id);
} catch (...) {
MS_LOG(ERROR) << "stoull failed on extracted_stream_id to get stream_id.";
} catch (std::invalid_argument &e) {
MS_LOG(ERROR) << "stoull failed on extracted_stream_id to get stream_id, invalid argument.";
return false;
} catch (std::out_of_range &e) {
MS_LOG(ERROR) << "stoull failed on extracted_stream_id to get stream_id, out of range.";
return false;
}
} else {

54
mindspore/ccsrc/debug/debug_services.h
View File

@ -246,6 +246,20 @@ class DebugServices {
void RemoveWatchpoint(unsigned int id);
#ifdef OFFLINE_DBG_MODE
void ProcessCheckpointsOutofMemory(
const bool no_mem_to_read, const std::vector<watchpoint_t> watchpoints_to_check, const int chunk_id,
partitioned_names *const chunk_names, partitioned_names *const chunk_slots,
partitioned_numbers *const chunk_conditions, partitioned_id *const chunk_watchpoint_id,
partitioned_parameters *const chunk_parameters, partitioned_error_code *const chunk_error_codes,
partitioned_numbers *const chunk_exec_orders, partitioned_names *const chunk_time_stamp,
partitioned_id *const chunk_device_id, partitioned_id *const chunk_root_graph_id,
std::vector<unsigned int> *const device_id, std::vector<unsigned int> *const root_graph_id, const int exec_order,
const std::string time_stamp, const std::string &qualified_tensor_name, const std::string &tensor_slot,
const unsigned int device_id_val, const unsigned int root_graph_id_val,
const std::vector<parameter_t> &parameter_list);
#endif
void CheckWatchpointsForTensor(partitioned_names *chunk_names, partitioned_names *chunk_slots,
partitioned_numbers *chunk_conditions, partitioned_id *const chunk_watchpoint_id,
partitioned_parameters *chunk_parameters, partitioned_error_code *chunk_error_codes,
@ -299,16 +313,21 @@ class DebugServices {
const unsigned int iteration, const unsigned int device_id, const unsigned int root_graph_id,
const bool is_output, const std::size_t data_size, const std::string &type_name,
const std::vector<int64_t> &shape, std::vector<char> *buffer,
std::vector<std::shared_ptr<TensorData>> *result_list);
std::vector<std::shared_ptr<TensorData>> *const result_list);
void SetPrefixToCheck(std::string *prefix_dump_file_name, std::string *slot_string_to_check,
std::string *dump_style_kernel_name, size_t slot, bool is_output);
void SetPrefixToCheck(std::string *const prefix_dump_file_name, std::string *const slot_string_to_check,
std::string *const dump_style_kernel_name, size_t slot, bool is_output);
void ReadDumpedTensor(std::vector<std::string> backend_name, std::vector<size_t> slot,
std::vector<unsigned int> device_id, std::vector<unsigned int> iteration,
std::vector<unsigned int> root_graph_id, const std::vector<bool> &is_output,
const std::vector<std::string> &async_file_pool,
std::vector<std::shared_ptr<TensorData>> *result_list, bool *no_mem_to_read = nullptr);
std::vector<std::shared_ptr<TensorData>> *const result_list, bool *no_mem_to_read = nullptr);
void ProcessTensorDataSync(const std::vector<std::tuple<std::string, std::string>> &proto_to_dump,
const std::string &abspath, const std::string &specific_dump_dir, unsigned int iteration,
unsigned int device_id, unsigned int root_graph_id,
std::vector<std::shared_ptr<TensorData>> *const tensor_list);
void ReadDumpedTensorSync(const std::string &prefix_dump_file_name, const std::string &specific_dump_dir,
const std::string &backend_name, size_t slot, unsigned int device_id,
@ -322,29 +341,38 @@ class DebugServices {
std::vector<std::shared_ptr<TensorData>> *result_list, bool *no_mem_to_read);
std::vector<std::shared_ptr<TensorData>> ReadNeededDumpedTensors(unsigned int iteration,
std::vector<std::string> *async_file_pool);
std::vector<std::string> *const async_file_pool);
void *GetPrevTensor(const std::shared_ptr<TensorData> &tensor, bool previous_iter_tensor_needed,
uint32_t *prev_num_elements);
void ReadTensorFromNpy(const std::string &tensor_name, const std::string &file_name, std::string *tensor_type,
std::size_t *size, std::vector<int64_t> *shape, std::vector<char> **data_buffer,
bool *no_mem_to_read);
void ReadTensorFromNpy(const std::string &tensor_name, const std::string &file_name, std::string *const tensor_type,
std::size_t *const size, std::vector<int64_t> *const shape,
std::vector<char> **const data_buffer, bool *no_mem_to_read);
void ConvertToHostFormat(const std::map<std::string, std::vector<std::string>> &dir_to_files_map,
std::vector<std::string> *result_list);
std::vector<std::string> *const result_list);
void ProcessConvertToHostFormat(const std::vector<std::string> &files_after_convert_in_dir,
const std::string &dump_key, std::vector<std::string> *const result_list,
const std::string &file_format);
void ConvertReadTensors(std::vector<std::string> backend_name, std::vector<size_t> slot,
std::vector<unsigned int> device_id, std::vector<unsigned int> iteration,
std::vector<unsigned int> root_graph_id, std::vector<std::string> *result_list);
std::vector<unsigned int> root_graph_id, std::vector<std::string> *const result_list);
void ConvertWatchPointNodes(const std::vector<std::tuple<std::string, std::string>> &proto_dump,
const std::string &specific_dump_dir, std::vector<std::string> *result_list);
const std::string &specific_dump_dir, std::vector<std::string> *const result_list);
void ProcessConvertList(const std::string &prefix_dump_file_name, const std::string &file_format,
const std::string &specific_dump_dir,
std::map<std::string, std::vector<std::string>> *dir_to_files_map,
std::vector<std::string> *const result_list);
void GetTensorDataInfoAsync(const std::vector<std::tuple<std::string, std::string>> &proto_dump,
const std::string &specific_dump_dir, uint32_t iteration, uint32_t device_id,
uint32_t root_graph_id, const std::vector<std::string> &async_file_pool,
std::vector<std::shared_ptr<TensorData>> *tensor_list);
std::vector<std::shared_ptr<TensorData>> *const tensor_list);
std::string GetStrippedFilename(const std::string &file_name);
@ -385,7 +413,7 @@ class DebugServices {
bool CheckOpOverflow(std::string node_name_to_find, unsigned int device_id = 0, unsigned int root_graph_id = 0,
unsigned int iteration = 0);
bool GetAttrsFromAsyncFilename(const std::string &file_name, std::string *node_name, uint64_t *task_id,
bool GetAttrsFromAsyncFilename(const std::string &file_name, std::string *const node_name, uint64_t *task_id,
uint64_t *stream_id);
std::string RealPath(const std::string &input_path);

13
mindspore/ccsrc/debug/debugger/debugger.cc
View File

@ -123,10 +123,9 @@ void Debugger::Init(const uint32_t device_id, const std::string device_target) {
bool IsTypeDebuggerSupported(TypeId type) {
if (type < TypeId::kNumberTypeEnd && type > TypeId::kNumberTypeBegin && type != kNumberTypeComplex64) {
return true;
} else {
MS_LOG(INFO) << "Debugger does not support type: " << TypeIdLabel(type);
return false;
}
MS_LOG(INFO) << "Debugger does not support type: " << TypeIdLabel(type);
return false;
}
void Debugger::EnableDebugger() {
@ -949,9 +948,9 @@ void AddTensorStatInfo(const DebugServices::TensorStat &tensor_stat,
Statistics *tensor_statistics = tensor_summary_item.mutable_statistics();
tensor_statistics->set_is_bool(tensor_stat.is_bool);
tensor_statistics->set_max_value(tensor_stat.max_value);
tensor_statistics->set_min_value(tensor_stat.min_value);
tensor_statistics->set_avg_value(tensor_stat.avg_value);
tensor_statistics->set_max_value(static_cast<float>(tensor_stat.max_value));
tensor_statistics->set_min_value(static_cast<float>(tensor_stat.min_value));
tensor_statistics->set_avg_value(static_cast<float>(tensor_stat.avg_value));
tensor_statistics->set_count(tensor_stat.count);
tensor_statistics->set_neg_zero_count(tensor_stat.neg_zero_count);
tensor_statistics->set_pos_zero_count(tensor_stat.pos_zero_count);
@ -1058,7 +1057,7 @@ std::list<TensorBase> Debugger::LoadTensorsBase(const ProtoVector<TensorProto> &
// tensor was found creating tensor base object.
TensorBase tensor_base_item;
tensor_base_item.set_data_size((int64_t)tensor->GetByteSize());
tensor_base_item.set_data_type((int64_t)tensor->GetType());
tensor_base_item.set_data_type((int32_t)tensor->GetType());
for (auto elem : tensor->GetShape()) {
tensor_base_item.add_shape(elem);
}

4
mindspore/ccsrc/debug/debugger/grpc_client.h
View File

@ -37,16 +37,12 @@ using debugger::WatchpointHit;
namespace mindspore {
class GrpcClient {
public:
// constructor
GrpcClient(const std::string &host, const std::string &port);
// deconstructor
~GrpcClient() = default;
// init
void Init(const std::string &host, const std::string &port);
// reset
void Reset();
EventReply WaitForCommand(const Metadata &metadata);

1
mindspore/ccsrc/debug/debugger/offline_debug/dbg_services.cc
View File

@ -91,7 +91,6 @@ int32_t DbgServices::AddWatchpoint(
MS_LOG(DEBUG) << i << " ";
}
// std::vector<uint32_t> root_graph_id = std::get<std::vector<uint32_t>>(attr_map["root_graph_id"]);
std::vector<std::string> root_graph_id_str = std::get<std::vector<std::string>>(attr_map["root_graph_id"]);
std::vector<std::uint32_t> root_graph_id;
(void)std::transform(

41
mindspore/ccsrc/debug/debugger/tensor_summary.cc
View File

@ -83,9 +83,8 @@ double VarianceAndMeanCalculator::GetMean() const { return mean; }
double VarianceAndMeanCalculator::GetVariance() const {
if (count > 1) {
return m2 / (count - 1);
} else {
return 0.0;
}
return 0.0;
}
double VarianceAndMeanCalculator::GetStandardDeviation() { return sqrt(GetVariance()); }
@ -189,7 +188,7 @@ void TensorSummary<T>::TensorStatistics(DbgDataType dtype_value) {
sum_elements += current_value;
}
}
int value_count = zero_count_ + neg_zero_count_ + pos_zero_count_;
unsigned int value_count = zero_count_ + neg_zero_count_ + pos_zero_count_;
avg_ = sum_elements / value_count;
}
@ -254,27 +253,35 @@ double_t TensorSummary<T>::StatLookup(const std::string &parameter_name, const D
if (param_type == "max") {
return max_;
} else if (param_type == "min") {
}
if (param_type == "min") {
return min_;
} else if (param_type == "max_min") {
}
if (param_type == "max_min") {
return max_ - min_;
} else if (param_type == "mean") {
}
if (param_type == "mean") {
return current_mean_variance_.GetMean();
} else if (param_type == "sd") {
}
if (param_type == "sd") {
return current_mean_variance_.GetStandardDeviation();
} else if (param_type == "abs_mean") {
}
if (param_type == "abs_mean") {
if (means_.find("abs_current_mean") != means_.end()) {
return means_["abs_current_mean"]->GetMean();
}
} else if (param_type == "abs_mean_update_ratio" && prev_tensor_ptr_) {
}
if (param_type == "abs_mean_update_ratio" && prev_tensor_ptr_) {
if (means_.find("curr_prev_diff_mean") != means_.end() && means_.find("abs_prev_mean") != means_.end()) {
return means_["curr_prev_diff_mean"]->GetMean() / (means_["abs_prev_mean"]->GetMean() + epsilon_);
}
} else if (param_type == "range_percentage") {
}
if (param_type == "range_percentage") {
if (range_counts_.find(wp.id) != range_counts_.end()) {
return range_counts_[wp.id]->GetPercentInRange();
}
} else if (param_type == "zero_percentage") {
}
if (param_type == "zero_percentage") {
return GetZeroValPercent();
}
return std::numeric_limits<double_t>::quiet_NaN();
@ -285,13 +292,17 @@ double_t TensorSummary<T>::StatLookup(const DebugServices::watchpoint_t &wp) {
CONDITION_TYPE type = wp.condition.type;
if (type == CONDITION_TYPE::MAX_LT || type == CONDITION_TYPE::MAX_GT) {
return max_;
} else if (type == CONDITION_TYPE::MIN_LT || type == CONDITION_TYPE::MIN_GT) {
}
if (type == CONDITION_TYPE::MIN_LT || type == CONDITION_TYPE::MIN_GT) {
return min_;
} else if (type == CONDITION_TYPE::MEAN_LT || type == CONDITION_TYPE::MEAN_GT) {
}
if (type == CONDITION_TYPE::MEAN_LT || type == CONDITION_TYPE::MEAN_GT) {
return current_mean_variance_.GetMean();
} else if (type == CONDITION_TYPE::SD_LT || type == CONDITION_TYPE::SD_GT) {
}
if (type == CONDITION_TYPE::SD_LT || type == CONDITION_TYPE::SD_GT) {
return current_mean_variance_.GetStandardDeviation();
} else if (type == CONDITION_TYPE::MAX_MIN_GT || type == CONDITION_TYPE::MAX_MIN_LT) {
}
if (type == CONDITION_TYPE::MAX_MIN_GT || type == CONDITION_TYPE::MAX_MIN_LT) {
return max_ - min_;
}
return std::numeric_limits<double_t>::quiet_NaN();

20
mindspore/ccsrc/debug/tensor_data.h
View File

@ -422,22 +422,22 @@ class TensorData {
}
private:
char *data_ptr_; // pointer to the pre-allocated memory
uint64_t size_; // size_ in bytes
DbgDataType data_type_; // internal debugger type
unsigned int data_type_size_;
char *data_ptr_{nullptr}; // pointer to the pre-allocated memory
uint64_t size_{0}; // size_ in bytes
DbgDataType data_type_{DbgDataType::DT_UNDEFINED}; // internal debugger type
unsigned int data_type_size_{0};
std::vector<int64_t> shape_;
std::string name_;
uint64_t slot_;
unsigned int iteration_;
unsigned int device_id_;
unsigned int root_graph_id_;
bool is_output_;
int execution_order_;
unsigned int iteration_{0};
unsigned int device_id_{0};
unsigned int root_graph_id_{0};
bool is_output_{true};
int execution_order_{-1};
std::string time_stamp_;
#ifdef ONLINE_DBG_MODE
mindspore::tensor::TensorPtr tensor_ptr_;
mindspore::tensor::TensorPtr tensor_ptr_{nullptr};
#endif
};
#ifdef ONLINE_DBG_MODE