pgjmblvch
/
mindspore2022
forked from huawei/mindspore2022
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

change arg 'is_parameter' to 'is_output' and rename device_id to rank_id in python api

This commit is contained in:
TinaMengtingZhang 2021-06-16 13:30:36 -04:00
parent 35c1f14cf3
commit 4e8db5e115
8 changed files with 228 additions and 229 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

205
mindspore/ccsrc/debug/debug_services.cc
View File

@ -140,7 +140,8 @@ void *DebugServices::GetPrevTensor(const std::shared_ptr<TensorData> &tensor, bo
ReadDumpedTensor(std::vector<std::string>{tensor->GetName()}, std::vector<size_t>{tensor->GetSlot()},
std::vector<unsigned int>{tensor->GetDeviceId()},
std::vector<unsigned int>{tensor->GetIteration() - 1},
std::vector<unsigned int>{tensor->GetRootGraphId()}, file_paths, &result_list_prev);
std::vector<unsigned int>{tensor->GetRootGraphId()}, std::vector<bool>{tensor->GetIsOutput()},
file_paths, &result_list_prev);
tensor_prev = result_list_prev[0];
if (!tensor_prev->GetByteSize()) {
tensor_prev.reset();
@ -206,7 +207,8 @@ void DebugServices::CheckWatchpointsForTensor(
ReadDumpedTensor(std::vector<std::string>{tensor->GetName()}, std::vector<size_t>{tensor->GetSlot()},
std::vector<unsigned int>{tensor->GetDeviceId()},
std::vector<unsigned int>{tensor->GetIteration()},
std::vector<unsigned int>{tensor->GetRootGraphId()}, async_file_pool, &result_list);
std::vector<unsigned int>{tensor->GetRootGraphId()}, std::vector<bool>{tensor->GetIsOutput()},
async_file_pool, &result_list);
tensor = result_list[0];
if (!tensor->GetByteSize()) {
tensor.reset();
@ -467,7 +469,10 @@ void DebugServices::ConvertToHostFormat(const std::map<std::string, std::vector<
std::string file_n = file_to_find.substr(file_to_find.find_last_of("\\/") + 1);
if (candidate.find(file_n) != std::string::npos && candidate.rfind(file_format) != std::string::npos) {
// we found a converted file for this op
result_list->push_back(dump_key + "/" + candidate);
std::string found_file = dump_key + "/" + candidate;
if (std::find(result_list->begin(), result_list->end(), found_file) == result_list->end()) {
result_list->push_back(found_file);
}
}
}
}
@ -477,6 +482,36 @@ void DebugServices::ConvertToHostFormat(const std::map<std::string, std::vector<
}
}
void GetNodeNameWithoutScope(std::string *dump_style_name) {
if (dump_style_name == nullptr) {
return;
}
std::string node_name_without_scope = *dump_style_name;
std::size_t last_scope_marker;
std::string delim = "/";
last_scope_marker = node_name_without_scope.rfind(delim);
if (last_scope_marker != std::string::npos) {
node_name_without_scope = node_name_without_scope.substr(last_scope_marker + delim.size());
}
*dump_style_name = node_name_without_scope;
}
void ReplaceSrcFileName(std::string *dump_style_name) {
if (dump_style_name == nullptr) {
return;
}
const std::string strsrc = "/";
std::string strdst = "_";
std::string::size_type pos = 0;
std::string::size_type srclen = strsrc.size();
std::string::size_type dstlen = strdst.size();
while ((pos = dump_style_name->find(strsrc, pos)) != std::string::npos) {
dump_style_name->replace(pos, srclen, strdst);
pos += dstlen;
}
}
void DebugServices::ConvertReadTensors(std::vector<std::string> backend_name, std::vector<size_t> slot,
std::vector<unsigned int> device_id, std::vector<unsigned int> iteration,
std::vector<unsigned int> root_graph_id, std::vector<std::string> *result_list) {
@ -485,23 +520,12 @@ void DebugServices::ConvertReadTensors(std::vector<std::string> backend_name, st
for (unsigned int i = 0; i < backend_name.size(); i++) {
// form prefix of the tensor file to read from graph pb node name
std::string dump_style_kernel_name = backend_name[i];
const std::string strsrc = "/";
std::string strdst = "_";
std::string::size_type pos = 0;
std::string::size_type srclen = strsrc.size();
std::string::size_type dstlen = strdst.size();
ReplaceSrcFileName(&dump_style_kernel_name);
// remove slot from name
std::size_t found_colon = dump_style_kernel_name.find_last_of(":");
dump_style_kernel_name = dump_style_kernel_name.substr(0, found_colon);
while ((pos = dump_style_kernel_name.find(strsrc, pos)) != std::string::npos) {
dump_style_kernel_name.replace(pos, srclen, strdst);
pos += dstlen;
}
std::string prefix_dump_file_name = dump_style_kernel_name;
std::string specific_dump_dir = dump_dir + "/rank_" + std::to_string(device_id[i]) + "/" + net_name + "/" +
@ -524,7 +548,10 @@ void DebugServices::ConvertReadTensors(std::vector<std::string> backend_name, st
file_name.rfind(file_format) != std::string::npos) {
// otherwise, if file matches prefix and already has been converted to host format
// add to result of converted files.
result_list->push_back(specific_dump_dir + "/" + file_name);
std::string found_file = specific_dump_dir + "/" + file_name;
if (std::find(result_list->begin(), result_list->end(), found_file) == result_list->end()) {
result_list->push_back(found_file);
}
}
}
}
@ -541,6 +568,7 @@ void DebugServices::ConvertWatchPointNodes(const std::vector<std::tuple<std::str
std::map<std::string, std::vector<std::string>> dir_to_files_map;
for (const auto &node : proto_dump) {
std::string dump_name = std::get<1>(node);
dump_name = dump_name.substr(0, dump_name.rfind("."));
// search files in dir for the one that meets the filename prefix and read the file into memory
DIR *d;
d = opendir(specific_dump_dir.c_str());
@ -557,7 +585,10 @@ void DebugServices::ConvertWatchPointNodes(const std::vector<std::tuple<std::str
file_name.rfind(file_format) != std::string::npos) {
// otherwise, if file matches prefix and already has been converted to host format
// add to result of converted files.
result_list->push_back(specific_dump_dir + "/" + file_name);
std::string found_file = specific_dump_dir + "/" + file_name;
if (std::find(result_list->begin(), result_list->end(), found_file) == result_list->end()) {
result_list->push_back(found_file);
}
}
}
}
@ -573,10 +604,16 @@ void DebugServices::GetTensorDataInfoAsync(const std::vector<std::tuple<std::str
std::vector<std::shared_ptr<TensorData>> *tensor_list) {
for (auto &node : proto_dump) {
std::vector<size_t> slot_list;
std::string dump_style_name = std::get<1>(node);
// Get dump_name and output_str from the second element of tuple
std::size_t found_dot = dump_style_name.rfind(".");
std::string dump_name = dump_style_name.substr(0, found_dot);
std::string output_str = dump_style_name.substr(found_dot + 1);
bool output_flag = (output_str == "output");
for (const std::string &file_name : async_file_pool) {
std::string dump_name = std::get<1>(node);
std::size_t found = file_name.find(dump_name);
std::size_t found_out = file_name.find("output");
std::size_t found_out = file_name.find(output_str);
std::size_t found_dot_start = file_name.find(".", found_out);
std::size_t found_dot_end = file_name.find(".", found_dot_start);
@ -599,6 +636,7 @@ void DebugServices::GetTensorDataInfoAsync(const std::vector<std::tuple<std::str
tensor_data->SetByteSize(0);
tensor_data->SetType("");
tensor_data->SetShape(shape);
tensor_data->SetIsOutput(output_flag);
tensor_list->push_back(tensor_data);
}
@ -607,7 +645,7 @@ void DebugServices::GetTensorDataInfoAsync(const std::vector<std::tuple<std::str
void DebugServices::AddToTensorData(const std::string &backend_name, const std::size_t slot,
const unsigned int iteration, const unsigned int device_id,
const unsigned int root_graph_id, const std::size_t data_size,
const unsigned int root_graph_id, const bool is_output, const std::size_t data_size,
const std::string &type_name, const std::vector<int64_t> &shape,
std::vector<char> *buffer, std::vector<std::shared_ptr<TensorData>> *result_list) {
// call LoadNewTensor to store tensor in internal cache
@ -618,6 +656,7 @@ void DebugServices::AddToTensorData(const std::string &backend_name, const std::
tensor_data->SetIteration(iteration);
tensor_data->SetDeviceId(device_id);
tensor_data->SetRootGraphId(root_graph_id);
tensor_data->SetIsOutput(is_output);
if (data_size) {
tensor_data->SetDataPtr(buffer->data());
} else {
@ -635,57 +674,32 @@ void DebugServices::AddToTensorData(const std::string &backend_name, const std::
}
void DebugServices::SetPrefixToCheck(std::string *prefix_dump_file_name, std::string *dump_style_kernel_name,
size_t slot) {
size_t slot, bool is_output) {
std::string dump_style_name_part = *dump_style_kernel_name;
std::size_t last_scope_marker;
std::string delim;
if (is_sync_mode) {
delim = "--";
} else {
delim = "_";
}
last_scope_marker = dump_style_kernel_name->rfind(delim);
if (last_scope_marker != std::string::npos) {
dump_style_name_part = dump_style_kernel_name->substr(last_scope_marker + delim.size());
}
if (is_sync_mode) {
GetNodeNameWithoutScope(&dump_style_name_part);
if (is_output) {
dump_style_name_part += ".output." + std::to_string(slot);
} else {
dump_style_name_part += ".input." + std::to_string(slot);
}
*prefix_dump_file_name = dump_style_name_part;
}
void DebugServices::ReadDumpedTensor(std::vector<std::string> backend_name, std::vector<size_t> slot,
std::vector<unsigned int> device_id, std::vector<unsigned int> iteration,
std::vector<unsigned int> root_graph_id,
std::vector<unsigned int> root_graph_id, const std::vector<bool> &is_output,
const std::vector<std::string> &async_file_pool,
std::vector<std::shared_ptr<TensorData>> *result_list) {
for (unsigned int i = 0; i < backend_name.size(); i++) {
// form prefix of the tensor file to read from graph pb node name
std::string dump_style_kernel_name = backend_name[i];
const std::string strsrc = "/";
std::string strdst;
if (is_sync_mode) {
strdst = "--";
} else {
strdst = "_";
}
std::string::size_type pos = 0;
std::string::size_type srclen = strsrc.size();
std::string::size_type dstlen = strdst.size();
// remove slot from name
std::size_t found_colon = dump_style_kernel_name.find_last_of(":");
dump_style_kernel_name = dump_style_kernel_name.substr(0, found_colon);
while ((pos = dump_style_kernel_name.find(strsrc, pos)) != std::string::npos) {
dump_style_kernel_name.replace(pos, srclen, strdst);
pos += dstlen;
}
std::string prefix_dump_file_name;
SetPrefixToCheck(&prefix_dump_file_name, &dump_style_kernel_name, slot[i]);
SetPrefixToCheck(&prefix_dump_file_name, &dump_style_kernel_name, slot[i], is_output[i]);
std::string specific_dump_dir = dump_dir + "/rank_" + std::to_string(device_id[i]) + "/" + net_name + "/" +
std::to_string(root_graph_id[i]) + "/" + std::to_string(iteration[i]);
@ -717,14 +731,14 @@ void DebugServices::ReadDumpedTensor(std::vector<std::string> backend_name, std:
shape.clear();
std::string full_path = specific_dump_dir + "/" + file_name;
ReadTensorFromNpy(full_path, &type_name, &data_size, &shape, &buffer);
AddToTensorData(backend_name[i], slot[i], iteration[i], device_id[i], root_graph_id[i], data_size,
type_name, shape, buffer, result_list);
AddToTensorData(backend_name[i], slot[i], iteration[i], device_id[i], root_graph_id[i], is_output[i],
data_size, type_name, shape, buffer, result_list);
found_file = true;
}
}
if (!found_file) {
AddToTensorData(backend_name[i], slot[i], iteration[i], device_id[i], root_graph_id[i], 0, type_name, shape,
buffer, result_list);
AddToTensorData(backend_name[i], slot[i], iteration[i], device_id[i], root_graph_id[i], is_output[i], 0,
type_name, shape, buffer, result_list);
}
} else {
MS_LOG(INFO) << "directory does not exist!";
@ -734,19 +748,19 @@ void DebugServices::ReadDumpedTensor(std::vector<std::string> backend_name, std:
bool found = false;
// if async mode
for (const std::string &file_path : async_file_pool) {
if (file_path.find(prefix_dump_file_name) != std::string::npos &&
file_path.find(".output." + std::to_string(slot[i])) != std::string::npos) {
std::string stripped_file_name = GetStrippedFilename(file_path);
if (stripped_file_name.find(prefix_dump_file_name) != std::string::npos) {
found = true;
shape.clear();
ReadTensorFromNpy(file_path, &type_name, &data_size, &shape, &buffer);
AddToTensorData(backend_name[i], slot[i], iteration[i], device_id[i], root_graph_id[i], data_size, type_name,
shape, buffer, result_list);
AddToTensorData(backend_name[i], slot[i], iteration[i], device_id[i], root_graph_id[i], is_output[i],
data_size, type_name, shape, buffer, result_list);
}
}
// If no npy file is found, add empty tensor data.
if (!found) {
AddToTensorData(backend_name[i], slot[i], iteration[i], device_id[i], root_graph_id[i], 0, type_name, shape,
buffer, result_list);
AddToTensorData(backend_name[i], slot[i], iteration[i], device_id[i], root_graph_id[i], is_output[i], 0,
type_name, shape, buffer, result_list);
}
}
}
@ -775,51 +789,21 @@ std::string DebugServices::GetStrippedFilename(const std::string &file_name) {
return stripped_file_name;
}
void ReplaceSrcFileName(const bool is_sync_mode, std::string *dump_style_name) {
if (dump_style_name == nullptr) {
return;
}
const std::string strsrc = "/";
std::string strdst;
if (is_sync_mode) {
strdst = "--";
} else {
strdst = "_";
}
std::string::size_type pos = 0;
std::string::size_type srclen = strsrc.size();
std::string::size_type dstlen = strdst.size();
while ((pos = dump_style_name->find(strsrc, pos)) != std::string::npos) {
dump_style_name->replace(pos, srclen, strdst);
pos += dstlen;
}
}
std::vector<std::shared_ptr<TensorData>> DebugServices::ReadNeededDumpedTensors(
unsigned int iteration, std::vector<std::string> *async_file_pool) {
// get a list of nodes and the devices they are on to monitor
std::vector<std::shared_ptr<TensorData>> tensor_list;
std::map<std::tuple<uint32_t, uint32_t>, std::unordered_set<std::string>> device_and_graph_to_nodes;
std::map<std::tuple<uint32_t, uint32_t>, std::vector<std::tuple<std::string, bool>>> device_and_graph_to_nodes;
for (auto w_table_item : watchpoint_table) {
auto wp = std::get<1>(w_table_item);
for (auto check_node : wp.check_node_list) {
unsigned int index = 0;
std::string w_name = std::get<0>(check_node);
bool w_is_param = std::get<1>(check_node);
std::string node_name = w_name;
if (w_is_param) {
std::size_t found = node_name.find_last_of("/");
node_name = node_name.substr(found + 1);
}
std::vector<uint32_t> devices = std::get<1>(wp.check_node_device_list[index]);
std::vector<uint32_t> graphs = std::get<1>(wp.check_node_graph_list[index]);
for (auto device : devices) {
for (auto graph : graphs) {
std::tuple<uint32_t, uint32_t> key(device, graph);
device_and_graph_to_nodes[key].insert(node_name);
device_and_graph_to_nodes[key].push_back(check_node);
}
}
@ -833,7 +817,7 @@ std::vector<std::shared_ptr<TensorData>> DebugServices::ReadNeededDumpedTensors(
std::tuple<uint32_t, uint32_t> device_and_graph = device_and_graph_item.first;
uint32_t device_id = std::get<0>(device_and_graph);
uint32_t root_graph_id = std::get<1>(device_and_graph);
std::unordered_set<std::string> wp_nodes = device_and_graph_item.second;
std::vector<std::tuple<std::string, bool>> wp_nodes = device_and_graph_item.second;
std::vector<std::tuple<std::string, std::string>> proto_to_dump;
std::string specific_dump_dir = dump_dir + "/rank_" + std::to_string(device_id) + "/" + net_name + "/" +
@ -841,17 +825,22 @@ std::vector<std::shared_ptr<TensorData>> DebugServices::ReadNeededDumpedTensors(
// convert node names to dump style
for (auto node : wp_nodes) {
std::string orig_name = node;
std::string dump_style_name = node;
ReplaceSrcFileName(is_sync_mode, &dump_style_name);
std::string orig_name = std::get<0>(node);
std::string dump_style_name = orig_name;
if (is_sync_mode) {
std::string dump_style_name_part = dump_style_name;
std::size_t last_scope_marker = dump_style_name.rfind("--");
if (last_scope_marker != std::string::npos) {
dump_style_name_part = dump_style_name.substr(last_scope_marker + 2);
}
dump_style_name = dump_style_name_part + ".output.";
// In sync mode, remove the scope from the fully qualified name to compare.
GetNodeNameWithoutScope(&dump_style_name);
} else {
// In async mode, keep the scope but replace delimiter with '_' in node name to compare.
ReplaceSrcFileName(&dump_style_name);
}
bool node_is_out = std::get<1>(node);
if (node_is_out) {
dump_style_name += ".output";
} else {
dump_style_name += ".input";
}
proto_to_dump.push_back(std::tuple<std::string, std::string>(orig_name, dump_style_name));
@ -880,10 +869,14 @@ std::vector<std::shared_ptr<TensorData>> DebugServices::ReadNeededDumpedTensors(
std::size_t found = stripped_file_name.rfind(dump_name, 0);
if (found == 0) {
size_t slot = std::stoul(stripped_file_name.substr(dump_name.length()));
size_t slot = std::stoul(stripped_file_name.substr(dump_name.length() + 1));
std::vector<int64_t> shape;
std::string orig_name = std::get<0>(node);
AddToTensorData(orig_name, slot, iteration, device_id, root_graph_id, 0, "", shape, NULL, &tensor_list);
std::string output_str = dump_name.substr(dump_name.rfind(".") + 1);
bool output_flag = (output_str == "output");
AddToTensorData(orig_name, slot, iteration, device_id, root_graph_id, output_flag, 0, "", shape, NULL,
&tensor_list);
break;
}
}

12
mindspore/ccsrc/debug/debug_services.h
View File

@ -222,15 +222,17 @@ class DebugServices {
#ifdef OFFLINE_DBG_MODE
void AddToTensorData(const std::string &backend_name, const std::size_t slot, const unsigned int iteration,
const unsigned int device_id, const unsigned int root_graph_id, const std::size_t data_size,
const std::string &type_name, const std::vector<int64_t> &shape, std::vector<char> *buffer,
std::vector<std::shared_ptr<TensorData>> *result_list);
const unsigned int device_id, const unsigned int root_graph_id, const bool is_output,
const std::size_t data_size, const std::string &type_name, const std::vector<int64_t> &shape,
std::vector<char> *buffer, std::vector<std::shared_ptr<TensorData>> *result_list);
void SetPrefixToCheck(std::string *prefix_dump_file_name, std::string *dump_style_kernel_name, size_t slot);
void SetPrefixToCheck(std::string *prefix_dump_file_name, std::string *dump_style_kernel_name, size_t slot,
bool is_output);
void ReadDumpedTensor(std::vector<std::string> backend_name, std::vector<size_t> slot,
std::vector<unsigned int> device_id, std::vector<unsigned int> iteration,
std::vector<unsigned int> root_graph_id, const std::vector<std::string> &async_file_pool,
std::vector<unsigned int> root_graph_id, const std::vector<bool> &is_output,
const std::vector<std::string> &async_file_pool,
std::vector<std::shared_ptr<TensorData>> *result_list);
std::vector<std::shared_ptr<TensorData>> ReadNeededDumpedTensors(unsigned int iteration,

67
mindspore/ccsrc/debug/debugger/offline_debug/dbg_services.cc
View File

@ -73,16 +73,15 @@ int32_t DbgServices::AddWatchpoint(
MS_LOG(INFO) << "cpp DbgServices AddWatchpoint name " << node.first;
auto attr_map = node.second;
bool is_parameter = std::get<bool>(attr_map["is_parameter"]);
MS_LOG(INFO) << "cpp DbgServices AddWatchpoint is_parameter " << is_parameter;
bool is_output = std::get<bool>(attr_map["is_output"]);
MS_LOG(INFO) << "cpp DbgServices AddWatchpoint is_output " << is_output;
// std::vector<uint32_t> device_id = std::get<std::vector<uint32_t>>(attr_map["device_id"]);
std::vector<std::string> device_id_str = std::get<std::vector<std::string>>(attr_map["device_id"]);
std::vector<std::uint32_t> device_id;
std::transform(device_id_str.begin(), device_id_str.end(), std::back_inserter(device_id),
std::vector<std::string> rank_id_str = std::get<std::vector<std::string>>(attr_map["rank_id"]);
std::vector<std::uint32_t> rank_id;
std::transform(rank_id_str.begin(), rank_id_str.end(), std::back_inserter(rank_id),
[](std::string &id_str) -> std::uint32_t { return static_cast<uint32_t>(std::stoul(id_str)); });
MS_LOG(INFO) << "cpp DbgServices AddWatchpoint device_id ";
for (auto const &i : device_id) {
MS_LOG(INFO) << "cpp DbgServices AddWatchpoint rank_id ";
for (auto const &i : rank_id) {
MS_LOG(INFO) << i << " ";
}
@ -114,18 +113,18 @@ int32_t DbgServices::AddWatchpoint(
std::transform(check_nodes.begin(), check_nodes.end(), std::back_inserter(check_node_list),
[](auto &node) -> std::tuple<std::string, bool> {
auto attr_map = node.second;
return std::make_tuple(node.first, std::get<bool>(attr_map["is_parameter"]));
return std::make_tuple(node.first, std::get<bool>(attr_map["is_output"]));
});
std::transform(check_nodes.begin(), check_nodes.end(), std::back_inserter(check_node_device_list),
[](auto &node) -> std::tuple<std::string, std::vector<uint32_t>> {
auto attr_map = node.second;
std::vector<std::string> device_id_str = std::get<std::vector<std::string>>(attr_map["device_id"]);
std::vector<std::uint32_t> device_id;
std::vector<std::string> rank_id_str = std::get<std::vector<std::string>>(attr_map["rank_id"]);
std::vector<std::uint32_t> rank_id;
std::transform(
device_id_str.begin(), device_id_str.end(), std::back_inserter(device_id),
rank_id_str.begin(), rank_id_str.end(), std::back_inserter(rank_id),
[](std::string &id_str) -> std::uint32_t { return static_cast<uint32_t>(std::stoul(id_str)); });
return std::make_tuple(node.first, device_id);
return std::make_tuple(node.first, rank_id);
});
std::transform(
@ -168,7 +167,7 @@ std::vector<watchpoint_hit_t> DbgServices::CheckWatchpoints(unsigned int iterati
std::vector<std::string> overflow_ops;
std::vector<std::vector<DebugServices::parameter_t>> parameters;
std::vector<int32_t> error_codes;
std::vector<unsigned int> device_id;
std::vector<unsigned int> rank_id;
std::vector<unsigned int> root_graph_id;
// #ifdef ENABLE_D
// overflow_ops = CheckOpOverflow();
@ -180,7 +179,7 @@ std::vector<watchpoint_hit_t> DbgServices::CheckWatchpoints(unsigned int iterati
tensor_list = debug_services->ReadNeededDumpedTensors(iteration, &file_paths);
debug_services->CheckWatchpoints(&name, &slot, &condition, &watchpoint_id, &parameters, &error_codes, overflow_ops,
file_paths, &tensor_list, false, true, true, &device_id, &root_graph_id);
file_paths, &tensor_list, false, true, true, &rank_id, &root_graph_id);
std::vector<watchpoint_hit_t> hits;
for (unsigned int i = 0; i < name.size(); i++) {
@ -191,13 +190,13 @@ std::vector<watchpoint_hit_t> DbgServices::CheckWatchpoints(unsigned int iterati
api_parameter_vector.push_back(api_parameter);
}
watchpoint_hit_t hit(name[i], std::stoi(slot[i]), condition[i], watchpoint_id[i], api_parameter_vector,
error_codes[i], device_id[i], root_graph_id[i]);
error_codes[i], rank_id[i], root_graph_id[i]);
MS_LOG(INFO) << "cpp DbgServices watchpoint_hit_t name " << hit.name;
MS_LOG(INFO) << "cpp DbgServices watchpoint_hit_t slot " << hit.slot;
MS_LOG(INFO) << "cpp DbgServices watchpoint_hit_t watchpoint_id " << hit.watchpoint_id;
MS_LOG(INFO) << "cpp DbgServices watchpoint_hit_t error_code " << hit.error_code;
MS_LOG(INFO) << "cpp DbgServices watchpoint_hit_t device_id " << hit.device_id;
MS_LOG(INFO) << "cpp DbgServices watchpoint_hit_t rank_id " << hit.rank_id;
MS_LOG(INFO) << "cpp DbgServices watchpoint_hit_t root_graph_id " << hit.root_graph_id;
for (auto const &parameter_i : api_parameter_vector) {
@ -213,20 +212,9 @@ std::vector<watchpoint_hit_t> DbgServices::CheckWatchpoints(unsigned int iterati
return hits;
}
std::string GetTensorFullName(tensor_info_t info) {
std::string node_name = info.node_name;
if (info.is_parameter) {
// scopes in node name are separated by '/'
// use the name without scope if truncate is true
auto found = node_name.find_last_of("/");
if (found != std::string::npos) {
node_name = node_name.substr(found + 1);
}
}
return node_name + ":" + std::to_string(info.slot);
}
std::string GetTensorFullName(tensor_info_t info) { return info.node_name + ":" + std::to_string(info.slot); }
unsigned int GetTensorDeviceId(tensor_info_t info) { return info.device_id; }
unsigned int GetTensorRankId(tensor_info_t info) { return info.rank_id; }
unsigned int GetTensorRootGraphId(tensor_info_t info) { return info.root_graph_id; }
@ -234,30 +222,39 @@ unsigned int GetTensorIteration(tensor_info_t info) { return info.iteration; }
unsigned int GetTensorSlot(tensor_info_t info) { return info.slot; }
bool GetTensorIsOutput(tensor_info_t info) { return info.is_output; }
std::vector<tensor_data_t> DbgServices::ReadTensors(std::vector<tensor_info_t> info) {
for (auto i : info) {
MS_LOG(INFO) << "cpp DbgServices ReadTensor info name " << i.node_name << ", slot " << i.slot << ", iteration "
<< i.iteration << ", device_id " << i.device_id << ", root_graph_id " << i.root_graph_id;
<< i.iteration << ", rank_id " << i.rank_id << ", root_graph_id " << i.root_graph_id << ", is_output "
<< i.is_output;
}
std::vector<std::string> backend_name;
std::vector<unsigned int> device_id;
std::vector<unsigned int> rank_id;
std::vector<unsigned int> root_graph_id;
std::vector<unsigned int> iteration;
std::vector<size_t> slot;
std::vector<std::shared_ptr<TensorData>> result_list;
std::vector<tensor_data_t> tensors_read;
std::vector<bool> is_output;
std::transform(info.begin(), info.end(), std::back_inserter(backend_name), GetTensorFullName);
std::transform(info.begin(), info.end(), std::back_inserter(slot), GetTensorSlot);
std::transform(info.begin(), info.end(), std::back_inserter(device_id), GetTensorDeviceId);
std::transform(info.begin(), info.end(), std::back_inserter(rank_id), GetTensorRankId);
std::transform(info.begin(), info.end(), std::back_inserter(root_graph_id), GetTensorRootGraphId);
std::transform(info.begin(), info.end(), std::back_inserter(iteration), GetTensorIteration);
std::transform(info.begin(), info.end(), std::back_inserter(is_output), GetTensorIsOutput);
MS_LOG(INFO) << "cpp before";
std::vector<std::string> file_paths;
auto t1 = std::chrono::high_resolution_clock::now();
debug_services->ConvertReadTensors(backend_name, slot, device_id, iteration, root_graph_id, &file_paths);
debug_services->ReadDumpedTensor(backend_name, slot, device_id, iteration, root_graph_id, file_paths, &result_list);
// Convert the dumped data to npy format if it's async mode.
if (!debug_services->GetSyncMode()) {
debug_services->ConvertReadTensors(backend_name, slot, rank_id, iteration, root_graph_id, &file_paths);
}
debug_services->ReadDumpedTensor(backend_name, slot, rank_id, iteration, root_graph_id, is_output, file_paths,
&result_list);
auto t2 = std::chrono::high_resolution_clock::now();
/* Getting number of milliseconds as a double. */
std::chrono::duration<double, std::milli> ms_double = t2 - t1;

24
mindspore/ccsrc/debug/debugger/offline_debug/dbg_services.h
View File

@ -47,7 +47,7 @@ struct parameter_t {
struct watchpoint_hit_t {
watchpoint_hit_t(const std::string &name, uint32_t slot, int condition, uint32_t watchpoint_id,
const std::vector<parameter_t> &parameters, int32_t error_code, uint32_t device_id,
const std::vector<parameter_t> &parameters, int32_t error_code, uint32_t rank_id,
uint32_t root_graph_id)
: name(name),
slot(slot),
@ -55,7 +55,7 @@ struct watchpoint_hit_t {
watchpoint_id(watchpoint_id),
parameters(parameters),
error_code(error_code),
device_id(device_id),
rank_id(rank_id),
root_graph_id(root_graph_id) {}
const std::string get_name() const { return name; }
const uint32_t get_slot() const { return slot; }
@ -63,7 +63,7 @@ struct watchpoint_hit_t {
const uint32_t get_watchpoint_id() const { return watchpoint_id; }
const std::vector<parameter_t> get_parameters() const { return parameters; }
const int32_t get_error_code() const { return error_code; }
const uint32_t get_device_id() const { return device_id; }
const uint32_t get_rank_id() const { return rank_id; }
const uint32_t get_root_graph_id() const { return root_graph_id; }
std::string name;
uint32_t slot;
@ -71,31 +71,31 @@ struct watchpoint_hit_t {
uint32_t watchpoint_id;
std::vector<parameter_t> parameters;
int32_t error_code;
uint32_t device_id;
uint32_t rank_id;
uint32_t root_graph_id;
};
struct tensor_info_t {
tensor_info_t(const std::string &node_name, uint32_t slot, uint32_t iteration, uint32_t device_id,
uint32_t root_graph_id, bool is_parameter)
tensor_info_t(const std::string &node_name, uint32_t slot, uint32_t iteration, uint32_t rank_id,
uint32_t root_graph_id, bool is_output)
: node_name(node_name),
slot(slot),
iteration(iteration),
device_id(device_id),
rank_id(rank_id),
root_graph_id(root_graph_id),
is_parameter(is_parameter) {}
is_output(is_output) {}
const std::string get_node_name() const { return node_name; }
const uint32_t get_slot() const { return slot; }
const uint32_t get_iteration() const { return iteration; }
const uint32_t get_device_id() const { return device_id; }
const uint32_t get_rank_id() const { return rank_id; }
const uint32_t get_root_graph_id() const { return root_graph_id; }
const bool get_is_parameter() const { return is_parameter; }
const bool get_is_output() const { return is_output; }
std::string node_name;
uint32_t slot;
uint32_t iteration;
uint32_t device_id;
uint32_t rank_id;
uint32_t root_graph_id;
bool is_parameter;
bool is_output;
};
struct tensor_data_t {

6
mindspore/ccsrc/debug/debugger/offline_debug/mi_pybind_register.cc
View File

@ -45,7 +45,7 @@ PYBIND11_MODULE(_mindspore_offline_debug, m) {
.def("get_watchpoint_id", &watchpoint_hit_t::get_watchpoint_id)
.def("get_parameters", &watchpoint_hit_t::get_parameters)
.def("get_error_code", &watchpoint_hit_t::get_error_code)
.def("get_device_id", &watchpoint_hit_t::get_device_id)
.def("get_rank_id", &watchpoint_hit_t::get_rank_id)
.def("get_root_graph_id", &watchpoint_hit_t::get_root_graph_id);
py::class_<tensor_info_t>(m, "tensor_info")
@ -53,9 +53,9 @@ PYBIND11_MODULE(_mindspore_offline_debug, m) {
.def("get_node_name", &tensor_info_t::get_node_name)
.def("get_slot", &tensor_info_t::get_slot)
.def("get_iteration", &tensor_info_t::get_iteration)
.def("get_device_id", &tensor_info_t::get_device_id)
.def("get_rank_id", &tensor_info_t::get_rank_id)
.def("get_root_graph_id", &tensor_info_t::get_root_graph_id)
.def("get_is_parameter", &tensor_info_t::get_is_parameter);
.def("get_is_output", &tensor_info_t::get_is_output);
py::class_<tensor_data_t>(m, "tensor_data")
.def(py::init<char *, uint64_t, int, std::vector<int64_t>>())

5
mindspore/ccsrc/debug/tensor_data.h
View File

@ -226,6 +226,10 @@ class TensorData {
void SetType(std::string type_name) { ConvertStringToDbgType(type_name); }
bool GetIsOutput() { return is_output; }
void SetIsOutput(bool is_output) { this->is_output = is_output; }
void ConvertMsToDbgType(uint32_t type) {
switch (type) {
case MsTypeId::kNumberTypeBool:
@ -411,6 +415,7 @@ class TensorData {
unsigned int iteration;
unsigned int device_id;
unsigned int root_graph_id;
bool is_output;
int execution_order;
#ifdef ONLINE_DBG_MODE
mindspore::tensor::TensorPtr tensor_ptr;

118
mindspore/offline_debug/dbg_services.py
View File

@ -113,8 +113,8 @@ class DbgServices():
watchpoint_id (int): Watchpoint id
watch_condition (int): A representation of the condition to be checked.
check_node_list (dict): Dictionary of node names (str or '*' to check all nodes) as key,
mapping to device_id (list of ints or '*' to check all devices),
root_graph_id (list of ints or '*' to check all graphs) and is_parameter (bool).
mapping to rank_id (list of ints or '*' to check all devices),
root_graph_id (list of ints or '*' to check all graphs) and is_output (bool).
parameter_list (list): List of parameters in watchpoint. Parameters should be instances of Parameter class.
Each parameter describes the value to be checked in watchpoint.
@ -128,8 +128,8 @@ class DbgServices():
>>> d_init = d.initialize(is_sync_mode=True)
>>> d_wp = d_init.add_watchpoint(watchpoint_id=1,
>>> watch_condition=6,
>>> check_node_list={"conv2.bias" : {"device_id": [0],
root_graph_id: [0], "is_parameter": True}},
>>> check_node_list={"conv2.bias" : {"rank_id": [0],
root_graph_id: [0], "is_output": True}},
>>> parameter_list=[dbg_services.Parameter(name="param",
>>> disabled=False,
>>> value=0.0,
@ -140,7 +140,7 @@ class DbgServices():
log("in Python AddWatchpoint")
for node_name, node_info in check_node_list.items():
for info_name, info_param in node_info.items():
if info_name in ["device_id", "root_graph_id"]:
if info_name in ["rank_id", "root_graph_id"]:
if info_param in ["*"]:
check_node_list[node_name][info_name] = ["*"]
else:
@ -169,8 +169,8 @@ class DbgServices():
>>> d_init = d.initialize(is_sync_mode=True)
>>> d_wp = d_init.add_watchpoint(watchpoint_id=1,
>>> watch_condition=6,
>>> check_node_list={"conv2.bias" : {"device_id": [5],
root_graph_id: [0], "is_parameter": True}},
>>> check_node_list={"conv2.bias" : {"rank_id": [5],
root_graph_id: [0], "is_output": True}},
>>> parameter_list=[dbg_services.Parameter(name="param",
>>> disabled=False,
>>> value=0.0,
@ -201,8 +201,8 @@ class DbgServices():
>>> d_init = d.initialize(is_sync_mode=True)
>>> d_wp = d_init.add_watchpoint(id=1,
>>> watch_condition=6,
>>> check_node_list={"conv2.bias" : {"device_id": [5],
root_graph_id: [0], "is_parameter": True}},
>>> check_node_list={"conv2.bias" : {"rank_id": [5],
root_graph_id: [0], "is_output": True}},
>>> parameter_list=[dbg_services.Parameter(name="param",
>>> disabled=False,
>>> value=0.0,
@ -221,7 +221,7 @@ class DbgServices():
watchpoint_id = watchpoint.get_watchpoint_id()
parameters = watchpoint.get_parameters()
error_code = watchpoint.get_error_code()
device_id = watchpoint.get_device_id()
rank_id = watchpoint.get_rank_id()
root_graph_id = watchpoint.get_root_graph_id()
param_list = []
for param in parameters:
@ -232,7 +232,7 @@ class DbgServices():
actual_value = param.get_actual_value()
param_list.append(Parameter(p_name, disabled, value, hit, actual_value))
watchpoint_hit_list.append(WatchpointHit(name, slot, condition, watchpoint_id,
param_list, error_code, device_id, root_graph_id))
param_list, error_code, rank_id, root_graph_id))
return watchpoint_hit_list
@check_initialize_done
@ -255,9 +255,9 @@ class DbgServices():
>>> tensor_data_list = d_init.read_tensors([dbg_services.TensorInfo(node_name="conv2.bias",
>>> slot=0,
>>> iteration=8,
>>> device_id=5,
>>> rank_id=5,
>>> root_graph_id=0,
>>> is_parameter=True)])
>>> is_output=True)])
"""
log("in Python ReadTensors info ", info)
@ -283,22 +283,22 @@ class TensorInfo():
node_name (str): Fully qualified name of the desired node.
slot (int): The particular output for the requested node.
iteration (int): The desired itraretion to gather tensor information.
device_id (int): The desired device id to gather tensor information.
is_parameter (bool): Whether node is a parameter (input, constant, bias, parameter).
rank_id (int): The desired rank id to gather tensor information.
is_output (bool): Whether node is an output or input.
Examples:
>>> from mindspore.ccsrc.debug.debugger.offline_debug import dbg_services
>>> tensor_info = dbg_services.TensorInfo(node_name="conv2.bias",
>>> slot=0,
>>> iteration=8,
>>> device_id=5,
>>> rank_id=5,
>>> root_graph_id=0,
>>> is_parameter=True)
>>> is_output=True)
"""
@check_tensor_info_init
def __init__(self, node_name, slot, iteration, device_id, root_graph_id, is_parameter):
self.instance = cds.tensor_info(node_name, slot, iteration, device_id, root_graph_id, is_parameter)
def __init__(self, node_name, slot, iteration, rank_id, root_graph_id, is_output=True):
self.instance = cds.tensor_info(node_name, slot, iteration, rank_id, root_graph_id, is_output)
@property
def node_name(self):
@ -313,9 +313,9 @@ class TensorInfo():
>>> tensor_info = dbg_services.TensorInfo(node_name="conv2.bias",
>>> slot=0,
>>> iteration=8,
>>> device_id=5,
>>> rank_id=5,
>>> root_graph_id=0,
>>> is_parameter=True)
>>> is_output=True)
>>> name = tensor_info.node_name
"""
@ -334,9 +334,9 @@ class TensorInfo():
>>> tensor_info = dbg_services.TensorInfo(node_name="conv2.bias",
>>> slot=0,
>>> iteration=8,
>>> device_id=5,
>>> rank_id=5,
>>> root_graph_id=0,
>>> is_parameter=True)
>>> is_output=True)
>>> slot = tensor_info.slot
"""
@ -355,33 +355,35 @@ class TensorInfo():
>>> tensor_info = dbg_services.TensorInfo(node_name="conv2.bias",
>>> slot=0,
>>> iteration=8,
>>> device_id=5,
>>> rank_id=5,
>>> root_graph_id=0,
>>> is_parameter=True)
>>> is_output=True)
>>> iteration = tensor_info.iteration
"""
return self.instance.get_iteration()
@property
def device_id(self):
def rank_id(self):
"""
Function to receive TensorInfo device_id.
Function to receive TensorInfo rank_id.
Returns:
device_id of TensorInfo instance (int).
rank_id of TensorInfo instance (int).
Examples:
>>> from mindspore.ccsrc.debug.debugger.offline_debug import dbg_services
>>> tensor_info = dbg_services.TensorInfo(node_name="conv2.bias",
>>> slot=0,
>>> iteration=8,
>>> device_id=5,
>>> rank_id=5,
>>> root_graph_id=0,
>>> is_parameter=True)
>>> device_id = tensor_info.device_id
>>> is_output=True)
>>> rank_id = tensor_info.rank_id
"""
return self.instance.get_rank_id()
@property
def root_graph_id(self):
"""
@ -395,34 +397,34 @@ class TensorInfo():
>>> tensor_info = dbg_services.TensorInfo(node_name="conv2.bias",
>>> slot=0,
>>> iteration=8,
>>> device_id=5,
>>> rank_id=5,
>>> root_graph_id=0,
>>> is_parameter=True)
>>> device_id = tensor_info.root_graph_id
>>> is_output=True)
>>> rank_id = tensor_info.root_graph_id
"""
return self.instance.get_root_graph_id()
@property
def is_parameter(self):
def is_output(self):
"""
Function to receive TensorInfo is_parameter.
Function to receive TensorInfo is_output.
Returns:
is_parameter of TensorInfo instance (bool).
is_output of TensorInfo instance (bool).
Examples:
>>> from mindspore.ccsrc.debug.debugger.offline_debug import dbg_services
>>> tensor_info = dbg_services.TensorInfo(node_name="conv2.bias",
>>> slot=0,
>>> iteration=8,
>>> device_id=5,
>>> rank_id=5,
>>> root_graph_id=0,
>>> is_parameter=True)
>>> is_parameter = tensor_info.is_parameter
>>> is_output=True)
>>> is_output = tensor_info.is_output
"""
return self.instance.get_is_parameter()
return self.instance.get_is_output()
class TensorData():
"""
@ -534,7 +536,7 @@ class WatchpointHit():
parameters (list): A list of all parameters for WatchpointHit instance.
Parameters have to be instances of Parameter class.
error_code (int): An explanation of certain scenarios where watchpoint could not be checked.
device_id (int): Device id where the watchpoint is hit.
rank_id (int): Rank id where the watchpoint is hit.
root_graph_id (int): Root graph id where the watchpoint is hit.
Examples:
@ -545,17 +547,17 @@ class WatchpointHit():
>>> watchpoint_id=3,
>>> parameters=[param1, param2],
>>> error_code=0,
>>> device_id=1,
>>> rank_id=1,
>>> root_graph_id=1)
"""
@check_watchpoint_hit_init
def __init__(self, name, slot, condition, watchpoint_id, parameters, error_code, device_id, root_graph_id):
def __init__(self, name, slot, condition, watchpoint_id, parameters, error_code, rank_id, root_graph_id):
parameter_list_inst = []
for elem in parameters:
parameter_list_inst.append(elem.instance)
self.instance = cds.watchpoint_hit(name, slot, condition, watchpoint_id,
parameter_list_inst, error_code, device_id, root_graph_id)
parameter_list_inst, error_code, rank_id, root_graph_id)
@property
def name(self):
@ -573,7 +575,7 @@ class WatchpointHit():
>>> watchpoint_id=3,
>>> parameters=[param1, param2],
>>> error_code=0,
>>> device_id=1,
>>> rank_id=1,
>>> root_graph_id=1)
>>> name = watchpoint_hit.name
"""
@ -596,7 +598,7 @@ class WatchpointHit():
>>> watchpoint_id=3,
>>> parameters=[param1, param2],
>>> error_code=0,
>>> device_id=1,
>>> rank_id=1,
>>> root_graph_id=1)
>>> slot = watchpoint_hit.slot
"""
@ -619,7 +621,7 @@ class WatchpointHit():
>>> watchpoint_id=3,
>>> parameters=[param1, param2],
>>> error_code=0,
>>> device_id=1,
>>> rank_id=1,
>>> root_graph_id=1)
>>> condition = watchpoint_hit.condition
"""
@ -642,7 +644,7 @@ class WatchpointHit():
>>> watchpoint_id=3,
>>> parameters=[param1, param2],
>>> error_code=0,
>>> device_id=1,
>>> rank_id=1,
>>> root_graph_id=1)
>>> watchpoint_id = watchpoint_hit.watchpoint_id
"""
@ -665,7 +667,7 @@ class WatchpointHit():
>>> watchpoint_id=3,
>>> parameters=[param1, param2],
>>> error_code=0,
>>> device_id=1,
>>> rank_id=1,
>>> root_graph_id=1)
>>> parameters = watchpoint_hit.parameters
"""
@ -697,7 +699,7 @@ class WatchpointHit():
>>> watchpoint_id=3,
>>> parameters=[param1, param2],
>>> error_code=0,
>>> device_id=1,
>>> rank_id=1,
>>> root_graph_id=1)
>>> error_code = watchpoint_hit.error_code
"""
@ -705,12 +707,12 @@ class WatchpointHit():
return self.instance.get_error_code()
@property
def device_id(self):
def rank_id(self):
"""
Function to receive WatchpointHit device_id.
Function to receive WatchpointHit rank_id.
Returns:
device_id of WatchpointHit instance (int).
rank_id of WatchpointHit instance (int).
Examples:
>>> from mindspore.ccsrc.debug.debugger.offline_debug import dbg_services
@ -720,12 +722,12 @@ class WatchpointHit():
>>> watchpoint_id=3,
>>> parameters=[param1, param2],
>>> error_code=0,
>>> device_id=1,
>>> rank_id=1,
>>> root_graph_id=1)
>>> device_id = watchpoint_hit.device_id
>>> rank_id = watchpoint_hit.rank_id
"""
return self.instance.get_device_id()
return self.instance.get_rank_id()
@property
def root_graph_id(self):
@ -743,7 +745,7 @@ class WatchpointHit():
>>> watchpoint_id=3,
>>> parameters=[param1, param2],
>>> error_code=0,
>>> device_id=1,
>>> rank_id=1,
>>> root_graph_id=1)
>>> root_graph_id = watchpoint_hit.root_graph_id
"""

20
mindspore/offline_debug/mi_validators.py
View File

@ -67,13 +67,13 @@ def check_add_watchpoint(method):
type_check(node_info, (dict,), "node_info")
for info_name, info_param in node_info.items():
type_check(info_name, (str,), "node parameter name")
if info_name in ["device_id"]:
if info_name in ["rank_id"]:
if isinstance(info_param, str):
if info_param not in ["*"]:
raise ValueError("Node parameter {} only accepts '*' as string.".format(info_name))
else:
for param in info_param:
check_uint32(param, "device_id")
check_uint32(param, "rank_id")
elif info_name in ["root_graph_id"]:
if isinstance(info_param, str):
if info_param not in ["*"]:
@ -81,8 +81,8 @@ def check_add_watchpoint(method):
else:
for param in info_param:
check_uint32(param, "root_graph_id")
elif info_name in ["is_parameter"]:
type_check(info_param, (bool,), "is_parameter")
elif info_name in ["is_output"]:
type_check(info_param, (bool,), "is_output")
else:
raise ValueError("Node parameter {} is not defined.".format(info_name))
param_names = ["param_{0}".format(i) for i in range(len(parameter_list))]
@ -154,15 +154,15 @@ def check_tensor_info_init(method):
@wraps(method)
def new_method(self, *args, **kwargs):
[node_name, slot, iteration, device_id, root_graph_id,
is_parameter], _ = parse_user_args(method, *args, **kwargs)
[node_name, slot, iteration, rank_id, root_graph_id,
is_output], _ = parse_user_args(method, *args, **kwargs)
type_check(node_name, (str,), "node_name")
check_uint32(slot, "slot")
check_iteration(iteration, "iteration")
check_uint32(device_id, "device_id")
check_uint32(rank_id, "rank_id")
check_uint32(root_graph_id, "root_graph_id")
type_check(is_parameter, (bool,), "is_parameter")
type_check(is_output, (bool,), "is_output")
return method(self, *args, **kwargs)
@ -196,7 +196,7 @@ def check_watchpoint_hit_init(method):
@wraps(method)
def new_method(self, *args, **kwargs):
[name, slot, condition, watchpoint_id,
parameters, error_code, device_id, root_graph_id], _ = parse_user_args(method, *args, **kwargs)
parameters, error_code, rank_id, root_graph_id], _ = parse_user_args(method, *args, **kwargs)
type_check(name, (str,), "name")
check_uint32(slot, "slot")
@ -205,7 +205,7 @@ def check_watchpoint_hit_init(method):
param_names = ["param_{0}".format(i) for i in range(len(parameters))]
type_check_list(parameters, (cds.Parameter,), param_names)
type_check(error_code, (int,), "error_code")
check_uint32(device_id, "device_id")
check_uint32(rank_id, "rank_id")
check_uint32(root_graph_id, "root_graph_id")
return method(self, *args, **kwargs)