forked from huawei/mindspore2022
299 lines
12 KiB
C++
299 lines
12 KiB
C++
/**
|
|
* Copyright 2019-2020 Huawei Technologies Co., Ltd
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
#include "debug/debug_services.h"
|
|
namespace mindspore {
|
|
|
|
DebugServices::DebugServices() {
|
|
tensor_loader_ = new TensorLoader();
|
|
uint32_t iter_num = -1;
|
|
tensor_loader_->set_iter_num(iter_num);
|
|
}
|
|
|
|
DebugServices::DebugServices(const DebugServices &other) {
|
|
tensor_loader_ = other.tensor_loader_;
|
|
watchpoint_table = other.watchpoint_table;
|
|
}
|
|
|
|
DebugServices &DebugServices::operator=(const DebugServices &other) {
|
|
if (this != &other) {
|
|
tensor_loader_ = other.tensor_loader_;
|
|
watchpoint_table = other.watchpoint_table;
|
|
}
|
|
return *this;
|
|
}
|
|
|
|
DebugServices::~DebugServices() { delete tensor_loader_; }
|
|
|
|
void DebugServices::AddWatchpoint(unsigned int id, unsigned int watch_condition,
|
|
const std::vector<std::tuple<std::string, bool>> &check_node_list) {
|
|
std::lock_guard<std::mutex> lg(lock_);
|
|
|
|
watchpoint_t watchpoint_item;
|
|
|
|
watchpoint_item.id = id;
|
|
|
|
if (watch_condition == 0) {
|
|
watchpoint_item.conditions.nan.enabled = true;
|
|
} else if (watch_condition == 1) {
|
|
watchpoint_item.conditions.inf.enabled = true;
|
|
watchpoint_item.conditions.neg_inf.enabled = true;
|
|
} else if (watch_condition == 2) {
|
|
watchpoint_item.conditions.overflow.enabled = true;
|
|
}
|
|
|
|
watchpoint_item.check_node_list = check_node_list;
|
|
|
|
watchpoint_table[id] = watchpoint_item;
|
|
}
|
|
|
|
void DebugServices::RemoveWatchpoint(unsigned int id) {
|
|
std::lock_guard<std::mutex> lg(lock_);
|
|
watchpoint_table.erase(id);
|
|
}
|
|
|
|
void DebugServices::CheckWatchpoints(std::vector<std::string> *name, std::vector<std::string> *slot,
|
|
std::vector<int> *condition, std::vector<unsigned int> *watchpoint_id,
|
|
const std::vector<std::string> &op_overflows) {
|
|
std::lock_guard<std::mutex> lg(lock_);
|
|
|
|
std::vector<std::shared_ptr<TensorData>> tensor_list = tensor_loader_->GetTensor();
|
|
|
|
std::string current_tensor_name;
|
|
std::unordered_map<unsigned int, watchpoint_t> watchpoints_to_check_table;
|
|
const size_t location = 0;
|
|
|
|
for (std::size_t i = 0; i < tensor_list.size(); i++) {
|
|
current_tensor_name = tensor_list[i]->GetName();
|
|
std::string tensor_slot = std::to_string(tensor_list[i]->GetSlot());
|
|
mindspore::tensor::TensorPtr tensor_ptr = tensor_list[i]->GetTensor();
|
|
int tensor_data_type = tensor_ptr->data_type_c();
|
|
|
|
// check if we need to analyze this node and for which watchpoints we will check
|
|
// create a list of watchpoints to check
|
|
watchpoints_to_check_table.clear();
|
|
for (auto w_table_item : watchpoint_table) {
|
|
// if the watchpoint is checking for a nan or inf and the current tensor is not of a float type, then
|
|
// don't check the watchpoint for this tensor
|
|
if (std::get<1>(w_table_item).conditions.inf.enabled || std::get<1>(w_table_item).conditions.neg_inf.enabled ||
|
|
std::get<1>(w_table_item).conditions.nan.enabled) {
|
|
if (tensor_data_type != kNumberTypeFloat16 && tensor_data_type != kNumberTypeFloat &&
|
|
tensor_data_type != kNumberTypeFloat32 && tensor_data_type != kNumberTypeFloat64) {
|
|
continue;
|
|
}
|
|
}
|
|
|
|
auto check_node_list = std::get<1>(w_table_item).check_node_list;
|
|
|
|
for (auto check_node : check_node_list) {
|
|
std::string w_name = std::get<0>(check_node);
|
|
bool w_type = std::get<1>(check_node);
|
|
|
|
// check if the current node tensor name is included the watchpoint
|
|
std::string current_node_name = current_tensor_name.substr(0, current_tensor_name.find_first_of(":"));
|
|
if ((w_type == true && (current_tensor_name.find(w_name) == location || w_name == "*")) ||
|
|
(w_type == false && current_node_name == w_name)) {
|
|
watchpoints_to_check_table[w_table_item.second.id] = w_table_item.second;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
std::vector<unsigned int> hit_encountered;
|
|
|
|
// handle watchpoint conditions that do not require per element checks
|
|
for (auto it_w_table_check = watchpoints_to_check_table.begin();
|
|
it_w_table_check != watchpoints_to_check_table.end(); ++it_w_table_check) {
|
|
if (it_w_table_check->second.conditions.overflow.enabled) {
|
|
std::string name_no_slot = current_tensor_name.substr(0, current_tensor_name.find_first_of(":"));
|
|
if (std::find(op_overflows.begin(), op_overflows.end(), name_no_slot) != op_overflows.end()) {
|
|
hit_encountered.push_back(it_w_table_check->second.id);
|
|
}
|
|
}
|
|
}
|
|
|
|
if (hit_encountered.size()) {
|
|
HandleWatchpointHits(hit_encountered, name, slot, condition, watchpoint_id, current_tensor_name,
|
|
&watchpoints_to_check_table, tensor_slot);
|
|
hit_encountered.clear();
|
|
}
|
|
|
|
// need to add support for float16 and float64, and other types when we support conditions beyond inf and nan
|
|
if (tensor_data_type != kNumberTypeFloat && tensor_data_type != kNumberTypeFloat32) {
|
|
continue;
|
|
}
|
|
|
|
// check if no watchpoints are remaining
|
|
if (watchpoints_to_check_table.empty()) {
|
|
continue;
|
|
}
|
|
|
|
float *start_addr = reinterpret_cast<float *>(tensor_ptr->data_c());
|
|
unsigned int num_elements = (tensor_ptr->data().nbytes()) / sizeof(float);
|
|
std::unordered_map<unsigned int, watchpoint_t>::iterator it_w_table_check;
|
|
|
|
for (unsigned int index = 0; index < num_elements; index++) {
|
|
float x = start_addr[index];
|
|
it_w_table_check = watchpoints_to_check_table.begin();
|
|
|
|
while (it_w_table_check != watchpoints_to_check_table.end()) {
|
|
if ((it_w_table_check->second.conditions.inf.enabled || it_w_table_check->second.conditions.neg_inf.enabled) &&
|
|
isinf(x)) {
|
|
hit_encountered.push_back(it_w_table_check->second.id);
|
|
} else if (it_w_table_check->second.conditions.nan.enabled && isnan(x)) {
|
|
hit_encountered.push_back(it_w_table_check->second.id);
|
|
}
|
|
++it_w_table_check;
|
|
}
|
|
|
|
if (hit_encountered.size()) {
|
|
HandleWatchpointHits(hit_encountered, name, slot, condition, watchpoint_id, current_tensor_name,
|
|
&watchpoints_to_check_table, tensor_slot);
|
|
hit_encountered.clear();
|
|
}
|
|
|
|
if (watchpoints_to_check_table.empty()) {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void DebugServices::HandleWatchpointHits(const std::vector<unsigned int> &hit_encountered,
|
|
std::vector<std::string> *name, std::vector<std::string> *slot,
|
|
std::vector<int> *condition, std::vector<unsigned int> *watchpoint_id,
|
|
std::string current_tensor_name,
|
|
std::unordered_map<unsigned int, watchpoint_t> *watchpoints_to_check_table,
|
|
std::string tensor_slot) {
|
|
for (auto it_hit_id = hit_encountered.begin(); it_hit_id != hit_encountered.end(); ++it_hit_id) {
|
|
if (watchpoint_table.find(*it_hit_id) != watchpoint_table.end()) {
|
|
std::string name_no_slot = current_tensor_name.substr(0, current_tensor_name.find_first_of(":"));
|
|
name->push_back(name_no_slot);
|
|
slot->push_back(tensor_slot);
|
|
|
|
int condition_item = -1;
|
|
if (watchpoint_table[*it_hit_id].conditions.nan.enabled) {
|
|
condition_item = 0;
|
|
} else if (watchpoint_table[*it_hit_id].conditions.inf.enabled ||
|
|
watchpoint_table[*it_hit_id].conditions.neg_inf.enabled) {
|
|
condition_item = 1;
|
|
} else if (watchpoint_table[*it_hit_id].conditions.overflow.enabled) {
|
|
condition_item = 2;
|
|
}
|
|
condition->push_back(condition_item);
|
|
watchpoint_id->push_back(*it_hit_id);
|
|
}
|
|
watchpoints_to_check_table->erase(*it_hit_id);
|
|
}
|
|
}
|
|
|
|
void DebugServices::CheckSingleWatchpoint(std::shared_ptr<TensorData> watchtensor, std::string *name, std::string *slot,
|
|
char **data_ptr, unsigned int *data_size, int *condition,
|
|
unsigned int *wacthpoint_id) {
|
|
std::lock_guard<std::mutex> lg(lock_);
|
|
|
|
std::string current_watchtensor_name;
|
|
current_watchtensor_name = watchtensor->GetName();
|
|
mindspore::tensor::TensorPtr tensor_ptr = watchtensor->GetTensor();
|
|
int tensor_data_type = tensor_ptr->data_type_c();
|
|
watchpoint_t watchpoint_to_check;
|
|
|
|
for (auto w_table_item : watchpoint_table) {
|
|
auto check_node_list = std::get<1>(w_table_item).check_node_list;
|
|
for (auto check_node : check_node_list) {
|
|
std::string w_name = std::get<0>(check_node);
|
|
bool w_type = std::get<1>(check_node);
|
|
// get current the full info including condition, id..., for current watchtensor
|
|
std::string current_node_name = current_watchtensor_name.substr(0, current_watchtensor_name.find_first_of(":"));
|
|
if ((w_type == true && (current_watchtensor_name.find(w_name) != string::npos || w_name == "*")) ||
|
|
(w_type == false && current_node_name == w_name)) {
|
|
watchpoint_to_check = w_table_item.second;
|
|
// need to add support for float16 and float64, and other types when we support conditions beyond inf and nan
|
|
if (tensor_data_type != kNumberTypeFloat && tensor_data_type != kNumberTypeFloat32) {
|
|
return;
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
float *start_addr = reinterpret_cast<float *>(tensor_ptr->data_c());
|
|
unsigned int num_elements = (tensor_ptr->data().nbytes()) / sizeof(float);
|
|
|
|
for (unsigned int index = 0; index < num_elements; index++) {
|
|
float x = start_addr[index];
|
|
if (((watchpoint_to_check.conditions.inf.enabled || watchpoint_to_check.conditions.neg_inf.enabled) && isinf(x)) ||
|
|
(watchpoint_to_check.conditions.nan.enabled && isnan(x))) {
|
|
std::string name_no_slot = current_watchtensor_name.substr(0, current_watchtensor_name.find_first_of(":"));
|
|
*name = name_no_slot;
|
|
*slot = std::to_string(watchtensor->GetSlot());
|
|
*data_ptr = reinterpret_cast<char *>(tensor_ptr->data_c());
|
|
*data_size = tensor_ptr->data().nbytes();
|
|
int condition_item = -1;
|
|
if (watchpoint_to_check.conditions.nan.enabled) {
|
|
condition_item = 0;
|
|
} else if (watchpoint_to_check.conditions.inf.enabled || watchpoint_to_check.conditions.neg_inf.enabled) {
|
|
condition_item = 1;
|
|
}
|
|
*condition = condition_item;
|
|
|
|
*wacthpoint_id = watchpoint_to_check.id;
|
|
}
|
|
}
|
|
}
|
|
|
|
void DebugServices::ReadNodesTensors(std::vector<std::string> name, std::vector<std::string> *ret_name,
|
|
std::vector<char *> *data_ptr, std::vector<unsigned int> *data_size,
|
|
std::vector<TypePtr> *dtype, std::vector<std::vector<int>> *shape) {
|
|
std::vector<std::tuple<std::string, std::shared_ptr<TensorData>>> result_list;
|
|
tensor_loader_->SearchTensors(name, &result_list);
|
|
|
|
for (auto result : result_list) {
|
|
if (!std::get<1>(result)) {
|
|
continue;
|
|
}
|
|
ret_name->push_back(std::get<0>(result));
|
|
data_ptr->push_back(reinterpret_cast<char *>(std::get<1>(result)->GetTensor()->data_c()));
|
|
data_size->push_back(std::get<1>(result)->GetTensor()->data().nbytes());
|
|
dtype->push_back(std::get<1>(result)->GetTensor()->Dtype());
|
|
shape->push_back(std::get<1>(result)->GetTensor()->shape());
|
|
}
|
|
}
|
|
|
|
bool DebugServices::IsWatchPoint(std::string kernel_name,
|
|
std::unordered_map<unsigned int, watchpoint_t> watchpoint_table) {
|
|
bool ret = false;
|
|
for (auto w_table_item : watchpoint_table) {
|
|
auto check_node_list = std::get<1>(w_table_item).check_node_list;
|
|
for (auto check_node : check_node_list) {
|
|
std::string w_name = std::get<0>(check_node);
|
|
bool w_type = std::get<1>(check_node);
|
|
if ((w_type == true &&
|
|
((kernel_name.find(w_name) != string::npos && kernel_name.rfind(w_name, 0) == 0) || w_name == "*")) ||
|
|
(w_type == false && kernel_name == w_name)) {
|
|
ret = true;
|
|
return ret;
|
|
}
|
|
}
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
TensorLoader *DebugServices::tensor_loader() const { return tensor_loader_; }
|
|
std::unordered_map<unsigned int, DebugServices::watchpoint_t> DebugServices::GetWatchpointTable() {
|
|
return watchpoint_table;
|
|
}
|
|
|
|
} // namespace mindspore
|