timekiller
/
mindspore2022
forked from huawei/mindspore2022
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fix DumpParameters issue when only need dump input data 

Fix code warnings in debug

Fix code warnings2
This commit is contained in:
maning202007 2022-02-18 11:52:11 +08:00
parent 6e9289dfff
commit 54a69ebd82
9 changed files with 55 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
mindspore/ccsrc/debug/data_dump/cpu_e2e_dump.cc
View File

@ -179,6 +179,10 @@ void CPUE2eDump::DumpSingleAnfNode(const AnfNodePtr &anf_node, const size_t outp
void CPUE2eDump::DumpParameters(const session::KernelGraph *graph, uint32_t graph_id) {
MS_EXCEPTION_IF_NULL(graph);
auto &dump_json_parser = DumpJsonParser::GetInstance();
if (!dump_json_parser.OutputNeedDump()) {
return;
}
MS_LOG(INFO) << "Start e2e dump parameters.";
const std::string &dump_path = GenerateDumpPath(graph_id);
@ -198,6 +202,10 @@ void CPUE2eDump::DumpParametersData() {
void CPUE2eDump::DumpConstants(const session::KernelGraph *graph, uint32_t graph_id) {
MS_EXCEPTION_IF_NULL(graph);
auto &dump_json_parser = DumpJsonParser::GetInstance();
if (!dump_json_parser.OutputNeedDump()) {
return;
}
MS_LOG(INFO) << "Start e2e dump constant.";
uint32_t cur_iteration = DumpJsonParser::GetInstance().cur_dump_iter();
if (cur_iteration != 0) {

20
mindspore/ccsrc/debug/data_dump/e2e_dump.cc
View File

@ -186,7 +186,7 @@ void E2eDump::DumpOutputImpl(const CNodePtr &node, bool trans_flag, const std::s
if (DumpJsonParser::GetInstance().IsStatisticDump() &&
(IsDeviceTargetGPU() || Debugger::GetInstance()->GetAscendKernelByKernelFlag())) {
TensorStatDump stat_dump(op_type, op_name, task_id, stream_id, timestamp, false, j, j);
stat_dump.DumpTensorStatsToFile(GetKernelNodeName(node), dump_path, debugger);
(void)stat_dump.DumpTensorStatsToFile(GetKernelNodeName(node), dump_path, debugger);
}
if (DumpJsonParser::GetInstance().IsTensorDump()) {
if (IsDeviceTargetGPU()) {
@ -287,7 +287,7 @@ void E2eDump::DumpInputImpl(const CNodePtr &node, bool trans_flag, const std::st
if (DumpJsonParser::GetInstance().IsStatisticDump() &&
(IsDeviceTargetGPU() || Debugger::GetInstance()->GetAscendKernelByKernelFlag())) {
TensorStatDump stat_dump(op_type, op_name, task_id, stream_id, timestamp, true, j, slot);
stat_dump.DumpTensorStatsToFile(tensor_name, dump_path, debugger);
(void)stat_dump.DumpTensorStatsToFile(tensor_name, dump_path, debugger);
}
if (DumpJsonParser::GetInstance().IsTensorDump()) {
if (IsDeviceTargetGPU()) {
@ -345,7 +345,7 @@ void E2eDump::DumpSingleAnfNode(const AnfNodePtr &anf_node, const size_t output_
if (IsDeviceTargetGPU()) {
if (dump_json_parser.IsStatisticDump()) {
TensorStatDump stat_dump("Parameter", dump_name, task_id, stream_id, timestamp, false, 0, 0);
stat_dump.DumpTensorStatsToFile(node_name, dump_path, debugger);
(void)stat_dump.DumpTensorStatsToFile(node_name, dump_path, debugger);
}
if (dump_json_parser.IsTensorDump()) {
DumpGPUMemToFile(file_path, node_name, *addr, int_shapes, type, device_type, trans_flag, 0, debugger);
@ -367,7 +367,7 @@ void E2eDump::DumpSingleParameterNode(const AnfNodePtr &anf_node, const std::str
MS_EXCEPTION_IF_NULL(anf_node);
auto &dump_json_parser = DumpJsonParser::GetInstance();
std::string node_name = GetKernelNodeName(anf_node);
if (!anf_node->isa<Parameter>() || !dump_json_parser.NeedDump(node_name)) {
if (!anf_node->isa<Parameter>() || !dump_json_parser.NeedDump(node_name) || !dump_json_parser.OutputNeedDump()) {
return;
}
DumpJsonParser::GetInstance().MatchKernel(node_name);
@ -388,7 +388,7 @@ void E2eDump::DumpSingleParameterNode(const AnfNodePtr &anf_node, const std::str
if (IsDeviceTargetGPU()) {
if (dump_json_parser.IsStatisticDump()) {
TensorStatDump stat_dump("Parameter", node_name, task_id, stream_id, timestamp, false, 0, 0);
stat_dump.DumpTensorStatsToFile(node_name, dump_path, debugger);
(void)stat_dump.DumpTensorStatsToFile(node_name, dump_path, debugger);
}
if (dump_json_parser.IsTensorDump()) {
DumpGPUMemToFile(file_path, node_name, *addr, int_shapes, type, device_type, trans_flag, 0, debugger);
@ -534,8 +534,8 @@ void E2eDump::DumpRunIter(const KernelGraphPtr &graph, uint32_t rank_id) {
// dump history for all iterations in the epoch
Debugger::GetInstance()->UpdateGraphIterMap(graph->graph_id(), iter_num);
auto graph_iter_map = Debugger::GetInstance()->GetGraphIterMap();
auto step_per_epoch = graph_iter_map[graph->graph_id()];
for (int i = 0; i < step_per_epoch; i++) {
auto step_per_epoch = IntToSize(graph_iter_map[graph->graph_id()]);
for (size_t i = 0; i < step_per_epoch; i++) {
auto step = (json_parser.cur_dump_iter() * step_per_epoch) + i;
fout << (std::to_string(step) + "\n");
}
@ -567,7 +567,7 @@ void E2eDump::DumpData(const session::KernelGraph *graph, uint32_t rank_id, cons
MS_LOG(INFO) << "Current graph id is " << graph_id;
std::string dump_path = GenerateDumpPath(graph_id, rank_id);
if (dump_json_parser.IsStatisticDump()) {
TensorStatDump::OpenStatisticsFile(dump_path);
(void)TensorStatDump::OpenStatisticsFile(dump_path);
}
DumpInput(graph, dump_path, debugger);
DumpOutput(graph, dump_path, debugger);
@ -799,7 +799,7 @@ uint64_t UnpackUint64Value(char *ptr) {
#if defined(__APPLE__)
return *reinterpret_cast<const uint64_t *>(ptr);
#else
return le16toh(*reinterpret_cast<const uint64_t *>(ptr));
return le64toh(*reinterpret_cast<const uint64_t *>(ptr));
#endif
}
@ -811,7 +811,7 @@ std::string IntToHexString(const uint64_t value) {
nlohmann::json E2eDump::ParseOverflowInfo(char *data_ptr) {
uint32_t index = 0;
uint64_t model_id = UnpackUint64Value(data_ptr + index);
uint64_t model_id = UnpackUint64Value(data_ptr);
index += kUint64Size;
uint64_t stream_id = UnpackUint64Value(data_ptr + index);
index += kUint64Size;

9
mindspore/ccsrc/debug/data_dump/tensor_stat_dump.cc
View File

@ -17,6 +17,7 @@
#include "debug/data_dump/tensor_stat_dump.h"
#include <memory>
#include <map>
#include "utils/file_utils.h"
#include "debug/common.h"
#include "debug/debug_services.h"
@ -65,14 +66,14 @@ bool CsvWriter::OpenFile(const std::string &path, const std::string &header) {
}
if (first_time_opening) {
file_ << header;
file_.flush();
(void)file_.flush();
file_path_str_ = path;
}
MS_LOG(INFO) << "Opened file: " << file_path_value;
return true;
}
void CsvWriter::CloseFile() {
void CsvWriter::CloseFile() noexcept {
if (file_.is_open()) {
file_.close();
ChangeFileMode(file_path_str_, S_IRUSR);
@ -85,7 +86,7 @@ void CsvWriter::WriteToCsv(const T &val, bool end_line) {
file_ << val;
if (end_line) {
file_ << kEndLine;
file_.flush();
(void)file_.flush();
} else {
file_ << kSeparator;
}
@ -156,7 +157,7 @@ bool TensorStatDump::DumpTensorStatsToFile(const std::string &original_kernel_na
return DumpTensorStatsToFile(dump_path, data);
}
bool TensorStatDump::DumpTensorStatsToFile(const std::string &dump_path, std::shared_ptr<TensorData> data) {
bool TensorStatDump::DumpTensorStatsToFile(const std::string &dump_path, const std::shared_ptr<TensorData> data) {
if (data == nullptr) {
MS_LOG(INFO) << "Tensor data is empty, skipping current statistics";
return false;

4
mindspore/ccsrc/debug/data_dump/tensor_stat_dump.h
View File

@ -37,7 +37,7 @@ class CsvWriter {
~CsvWriter();
DISABLE_COPY_AND_ASSIGN(CsvWriter)
bool OpenFile(const std::string &path, const std::string &header = "");
void CloseFile();
void CloseFile() noexcept;
template <typename T>
void WriteToCsv(const T &val, bool end_line = false);
@ -57,7 +57,7 @@ class TensorStatDump {
TensorStatDump(const std::string &op_type, const std::string &op_name, const std::string &task_id,
const std::string &stream_id, const std::string &timestamp, const std::string &io, size_t slot,
size_t tensor_loader_slot);
bool DumpTensorStatsToFile(const std::string &dump_path, std::shared_ptr<TensorData> data);
bool DumpTensorStatsToFile(const std::string &dump_path, const std::shared_ptr<TensorData> data);
bool DumpTensorStatsToFile(const std::string &original_kernel_name, const std::string &dump_path,
const Debugger *debugger);

8
mindspore/ccsrc/debug/debug_services.cc
View File

@ -897,7 +897,7 @@ void DebugServices::ProcessConvertToHostFormat(const std::vector<std::string> &f
if (candidate.find(file_n + ".") != std::string::npos && candidate.rfind(kNpyExt) != std::string::npos) {
// we found a converted file for this op
std::string found_file = dump_key + "/" + candidate;
result_list->insert(found_file);
(void)result_list->insert(found_file);
}
}
}
@ -1008,13 +1008,13 @@ void DebugServices::ProcessConvertList(const std::string &prefix_dump_file_name,
}
if (file_name.rfind(kNpyExt) == std::string::npos) {
std::size_t second_dot = file_name.find(".", file_name.find(prefix_dump_file_name + ".", type_pos + 1));
file_name_w_o_perfix.replace(type_pos + 1, second_dot - type_pos - 1, prefix_dump_file_name);
(void)file_name_w_o_perfix.replace(type_pos + 1, second_dot - type_pos - 1, prefix_dump_file_name);
// if file matches prefix and is in device format add to candidate files to convert.
(*dir_to_files_map)[specific_dump_dir].push_back(std::make_pair(file_name, file_name_w_o_perfix));
} else {
// otherwise, if file matches prefix and already has been converted to host format
// add to result of converted files.
result_list->insert(file_path);
(void)result_list->insert(file_path);
}
}
(void)closedir(d);
@ -1803,7 +1803,7 @@ uint32_t DebugServices::GetPrevIteration(const std::shared_ptr<TensorData> &tens
<< " is the first run iteration for tensor: " << tensor->GetName();
return UINT32_MAX;
}
it--;
(void)it--;
prev_iter = *it;
tensor->SetPrevIteration(prev_iter);
return prev_iter;

18
mindspore/ccsrc/debug/debugger/debugger.cc
View File

@ -490,7 +490,7 @@ void Debugger::DumpParamsAndConstAndHistory() {
return;
}
LoadParametersAllGraphs();
(void)E2eDump::DumpParametersData(GetRankID(), debugger_.get());
E2eDump::DumpParametersData(GetRankID(), debugger_.get());
// Whether constant data was already dumped for the current root graph.
bool cur_root_graph_checked = std::find(visited_root_graph_ids_.begin(), visited_root_graph_ids_.end(),
cur_root_graph_id_) != visited_root_graph_ids_.end();
@ -1131,13 +1131,13 @@ void AddTensorStatInfo(const DebugServices::TensorStat &tensor_stat,
tensor_statistics->set_max_value(static_cast<float>(tensor_stat.max_value));
tensor_statistics->set_min_value(static_cast<float>(tensor_stat.min_value));
tensor_statistics->set_avg_value(static_cast<float>(tensor_stat.avg_value));
tensor_statistics->set_count(tensor_stat.count);
tensor_statistics->set_neg_zero_count(tensor_stat.neg_zero_count);
tensor_statistics->set_pos_zero_count(tensor_stat.pos_zero_count);
tensor_statistics->set_nan_count(tensor_stat.nan_count);
tensor_statistics->set_neg_inf_count(tensor_stat.neg_inf_count);
tensor_statistics->set_pos_inf_count(tensor_stat.pos_inf_count);
tensor_statistics->set_zero_count(tensor_stat.zero_count);
tensor_statistics->set_count(SizeToInt(tensor_stat.count));
tensor_statistics->set_neg_zero_count(SizeToInt(tensor_stat.neg_zero_count));
tensor_statistics->set_pos_zero_count(SizeToInt(tensor_stat.pos_zero_count));
tensor_statistics->set_nan_count(SizeToInt(tensor_stat.nan_count));
tensor_statistics->set_neg_inf_count(SizeToInt(tensor_stat.neg_inf_count));
tensor_statistics->set_pos_inf_count(SizeToInt(tensor_stat.pos_inf_count));
tensor_statistics->set_zero_count(SizeToInt(tensor_stat.zero_count));
tensor_summary_list->push_back(tensor_summary_item);
}
@ -1780,7 +1780,7 @@ std::shared_ptr<DumpDataBuilder> Debugger::LoadDumpDataBuilder(const std::string
return dump_data_construct_map_[node_name];
}
void Debugger::ClearDumpDataBuilder(const std::string &node_name) { dump_data_construct_map_.erase(node_name); }
void Debugger::ClearDumpDataBuilder(const std::string &node_name) { (void)dump_data_construct_map_.erase(node_name); }
#endif
} // namespace mindspore

8
mindspore/ccsrc/debug/debugger/debugger_utils.cc
View File

@ -88,8 +88,8 @@ void LoadInputs(const CNodePtr &cnode, const KernelLaunchInfo *launch_info, uint
auto device_addr = device_context->CreateDeviceAddress(addr->addr, addr->size, format, type);
string input_tensor_name = input_kernel_name + ':' + "0";
ShapeVector int_shapes = trans::GetRuntimePaddingShape(input_kernel, PARAMETER_OUTPUT_INDEX);
auto ret =
device_addr->LoadMemToHost(input_tensor_name, exec_order, format, int_shapes, type, 0, true, root_graph_id);
auto ret = device_addr->LoadMemToHost(input_tensor_name, UintToInt(exec_order), format, int_shapes, type, 0, true,
root_graph_id);
if (!ret) {
MS_LOG(ERROR) << "LoadMemToHost:"
<< ", tensor_name:" << input_tensor_name << ", host_format:" << format << ".!";
@ -124,7 +124,8 @@ void LoadOutputs(const CNodePtr &cnode, const KernelLaunchInfo *launch_info, uin
auto device_addr = device_context->CreateDeviceAddress(addr->addr, addr->size, format, type);
string tensor_name = kernel_name + ':' + std::to_string(j);
ShapeVector int_shapes = trans::GetRuntimePaddingShape(cnode, j);
auto ret = device_addr->LoadMemToHost(tensor_name, exec_order, format, int_shapes, type, j, false, root_graph_id);
auto ret =
device_addr->LoadMemToHost(tensor_name, UintToInt(exec_order), format, int_shapes, type, j, false, root_graph_id);
if (!ret) {
MS_LOG(ERROR) << "LoadMemToHost:"
<< ", tensor_name:" << tensor_name << ", host_format:" << format << ".!";
@ -299,6 +300,7 @@ void SuperKernelE2eDump(const KernelGraphPtr &graph) {
*/
int32_t DumpDataCallBack(const DumpChunk *dump_chunk, int32_t size) {
MS_LOG(DEBUG) << "ADX DumpDataCallBack is called";
MS_LOG(DEBUG) << "The dump_chunk size is: " << size;
string file_name = dump_chunk->fileName;
uint32_t isLastChunk = dump_chunk->isLastChunk;

20
mindspore/ccsrc/debug/debugger/tensor_summary.cc
View File

@ -174,31 +174,31 @@ void TensorSummary<T>::TensorStatistics(DbgDataType dtype_value) {
if (dtype_value == DT_BOOL) {
is_bool_ = true;
}
const int default_threads = 32;
const int default_elements_per_thread = 10000;
const uint64_t default_threads = 32;
const uint64_t default_elements_per_thread = 10000;
if (num_elements_ <= default_elements_per_thread) {
return TensorStatisticsSingleThread();
}
int desired_threads = num_elements_ / default_elements_per_thread;
int actual_threads = std::min(desired_threads, default_threads);
int actual_elements_per_thread = num_elements_ / actual_threads;
uint64_t desired_threads = num_elements_ / default_elements_per_thread;
uint64_t actual_threads = std::min(desired_threads, default_threads);
uint64_t actual_elements_per_thread = num_elements_ / actual_threads;
// Use multithread to calculate statistic on chunks of data
void *previous_tensor_ptr = nullptr;
size_t offset = 0;
std::vector<std::unique_ptr<TensorSummary<T>>> summary_vec;
std::vector<std::future<void>> summary_future_vec;
for (int i = 0; i < actual_threads; i++) {
int num_elements_for_thread;
for (uint64_t i = 0; i < actual_threads; i++) {
uint64_t num_elements_for_thread;
if (i == actual_threads - 1) {
num_elements_for_thread = num_elements_ - offset;
} else {
num_elements_for_thread = actual_elements_per_thread;
}
summary_vec.emplace_back(std::make_unique<TensorSummary<T>>(current_tensor_ptr_ + offset, previous_tensor_ptr,
num_elements_for_thread, 0));
summary_future_vec.emplace_back(
(void)summary_vec.emplace_back(std::make_unique<TensorSummary<T>>(current_tensor_ptr_ + offset, previous_tensor_ptr,
num_elements_for_thread, 0));
(void)summary_future_vec.emplace_back(
std::async(std::launch::async, &TensorSummary<T>::TensorStatisticsSingleThread, summary_vec[i].get()));
offset += num_elements_for_thread;
}

4
tests/st/debugger/test_read_tensors.py
View File

@ -19,9 +19,9 @@ Read tensor test script for offline debugger APIs.
import os
import json
import tempfile
import mindspore.offline_debug.dbg_services as d
import numpy as np
import pytest
import numpy as np
import mindspore.offline_debug.dbg_services as d
from tests.security_utils import security_off_wrap
from dump_test_utils import build_dump_structure, write_tensor_to_json