pgjmblvch
/
mindspore2022
forked from huawei/mindspore2022
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

!18496 overflow dump may not gen data 

Merge pull request !18496 from john_tzanakakis/jt_bug_fixes
This commit is contained in:
i-robot 2021-06-18 07:38:15 +00:00 committed by Gitee
parent 0b7f3ebd89 690cf18cba
commit 4dd14e7b8f
1 changed files with 26 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
mindspore/ccsrc/debug/data_dump/e2e_dump.cc
View File

@ -335,18 +335,34 @@ bool E2eDump::DumpData(const session::KernelGraph *graph, uint32_t rank_id, cons
MS_LOG(EXCEPTION) << "failed at CreateNotExistDirs for " << cur_iter_dump_path;
}
// move contents from active dump dir to final dump dir
command = "mv " + zero_dir_dump_path + "/* " + cur_iter_dump_path + "/.";
MS_LOG(INFO) << "mv command: " << command;
if (system(command.c_str())) {
MS_LOG(EXCEPTION) << "Ascend runtime has changed the dump dir structure!!!";
// test if zero_dir_dump_path exists (may not if there was
// no data dumped, for example for an overflow dump)
command = "test -d " + zero_dir_dump_path;
MS_LOG(INFO) << "test command: " << command;
if (!system(command.c_str())) {
// move contents from active dump dir to final dump dir
command = "mv " + zero_dir_dump_path + "/* " + cur_iter_dump_path + "/.";
MS_LOG(INFO) << "mv command: " << command;
if (system(command.c_str())) {
MS_LOG(EXCEPTION) << "Ascend runtime has changed the dump dir structure!!!";
}
} else {
MS_LOG(INFO) << "active dump dir, not created yet";
}
} else {
// delete contents from active dump dir
std::string command = "rm -f " + zero_dir_dump_path + "/*";
MS_LOG(INFO) << "rm command: " << command;
if (system(command.c_str())) {
MS_LOG(EXCEPTION) << "Ascend runtime has changed the dump dir structure!!!";
// test if zero_dir_dump_path exists (may not if there was
// no data dumped, for example for an overflow dump)
std::string command = "test -d " + zero_dir_dump_path;
MS_LOG(INFO) << "test command: " << command;
if (!system(command.c_str())) {
// delete contents from active dump dir
command = "rm -f " + zero_dir_dump_path + "/*";
MS_LOG(INFO) << "rm command: " << command;
if (system(command.c_str())) {
MS_LOG(EXCEPTION) << "Ascend runtime has changed the dump dir structure!!!";
}
} else {
MS_LOG(INFO) << "active dump dir, not created yet";
}
}