From 690cf18cba5793d57fe4de5cce061f80d3d0f1e3 Mon Sep 17 00:00:00 2001 From: John Tzanakakis Date: Thu, 17 Jun 2021 15:16:20 -0400 Subject: [PATCH] overflow dump may not gen data --- mindspore/ccsrc/debug/data_dump/e2e_dump.cc | 36 +++++++++++++++------ 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/mindspore/ccsrc/debug/data_dump/e2e_dump.cc b/mindspore/ccsrc/debug/data_dump/e2e_dump.cc index cafa30e71e5..e33b89beed7 100644 --- a/mindspore/ccsrc/debug/data_dump/e2e_dump.cc +++ b/mindspore/ccsrc/debug/data_dump/e2e_dump.cc @@ -335,18 +335,34 @@ bool E2eDump::DumpData(const session::KernelGraph *graph, uint32_t rank_id, cons MS_LOG(EXCEPTION) << "failed at CreateNotExistDirs for " << cur_iter_dump_path; } - // move contents from active dump dir to final dump dir - command = "mv " + zero_dir_dump_path + "/* " + cur_iter_dump_path + "/."; - MS_LOG(INFO) << "mv command: " << command; - if (system(command.c_str())) { - MS_LOG(EXCEPTION) << "Ascend runtime has changed the dump dir structure!!!"; + // test if zero_dir_dump_path exists (may not if there was + // no data dumped, for example for an overflow dump) + command = "test -d " + zero_dir_dump_path; + MS_LOG(INFO) << "test command: " << command; + if (!system(command.c_str())) { + // move contents from active dump dir to final dump dir + command = "mv " + zero_dir_dump_path + "/* " + cur_iter_dump_path + "/."; + MS_LOG(INFO) << "mv command: " << command; + if (system(command.c_str())) { + MS_LOG(EXCEPTION) << "Ascend runtime has changed the dump dir structure!!!"; + } + } else { + MS_LOG(INFO) << "active dump dir, not created yet"; } } else { - // delete contents from active dump dir - std::string command = "rm -f " + zero_dir_dump_path + "/*"; - MS_LOG(INFO) << "rm command: " << command; - if (system(command.c_str())) { - MS_LOG(EXCEPTION) << "Ascend runtime has changed the dump dir structure!!!"; + // test if zero_dir_dump_path exists (may not if there was + // no data dumped, for example for an overflow dump) + std::string command = "test -d " + zero_dir_dump_path; + MS_LOG(INFO) << "test command: " << command; + if (!system(command.c_str())) { + // delete contents from active dump dir + command = "rm -f " + zero_dir_dump_path + "/*"; + MS_LOG(INFO) << "rm command: " << command; + if (system(command.c_str())) { + MS_LOG(EXCEPTION) << "Ascend runtime has changed the dump dir structure!!!"; + } + } else { + MS_LOG(INFO) << "active dump dir, not created yet"; } }