diff --git a/mindspore/ccsrc/backend/kernel_compiler/tbe/tbe_utils.cc b/mindspore/ccsrc/backend/kernel_compiler/tbe/tbe_utils.cc index 13405f51fd7..30680fe8b7a 100644 --- a/mindspore/ccsrc/backend/kernel_compiler/tbe/tbe_utils.cc +++ b/mindspore/ccsrc/backend/kernel_compiler/tbe/tbe_utils.cc @@ -195,7 +195,7 @@ nlohmann::json TbeUtils::GenSocInfo() { void TbeUtils::SaveJsonInfo(const std::string &json_name, const std::string &info) { auto config_path = TbeUtils::GetOpDebugPath(); std::string path = config_path + kCceKernelMeta + json_name + kInfoSuffix; - auto realpath = Common::GetRealPath(path); + auto realpath = Common::CreatePrefixPath(path); if (!realpath.has_value()) { MS_LOG(WARNING) << "Get real path failed, invalid path: " << realpath.value(); return; diff --git a/mindspore/ccsrc/backend/session/ascend_session.cc b/mindspore/ccsrc/backend/session/ascend_session.cc index f31a1c55fb2..4617b3acbba 100644 --- a/mindspore/ccsrc/backend/session/ascend_session.cc +++ b/mindspore/ccsrc/backend/session/ascend_session.cc @@ -291,7 +291,7 @@ size_t LoadCtrlInputTensor(const std::shared_ptr &graph, std::vecto void UpdateCtrlInputTensor(const std::shared_ptr &graph, std::vector *inputs, size_t *input_ctrl_size) { if (graph->input_ctrl_tensors()) { - bool sink_mode = (ConfigManager::GetInstance().dataset_mode() == DS_SINK_MODE || graph->isDatasetGraph()); + bool sink_mode = (ConfigManager::GetInstance().dataset_mode() == DS_SINK_MODE || graph->IsDatasetGraph()); if (sink_mode) { *input_ctrl_size = LoadCtrlInputTensor(graph, inputs); } else { diff --git a/mindspore/ccsrc/backend/session/kernel_graph.cc b/mindspore/ccsrc/backend/session/kernel_graph.cc index ff2c64baf58..bfd7c39cfc6 100644 --- a/mindspore/ccsrc/backend/session/kernel_graph.cc +++ b/mindspore/ccsrc/backend/session/kernel_graph.cc @@ -1371,8 +1371,8 @@ void KernelGraph::SetOptimizerFlag() { } } -bool KernelGraph::isDatasetGraph() const { - // check if there is GetNext or InitDataSetQueue node +bool KernelGraph::IsDatasetGraph() const { + // check if there is InitDataSetQueue node const auto &nodes = execution_order_; for (const auto &node : nodes) { auto node_name = AnfAlgo::GetCNodeName(node); diff --git a/mindspore/ccsrc/backend/session/kernel_graph.h b/mindspore/ccsrc/backend/session/kernel_graph.h index a9270e19de8..b0740972530 100644 --- a/mindspore/ccsrc/backend/session/kernel_graph.h +++ b/mindspore/ccsrc/backend/session/kernel_graph.h @@ -373,7 +373,7 @@ class KernelGraph : public FuncGraph { void set_is_need_gil(bool flag) { is_need_gil_ = flag; } bool is_need_gil() { return is_need_gil_; } - bool isDatasetGraph() const; + bool IsDatasetGraph() const; private: // remove value node form graph diff --git a/mindspore/ccsrc/debug/common.cc b/mindspore/ccsrc/debug/common.cc index f42c4676324..299b9f0df94 100644 --- a/mindspore/ccsrc/debug/common.cc +++ b/mindspore/ccsrc/debug/common.cc @@ -87,96 +87,6 @@ bool Common::CommonFuncForConfigPath(const std::string &default_path, const std: return true; } -std::optional Common::GetRealPath(const std::string &input_path) { - if (input_path.length() >= PATH_MAX) { - MS_LOG(ERROR) << "The length of path: " << input_path << " exceeds limit: " << PATH_MAX; - return std::nullopt; - } - auto path_split_pos = input_path.find_last_of('/'); - if (path_split_pos == std::string::npos) { - path_split_pos = input_path.find_last_of('\\'); - } - // get real path - char real_path[PATH_MAX] = {0}; - // input_path is dir + file_name - if (path_split_pos != std::string::npos) { - std::string prefix_path = input_path.substr(0, path_split_pos); - std::string file_name = input_path.substr(path_split_pos); - if (!CreateNotExistDirs(prefix_path)) { - MS_LOG(ERROR) << "Create dir " << prefix_path << " Failed!"; - return std::nullopt; - } -#if defined(SYSTEM_ENV_POSIX) - if (file_name.length() > NAME_MAX) { - MS_LOG(ERROR) << "The length of file name : " << file_name.length() << " exceeds limit: " << NAME_MAX; - return std::nullopt; - } - if (realpath(common::SafeCStr(prefix_path), real_path) == nullptr) { - MS_LOG(ERROR) << "The dir " << prefix_path << " does not exist."; - return std::nullopt; - } -#elif defined(SYSTEM_ENV_WINDOWS) - if (_fullpath(real_path, common::SafeCStr(prefix_path), PATH_MAX) == nullptr) { - MS_LOG(ERROR) << "The dir " << prefix_path << " does not exist."; - return std::nullopt; - } -#endif - return std::string(real_path) + file_name; - } - // input_path is only file_name -#if defined(SYSTEM_ENV_POSIX) - if (input_path.length() > NAME_MAX) { - MS_LOG(ERROR) << "The length of file name : " << input_path.length() << " exceeds limit: " << NAME_MAX; - return std::nullopt; - } - if (realpath(common::SafeCStr(input_path), real_path) == nullptr) { - MS_LOG(INFO) << "The file " << input_path << " does not exist, it will be created."; - } -#elif defined(SYSTEM_ENV_WINDOWS) - if (_fullpath(real_path, common::SafeCStr(input_path), PATH_MAX) == nullptr) { - MS_LOG(INFO) << "The file " << input_path << " does not exist, it will be created."; - } -#endif - return std::string(real_path); -} - -bool Common::CreateNotExistDirs(const std::string &path) { - std::shared_ptr fs = system::Env::GetFileSystem(); - MS_EXCEPTION_IF_NULL(fs); - char temp_path[PATH_MAX] = {0}; - if (path.length() >= PATH_MAX) { - MS_LOG(ERROR) << "Path length is equal to or max than " << PATH_MAX; - return false; - } - for (uint32_t i = 0; i < path.length(); i++) { - temp_path[i] = path[i]; - if (temp_path[i] == '\\' || temp_path[i] == '/') { - if (i != 0) { - char tmp_char = temp_path[i]; - temp_path[i] = '\0'; - std::string path_handle(temp_path); - if (!fs->FileExist(path_handle)) { - MS_LOG(INFO) << "Dir " << path_handle << " does not exit, creating..."; - if (!fs->CreateDir(path_handle)) { - MS_LOG(ERROR) << "Create " << path_handle << " dir error"; - return false; - } - } - temp_path[i] = tmp_char; - } - } - } - - if (!fs->FileExist(path)) { - MS_LOG(INFO) << "Dir " << path << " does not exit, creating..."; - if (!fs->CreateDir(path)) { - MS_LOG(ERROR) << "Create " << path << " dir error"; - return false; - } - } - return true; -} - std::optional Common::GetConfigFile(const std::string &env) { if (env.empty()) { MS_LOG(EXCEPTION) << "Invalid env"; @@ -333,7 +243,7 @@ bool Common::SaveStringToFile(const std::string filename, const std::string stri MS_LOG(ERROR) << "File path " << filename << " is too long."; return false; } - auto real_path = GetRealPath(filename); + auto real_path = FileUtils::GetRealPath(common::SafeCStr(filename)); if (!real_path.has_value()) { MS_LOG(ERROR) << "Get real path failed. path=" << filename; return false; diff --git a/mindspore/ccsrc/debug/common.h b/mindspore/ccsrc/debug/common.h index 69cf704543e..295602abfaf 100644 --- a/mindspore/ccsrc/debug/common.h +++ b/mindspore/ccsrc/debug/common.h @@ -33,12 +33,10 @@ class Common { ~Common() = default; static std::optional CreatePrefixPath(const std::string &input_path, const bool support_relative_path = false); - static std::optional GetRealPath(const std::string &input_path); static std::optional GetConfigFile(const std::string &env); static bool IsStrLengthValid(const std::string &str, size_t length_limit, const std::string &error_message = ""); static bool IsPathValid(const std::string &path, size_t length_limit, const std::string &error_message = ""); static bool IsFilenameValid(const std::string &filename, size_t length_limit, const std::string &error_message = ""); - static bool CreateNotExistDirs(const std::string &path); static std::string AddId(const std::string &filename, const std::string &suffix); static bool SaveStringToFile(const std::string filename, const std::string string_info); diff --git a/mindspore/ccsrc/debug/data_dump/e2e_dump.cc b/mindspore/ccsrc/debug/data_dump/e2e_dump.cc index 74e040fcb54..11e11eea94f 100644 --- a/mindspore/ccsrc/debug/data_dump/e2e_dump.cc +++ b/mindspore/ccsrc/debug/data_dump/e2e_dump.cc @@ -282,29 +282,19 @@ void E2eDump::UpdateIterDumpSetup(const session::KernelGraph *graph, bool sink_m } else if (starting_graph_id == graph_id) { dump_json_parser.UpdateDumpIter(); } - } else { - // If device target is Ascend + return; + } + // If device target is Ascend + if (sink_mode && graph->IsDatasetGraph()) { + MS_LOG(INFO) << "No need to update iteration for dataset graph."; + return; + } + if (starting_graph_id == INT32_MAX) { // Identify the first graph id and not increasing dump iter for the first iteration (initial dump iter = 0). - if (starting_graph_id == INT32_MAX) { - if (sink_mode) { - if (!CheckDatasetGraph(graph)) { - starting_graph_id = graph_id; - } - } else { - starting_graph_id = graph_id; - } - - } else { - // Update dump iter for ascend. - // In multi network scripts, dump iter is equal to the number of networks that have been run so far. - if (sink_mode) { - if (!CheckDatasetGraph(graph)) { - dump_json_parser.UpdateDumpIter(); - } - } else { - dump_json_parser.UpdateDumpIter(); - } - } + starting_graph_id = graph_id; + } else { + // In multi network scripts, dump iter is equal to the number of networks that have been run so far. + dump_json_parser.UpdateDumpIter(); } } @@ -372,19 +362,7 @@ bool E2eDump::isDatasetGraph(const session::KernelGraph *graph) { const auto &nodes = graph->execution_order(); for (const auto &node : nodes) { auto node_name = AnfAlgo::GetCNodeName(node); - if (node_name == "GetNext" || node_name == "InitDataSetQueue") { - return true; - } - } - return false; -} - -bool E2eDump::CheckDatasetGraph(const session::KernelGraph *graph) { - // Check if there is a InitDataSetQueue node to identify dataset graph. - const auto &nodes = graph->execution_order(); - for (const auto &node : nodes) { - auto node_name = AnfAlgo::GetCNodeName(node); - if (node_name == "InitDataSetQueue") { + if (node_name == prim::kPrimGetNext->name() || node_name == prim::kPrimInitDataSetQueue->name()) { return true; } } diff --git a/mindspore/ccsrc/debug/data_dump/e2e_dump.h b/mindspore/ccsrc/debug/data_dump/e2e_dump.h index 717a80d8bb3..ca5b4daa67e 100644 --- a/mindspore/ccsrc/debug/data_dump/e2e_dump.h +++ b/mindspore/ccsrc/debug/data_dump/e2e_dump.h @@ -45,8 +45,6 @@ class E2eDump { static bool isDatasetGraph(const session::KernelGraph *graph); - static bool CheckDatasetGraph(const session::KernelGraph *graph); - // Dump data when task error. static void DumpInputImpl(const CNodePtr &node, bool trans_flag, const std::string &dump_path, std::string *kernel_name, const Debugger *debugger); diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sbu_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sbu_op.cc index 9cb73b95916..4bf452af614 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sbu_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sbu_op.cc @@ -21,13 +21,13 @@ #include #include -#include "debug/common.h" #include "minddata/dataset/core/config_manager.h" #include "minddata/dataset/core/tensor_shape.h" #include "minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.h" #include "minddata/dataset/engine/db_connector.h" #include "minddata/dataset/engine/execution_tree.h" #include "utils/ms_utils.h" +#include "utils/file_utils.h" namespace mindspore { namespace dataset { @@ -142,7 +142,7 @@ Status SBUOp::ParseSBUData() { const Path url_file_name("SBU_captioned_photo_dataset_urls.txt"); const Path caption_file_name("SBU_captioned_photo_dataset_captions.txt"); const Path image_folder_name("sbu_images"); - auto real_folder_path = Common::GetRealPath(folder_path_); + auto real_folder_path = FileUtils::GetRealPath(common::SafeCStr(folder_path_)); CHECK_FAIL_RETURN_UNEXPECTED(real_folder_path.has_value(), "Get real path failed: " + folder_path_); Path root_dir(real_folder_path.value()); diff --git a/mindspore/ccsrc/runtime/device/kernel_adjust.cc b/mindspore/ccsrc/runtime/device/kernel_adjust.cc index 6adbf96f193..59842d8a0ce 100644 --- a/mindspore/ccsrc/runtime/device/kernel_adjust.cc +++ b/mindspore/ccsrc/runtime/device/kernel_adjust.cc @@ -708,7 +708,7 @@ CNodePtr KernelAdjust::CreateStreamAssignAddnOP(const std::shared_ptr &kernel_graph_ptr) { auto &dump_json_parser = DumpJsonParser::GetInstance(); - bool sink_mode = (ConfigManager::GetInstance().dataset_mode() == DS_SINK_MODE || kernel_graph_ptr->isDatasetGraph()); + bool sink_mode = (ConfigManager::GetInstance().dataset_mode() == DS_SINK_MODE || kernel_graph_ptr->IsDatasetGraph()); if (!sink_mode && dump_json_parser.async_dump_enabled()) { InitCtrlInputs(kernel_graph_ptr); return true; diff --git a/mindspore/ccsrc/runtime/framework/actor/recorder_actor.cc b/mindspore/ccsrc/runtime/framework/actor/recorder_actor.cc index 79901f78fb5..9f0acee80f0 100644 --- a/mindspore/ccsrc/runtime/framework/actor/recorder_actor.cc +++ b/mindspore/ccsrc/runtime/framework/actor/recorder_actor.cc @@ -48,11 +48,12 @@ void RecorderActor::RecordInfo(const std::string op_name, const KernelLaunchInfo void RecorderActor::RecordOnStepEnd(OpContext *const op_context) { MS_EXCEPTION_IF_NULL(op_context); - // todo clear +#ifndef ENABLE_SECURITY // Record iter_start, fp_start and iter_end op name and timestamp at the step end. (GPU) if (profiler::ProfilerManager::GetInstance()->GetProfilingEnableFlag()) { profiler::ProfilerManager::GetInstance()->RecordOneStepStartEndInfo(); } +#endif } } // namespace runtime } // namespace mindspore diff --git a/tests/ut/cpp/CMakeLists.txt b/tests/ut/cpp/CMakeLists.txt index 824dea36b37..7a036571ca3 100644 --- a/tests/ut/cpp/CMakeLists.txt +++ b/tests/ut/cpp/CMakeLists.txt @@ -50,7 +50,6 @@ if(ENABLE_MINDDATA) ./dataset/*.cc ./ir/dtype/*.cc ${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/*.cc - ./debug/*.cc ./device/*.cc ./ir/*.cc ./kernel/*.cc @@ -70,7 +69,11 @@ if(ENABLE_MINDDATA) ./cxx_api/*.cc ./tbe/*.cc ) - + if(NOT ENABLE_SECURITY) + file(GLOB_RECURSE UT_SRCS_DEBUG RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + ./debug/*.cc) + list(APPEND UT_SRCS ${UT_SRCS_DEBUG}) + endif() if(NOT ENABLE_PYTHON) set(PYTHON_RELATED_SRCS dataset/filter_op_test.cc @@ -190,6 +193,7 @@ if(ENABLE_SECURITY) list(REMOVE_ITEM MINDSPORE_SRC_LIST "../../../mindspore/ccsrc/profiler/device/profiling.cc") list(REMOVE_ITEM MINDSPORE_SRC_LIST "../../../mindspore/ccsrc/profiler/device/ascend/memory_profiling.cc") list(REMOVE_ITEM MINDSPORE_SRC_LIST "../../../mindspore/ccsrc/profiler/device/ascend/ascend_profiling.cc") + list(REMOVE_ITEM MINDSPORE_SRC_LIST "../../../mindspore/ccsrc/debug/data_dump/dump_json_parser.cc") endif() add_library(_ut_mindspore_obj OBJECT ${MINDSPORE_SRC_LIST})