Col_lin
/
mindspore2022
forked from huawei/mindspore2022
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

delete Common::GetRealPath

This commit is contained in:
yelihua 2021-09-17 17:46:20 +08:00
parent 53e64f65e5
commit 83b3f6fef7
12 changed files with 30 additions and 141 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
mindspore/ccsrc/backend/kernel_compiler/tbe/tbe_utils.cc
View File

@ -195,7 +195,7 @@ nlohmann::json TbeUtils::GenSocInfo() {
void TbeUtils::SaveJsonInfo(const std::string &json_name, const std::string &info) {
auto config_path = TbeUtils::GetOpDebugPath();
std::string path = config_path + kCceKernelMeta + json_name + kInfoSuffix;
auto realpath = Common::GetRealPath(path);
auto realpath = Common::CreatePrefixPath(path);
if (!realpath.has_value()) {
MS_LOG(WARNING) << "Get real path failed, invalid path: " << realpath.value();
return;

2
mindspore/ccsrc/backend/session/ascend_session.cc
View File

@ -291,7 +291,7 @@ size_t LoadCtrlInputTensor(const std::shared_ptr<KernelGraph> &graph, std::vecto
void UpdateCtrlInputTensor(const std::shared_ptr<KernelGraph> &graph, std::vector<tensor::TensorPtr> *inputs,
size_t *input_ctrl_size) {
if (graph->input_ctrl_tensors()) {
bool sink_mode = (ConfigManager::GetInstance().dataset_mode() == DS_SINK_MODE || graph->isDatasetGraph());
bool sink_mode = (ConfigManager::GetInstance().dataset_mode() == DS_SINK_MODE || graph->IsDatasetGraph());
if (sink_mode) {
*input_ctrl_size = LoadCtrlInputTensor(graph, inputs);
} else {

4
mindspore/ccsrc/backend/session/kernel_graph.cc
View File

@ -1371,8 +1371,8 @@ void KernelGraph::SetOptimizerFlag() {
}
}
bool KernelGraph::isDatasetGraph() const {
// check if there is GetNext or InitDataSetQueue node
bool KernelGraph::IsDatasetGraph() const {
// check if there is InitDataSetQueue node
const auto &nodes = execution_order_;
for (const auto &node : nodes) {
auto node_name = AnfAlgo::GetCNodeName(node);

2
mindspore/ccsrc/backend/session/kernel_graph.h
View File

@ -373,7 +373,7 @@ class KernelGraph : public FuncGraph {
void set_is_need_gil(bool flag) { is_need_gil_ = flag; }
bool is_need_gil() { return is_need_gil_; }
bool isDatasetGraph() const;
bool IsDatasetGraph() const;
private:
// remove value node form graph

92
mindspore/ccsrc/debug/common.cc
View File

@ -87,96 +87,6 @@ bool Common::CommonFuncForConfigPath(const std::string &default_path, const std:
return true;
}
std::optional<std::string> Common::GetRealPath(const std::string &input_path) {
if (input_path.length() >= PATH_MAX) {
MS_LOG(ERROR) << "The length of path: " << input_path << " exceeds limit: " << PATH_MAX;
return std::nullopt;
}
auto path_split_pos = input_path.find_last_of('/');
if (path_split_pos == std::string::npos) {
path_split_pos = input_path.find_last_of('\\');
}
// get real path
char real_path[PATH_MAX] = {0};
// input_path is dir + file_name
if (path_split_pos != std::string::npos) {
std::string prefix_path = input_path.substr(0, path_split_pos);
std::string file_name = input_path.substr(path_split_pos);
if (!CreateNotExistDirs(prefix_path)) {
MS_LOG(ERROR) << "Create dir " << prefix_path << " Failed!";
return std::nullopt;
}
#if defined(SYSTEM_ENV_POSIX)
if (file_name.length() > NAME_MAX) {
MS_LOG(ERROR) << "The length of file name : " << file_name.length() << " exceeds limit: " << NAME_MAX;
return std::nullopt;
}
if (realpath(common::SafeCStr(prefix_path), real_path) == nullptr) {
MS_LOG(ERROR) << "The dir " << prefix_path << " does not exist.";
return std::nullopt;
}
#elif defined(SYSTEM_ENV_WINDOWS)
if (_fullpath(real_path, common::SafeCStr(prefix_path), PATH_MAX) == nullptr) {
MS_LOG(ERROR) << "The dir " << prefix_path << " does not exist.";
return std::nullopt;
}
#endif
return std::string(real_path) + file_name;
}
// input_path is only file_name
#if defined(SYSTEM_ENV_POSIX)
if (input_path.length() > NAME_MAX) {
MS_LOG(ERROR) << "The length of file name : " << input_path.length() << " exceeds limit: " << NAME_MAX;
return std::nullopt;
}
if (realpath(common::SafeCStr(input_path), real_path) == nullptr) {
MS_LOG(INFO) << "The file " << input_path << " does not exist, it will be created.";
}
#elif defined(SYSTEM_ENV_WINDOWS)
if (_fullpath(real_path, common::SafeCStr(input_path), PATH_MAX) == nullptr) {
MS_LOG(INFO) << "The file " << input_path << " does not exist, it will be created.";
}
#endif
return std::string(real_path);
}
bool Common::CreateNotExistDirs(const std::string &path) {
std::shared_ptr<system::FileSystem> fs = system::Env::GetFileSystem();
MS_EXCEPTION_IF_NULL(fs);
char temp_path[PATH_MAX] = {0};
if (path.length() >= PATH_MAX) {
MS_LOG(ERROR) << "Path length is equal to or max than " << PATH_MAX;
return false;
}
for (uint32_t i = 0; i < path.length(); i++) {
temp_path[i] = path[i];
if (temp_path[i] == '\\' || temp_path[i] == '/') {
if (i != 0) {
char tmp_char = temp_path[i];
temp_path[i] = '\0';
std::string path_handle(temp_path);
if (!fs->FileExist(path_handle)) {
MS_LOG(INFO) << "Dir " << path_handle << " does not exit, creating...";
if (!fs->CreateDir(path_handle)) {
MS_LOG(ERROR) << "Create " << path_handle << " dir error";
return false;
}
}
temp_path[i] = tmp_char;
}
}
}
if (!fs->FileExist(path)) {
MS_LOG(INFO) << "Dir " << path << " does not exit, creating...";
if (!fs->CreateDir(path)) {
MS_LOG(ERROR) << "Create " << path << " dir error";
return false;
}
}
return true;
}
std::optional<std::string> Common::GetConfigFile(const std::string &env) {
if (env.empty()) {
MS_LOG(EXCEPTION) << "Invalid env";
@ -333,7 +243,7 @@ bool Common::SaveStringToFile(const std::string filename, const std::string stri
MS_LOG(ERROR) << "File path " << filename << " is too long.";
return false;
}
auto real_path = GetRealPath(filename);
auto real_path = FileUtils::GetRealPath(common::SafeCStr(filename));
if (!real_path.has_value()) {
MS_LOG(ERROR) << "Get real path failed. path=" << filename;
return false;

2
mindspore/ccsrc/debug/common.h
View File

@ -33,12 +33,10 @@ class Common {
~Common() = default;
static std::optional<std::string> CreatePrefixPath(const std::string &input_path,
const bool support_relative_path = false);
static std::optional<std::string> GetRealPath(const std::string &input_path);
static std::optional<std::string> GetConfigFile(const std::string &env);
static bool IsStrLengthValid(const std::string &str, size_t length_limit, const std::string &error_message = "");
static bool IsPathValid(const std::string &path, size_t length_limit, const std::string &error_message = "");
static bool IsFilenameValid(const std::string &filename, size_t length_limit, const std::string &error_message = "");
static bool CreateNotExistDirs(const std::string &path);
static std::string AddId(const std::string &filename, const std::string &suffix);
static bool SaveStringToFile(const std::string filename, const std::string string_info);

48
mindspore/ccsrc/debug/data_dump/e2e_dump.cc
View File

@ -282,29 +282,19 @@ void E2eDump::UpdateIterDumpSetup(const session::KernelGraph *graph, bool sink_m
} else if (starting_graph_id == graph_id) {
dump_json_parser.UpdateDumpIter();
}
} else {
// If device target is Ascend
return;
}
// If device target is Ascend
if (sink_mode && graph->IsDatasetGraph()) {
MS_LOG(INFO) << "No need to update iteration for dataset graph.";
return;
}
if (starting_graph_id == INT32_MAX) {
// Identify the first graph id and not increasing dump iter for the first iteration (initial dump iter = 0).
if (starting_graph_id == INT32_MAX) {
if (sink_mode) {
if (!CheckDatasetGraph(graph)) {
starting_graph_id = graph_id;
}
} else {
starting_graph_id = graph_id;
}
} else {
// Update dump iter for ascend.
// In multi network scripts, dump iter is equal to the number of networks that have been run so far.
if (sink_mode) {
if (!CheckDatasetGraph(graph)) {
dump_json_parser.UpdateDumpIter();
}
} else {
dump_json_parser.UpdateDumpIter();
}
}
starting_graph_id = graph_id;
} else {
// In multi network scripts, dump iter is equal to the number of networks that have been run so far.
dump_json_parser.UpdateDumpIter();
}
}
@ -372,19 +362,7 @@ bool E2eDump::isDatasetGraph(const session::KernelGraph *graph) {
const auto &nodes = graph->execution_order();
for (const auto &node : nodes) {
auto node_name = AnfAlgo::GetCNodeName(node);
if (node_name == "GetNext" || node_name == "InitDataSetQueue") {
return true;
}
}
return false;
}
bool E2eDump::CheckDatasetGraph(const session::KernelGraph *graph) {
// Check if there is a InitDataSetQueue node to identify dataset graph.
const auto &nodes = graph->execution_order();
for (const auto &node : nodes) {
auto node_name = AnfAlgo::GetCNodeName(node);
if (node_name == "InitDataSetQueue") {
if (node_name == prim::kPrimGetNext->name() || node_name == prim::kPrimInitDataSetQueue->name()) {
return true;
}
}

2
mindspore/ccsrc/debug/data_dump/e2e_dump.h
View File

@ -45,8 +45,6 @@ class E2eDump {
static bool isDatasetGraph(const session::KernelGraph *graph);
static bool CheckDatasetGraph(const session::KernelGraph *graph);
// Dump data when task error.
static void DumpInputImpl(const CNodePtr &node, bool trans_flag, const std::string &dump_path,
std::string *kernel_name, const Debugger *debugger);

4
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sbu_op.cc
View File

@ -21,13 +21,13 @@
#include <set>
#include <utility>
#include "debug/common.h"
#include "minddata/dataset/core/config_manager.h"
#include "minddata/dataset/core/tensor_shape.h"
#include "minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.h"
#include "minddata/dataset/engine/db_connector.h"
#include "minddata/dataset/engine/execution_tree.h"
#include "utils/ms_utils.h"
#include "utils/file_utils.h"
namespace mindspore {
namespace dataset {
@ -142,7 +142,7 @@ Status SBUOp::ParseSBUData() {
const Path url_file_name("SBU_captioned_photo_dataset_urls.txt");
const Path caption_file_name("SBU_captioned_photo_dataset_captions.txt");
const Path image_folder_name("sbu_images");
auto real_folder_path = Common::GetRealPath(folder_path_);
auto real_folder_path = FileUtils::GetRealPath(common::SafeCStr(folder_path_));
CHECK_FAIL_RETURN_UNEXPECTED(real_folder_path.has_value(), "Get real path failed: " + folder_path_);
Path root_dir(real_folder_path.value());

2
mindspore/ccsrc/runtime/device/kernel_adjust.cc
View File

@ -708,7 +708,7 @@ CNodePtr KernelAdjust::CreateStreamAssignAddnOP(const std::shared_ptr<session::K
bool KernelAdjust::StepLoadCtrlInputs(const std::shared_ptr<session::KernelGraph> &kernel_graph_ptr) {
auto &dump_json_parser = DumpJsonParser::GetInstance();
bool sink_mode = (ConfigManager::GetInstance().dataset_mode() == DS_SINK_MODE || kernel_graph_ptr->isDatasetGraph());
bool sink_mode = (ConfigManager::GetInstance().dataset_mode() == DS_SINK_MODE || kernel_graph_ptr->IsDatasetGraph());
if (!sink_mode && dump_json_parser.async_dump_enabled()) {
InitCtrlInputs(kernel_graph_ptr);
return true;

3
mindspore/ccsrc/runtime/framework/actor/recorder_actor.cc
View File

@ -48,11 +48,12 @@ void RecorderActor::RecordInfo(const std::string op_name, const KernelLaunchInfo
void RecorderActor::RecordOnStepEnd(OpContext<DeviceTensor> *const op_context) {
MS_EXCEPTION_IF_NULL(op_context);
// todo clear
#ifndef ENABLE_SECURITY
// Record iter_start, fp_start and iter_end op name and timestamp at the step end. (GPU)
if (profiler::ProfilerManager::GetInstance()->GetProfilingEnableFlag()) {
profiler::ProfilerManager::GetInstance()->RecordOneStepStartEndInfo();
}
#endif
}
} // namespace runtime
} // namespace mindspore

8
tests/ut/cpp/CMakeLists.txt
View File

@ -50,7 +50,6 @@ if(ENABLE_MINDDATA)
./dataset/*.cc
./ir/dtype/*.cc
${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/*.cc
./debug/*.cc
./device/*.cc
./ir/*.cc
./kernel/*.cc
@ -70,7 +69,11 @@ if(ENABLE_MINDDATA)
./cxx_api/*.cc
./tbe/*.cc
)
if(NOT ENABLE_SECURITY)
file(GLOB_RECURSE UT_SRCS_DEBUG RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
./debug/*.cc)
list(APPEND UT_SRCS ${UT_SRCS_DEBUG})
endif()
if(NOT ENABLE_PYTHON)
set(PYTHON_RELATED_SRCS
dataset/filter_op_test.cc
@ -190,6 +193,7 @@ if(ENABLE_SECURITY)
list(REMOVE_ITEM MINDSPORE_SRC_LIST "../../../mindspore/ccsrc/profiler/device/profiling.cc")
list(REMOVE_ITEM MINDSPORE_SRC_LIST "../../../mindspore/ccsrc/profiler/device/ascend/memory_profiling.cc")
list(REMOVE_ITEM MINDSPORE_SRC_LIST "../../../mindspore/ccsrc/profiler/device/ascend/ascend_profiling.cc")
list(REMOVE_ITEM MINDSPORE_SRC_LIST "../../../mindspore/ccsrc/debug/data_dump/dump_json_parser.cc")
endif()
add_library(_ut_mindspore_obj OBJECT ${MINDSPORE_SRC_LIST})