From a748d2c8af2e6863e637b01ebdf3df2ca1888506 Mon Sep 17 00:00:00 2001 From: ougongchang Date: Tue, 16 Nov 2021 11:16:06 +0800 Subject: [PATCH] fix profiling parallel strategy can not get data in SaveCompileGraph phase --- mindspore/ccsrc/debug/dump_proto.cc | 25 ++++---- mindspore/ccsrc/pipeline/jit/pipeline.cc | 57 +++++++++++-------- .../ascend/parallel_strategy_profiling.cc | 1 + 3 files changed, 47 insertions(+), 36 deletions(-) diff --git a/mindspore/ccsrc/debug/dump_proto.cc b/mindspore/ccsrc/debug/dump_proto.cc index bf6ba16a088..d09623609e2 100644 --- a/mindspore/ccsrc/debug/dump_proto.cc +++ b/mindspore/ccsrc/debug/dump_proto.cc @@ -41,7 +41,7 @@ class ProtoExporter { private: void InitModelInfo(); - void GetOpNodeTypeAndAttrs(const FuncGraphPtr &func_graph, const AnfNodePtr &node, irpb::NodeProto *node_proto); + void GetOpNodeTypeAndAttrs(const FuncGraphPtr &func_graph, const CNodePtr &cnode, irpb::NodeProto *node_proto); std::string GetOpNodeInputId(const FuncGraphPtr &func_graph, const AnfNodePtr &node, const std::map &apply_map, std::map *const_map_ptr); @@ -326,23 +326,26 @@ void ProtoExporter::SetDictionaryToProto(const ValueDictionaryPtr &val, irpb::Va } } -void ProtoExporter::GetOpNodeTypeAndAttrs(const FuncGraphPtr &, const AnfNodePtr &node, irpb::NodeProto *node_proto) { - if (node == nullptr || node_proto == nullptr) { +void ProtoExporter::GetOpNodeTypeAndAttrs(const FuncGraphPtr &, const CNodePtr &cnode, irpb::NodeProto *node_proto) { + const auto &inputs = cnode->inputs(); + AnfNodePtr op_node = inputs[0]; + + if (op_node == nullptr || node_proto == nullptr) { return; } - if (node->isa() || node->isa() || IsValueNode(node)) { - MS_LOG(EXCEPTION) << "Op node can not be CNode, Parameter or ValueNode Graph. But got " << node->ToString(); + if (op_node->isa() || op_node->isa() || IsValueNode(op_node)) { + MS_LOG(EXCEPTION) << "Op node can not be CNode, Parameter or ValueNode Graph. But got " << op_node->ToString(); } - if (!IsValueNode(node)) { - MS_LOG(EXCEPTION) << "Op node is not primitive: " << node->ToString(); + if (!IsValueNode(op_node)) { + MS_LOG(EXCEPTION) << "Op node is not primitive: " << op_node->ToString(); } - const PrimitivePtr &prim = GetValueNode(node); + const PrimitivePtr &prim = GetValueNode(op_node); // set node parallel info - auto operator_info = node->user_data(); + auto operator_info = cnode->user_data(); if (operator_info != nullptr) { auto strategy = operator_info->strategy(); if (strategy != nullptr) { @@ -360,7 +363,7 @@ void ProtoExporter::GetOpNodeTypeAndAttrs(const FuncGraphPtr &, const AnfNodePtr attr_proto->set_name(attr.first); SetValueToProto(attr.second, attr_proto->mutable_value()); } - node_proto->set_scope(node->scope()->name()); + node_proto->set_scope(op_node->scope()->name()); } std::string ProtoExporter::GetOpNodeInputId(const FuncGraphPtr &, const AnfNodePtr &node, @@ -487,7 +490,7 @@ void ProtoExporter::ExportCNode(const FuncGraphPtr &func_graph, const CNodePtr & if (op->isa() || IsValueNode(op) || op->isa()) { MS_LOG(DEBUG) << "Operator must be a primitive"; } else { - GetOpNodeTypeAndAttrs(func_graph, op, node_proto); + GetOpNodeTypeAndAttrs(func_graph, node, node_proto); node_proto->set_name(std::to_string(apply_idx)); node_proto->set_scope(node->scope()->name()); node_proto->set_full_name(GetKernelNodeName(node)); diff --git a/mindspore/ccsrc/pipeline/jit/pipeline.cc b/mindspore/ccsrc/pipeline/jit/pipeline.cc index ee23b24e2df..dc658500903 100644 --- a/mindspore/ccsrc/pipeline/jit/pipeline.cc +++ b/mindspore/ccsrc/pipeline/jit/pipeline.cc @@ -657,12 +657,6 @@ void GraphExecutorPy::SaveCompiledGraph(const std::string &phase) { MS_LOG(INFO) << "Save compiled func graph(" << func_graph->ToString() << ") phase(" << phase << ")!"; info_[phase]->func_graph = func_graph; -#ifndef ENABLE_SECURITY -#ifdef ENABLE_D - profiler::ascend::DumpProfileParallelStrategy(func_graph); -#endif -#endif - if ((func_graph != nullptr) && func_graph->has_flag(parallel::AUTO_PARALLEL) && ((parallel_mode == parallel::AUTO_PARALLEL) || (parallel_mode == parallel::SEMI_AUTO_PARALLEL))) { MS_LOG(DEBUG) << "Save model parallel parameter layout graph!"; @@ -919,6 +913,30 @@ void CheckInterpretNodeLineInfos() { InterpretNodeRecorder::GetInstance().Clear(); } +#ifdef ENABLE_DUMP_IR +void RDRRecordGraph(const size_t action_index, const size_t action_size, const std::string &filename, + const FuncGraphPtr graph) { + if (mindspore::RecorderManager::Instance().RdrEnable()) { + MS_LOG(INFO) << "Recording FuncGraph in pipeline using RDR."; + if (graph != nullptr) { + auto graph_clone = BasicClone(graph); + if (graph_clone != nullptr) { + DumpGraphParams dump_params = {false, static_cast(kTopStack)}; + if (action_index == action_size) { + dump_params.dump_mode = static_cast(kWholeStack); + } + (void)mindspore::RDR::RecordAnfGraph(SUBMODULE_ID, filename, graph_clone, dump_params, ".ir"); + } else { + MS_LOG(WARNING) << "Clone FuncGraph failed in pipeline, no FuncGraph recording in RDR."; + } + } else { + MS_LOG(WARNING) << "Pipeline Resource has no FuncGraph, no FuncGraph recording in RDR"; + } + MS_LOG(INFO) << "Recording FuncGraph in pipeline end."; + } +} +#endif + void Pipeline::Run(const std::string &phase) { MS_LOG(INFO) << "Pipeline run"; MS_EXCEPTION_IF_NULL(resource_); @@ -942,6 +960,12 @@ void Pipeline::Run(const std::string &phase) { } else if (action.first == "validate") { CheckInterpretNodeLineInfos(); CacheValidateFuncGraph(phase, resource_); +#ifndef ENABLE_SECURITY +#ifdef ENABLE_D + FuncGraphPtr graph = resource_->func_graph(); + profiler::ascend::DumpProfileParallelStrategy(graph); +#endif +#endif } if (!result) { MS_LOG(EXCEPTION) << "Pipeline running to end, failed in step:" << action.first; @@ -949,25 +973,8 @@ void Pipeline::Run(const std::string &phase) { FuncGraphPtr graph = resource_->func_graph(); #ifdef ENABLE_DUMP_IR - if (mindspore::RecorderManager::Instance().RdrEnable()) { - MS_LOG(INFO) << "Recording FuncGraph in pipeline using RDR."; - std::string name = GetBaseNameForIR(SizeToLong(i), action.first); - if (graph != nullptr) { - auto graph_clone = BasicClone(graph); - if (graph_clone != nullptr) { - DumpGraphParams dump_params = {false, static_cast(kTopStack)}; - if (i == actions_.size()) { - dump_params.dump_mode = static_cast(kWholeStack); - } - (void)mindspore::RDR::RecordAnfGraph(SUBMODULE_ID, name, graph_clone, dump_params, ".ir"); - } else { - MS_LOG(WARNING) << "Clone FuncGraph failed in pipeline, no FuncGraph recording in RDR."; - } - } else { - MS_LOG(WARNING) << "Pipeline Resource has no FuncGraph, no FuncGraph recording in RDR"; - } - MS_LOG(INFO) << "Recording FuncGraph in pipeline end."; - } + std::string filename = GetBaseNameForIR(SizeToLong(i), action.first); + RDRRecordGraph(i, actions_.size(), filename, graph); if (MsContext::GetInstance()->get_param(MS_CTX_SAVE_GRAPHS_FLAG) && graph != nullptr) { user_graph = graph; diff --git a/mindspore/ccsrc/profiler/device/ascend/parallel_strategy_profiling.cc b/mindspore/ccsrc/profiler/device/ascend/parallel_strategy_profiling.cc index a708b4bfd94..29a050b3dd1 100644 --- a/mindspore/ccsrc/profiler/device/ascend/parallel_strategy_profiling.cc +++ b/mindspore/ccsrc/profiler/device/ascend/parallel_strategy_profiling.cc @@ -39,6 +39,7 @@ bool IsProfilingParallelStrategyEnabled() { auto ascend_profiler = AscendProfiler::GetInstance(); MS_EXCEPTION_IF_NULL(ascend_profiler); if (!ascend_profiler->GetProfilingEnableFlag()) { + MS_LOG(INFO) << "Profiling parallel strategy is disabled."; return false; }