Col_lin
/
mindspore2022
forked from huawei/mindspore2022
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix profiling parallel strategy can not get data in SaveCompileGraph phase

This commit is contained in:
ougongchang 2021-11-16 11:16:06 +08:00
parent 50bbdfef17
commit a748d2c8af
3 changed files with 47 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
mindspore/ccsrc/debug/dump_proto.cc
View File

@ -41,7 +41,7 @@ class ProtoExporter {
private:
void InitModelInfo();
void GetOpNodeTypeAndAttrs(const FuncGraphPtr &func_graph, const AnfNodePtr &node, irpb::NodeProto *node_proto);
void GetOpNodeTypeAndAttrs(const FuncGraphPtr &func_graph, const CNodePtr &cnode, irpb::NodeProto *node_proto);
std::string GetOpNodeInputId(const FuncGraphPtr &func_graph, const AnfNodePtr &node,
const std::map<AnfNodePtr, size_t> &apply_map,
std::map<AnfNodePtr, size_t> *const_map_ptr);
@ -326,23 +326,26 @@ void ProtoExporter::SetDictionaryToProto(const ValueDictionaryPtr &val, irpb::Va
}
}
void ProtoExporter::GetOpNodeTypeAndAttrs(const FuncGraphPtr &, const AnfNodePtr &node, irpb::NodeProto *node_proto) {
if (node == nullptr || node_proto == nullptr) {
void ProtoExporter::GetOpNodeTypeAndAttrs(const FuncGraphPtr &, const CNodePtr &cnode, irpb::NodeProto *node_proto) {
const auto &inputs = cnode->inputs();
AnfNodePtr op_node = inputs[0];
if (op_node == nullptr || node_proto == nullptr) {
return;
}
if (node->isa<CNode>() || node->isa<Parameter>() || IsValueNode<FuncGraph>(node)) {
MS_LOG(EXCEPTION) << "Op node can not be CNode, Parameter or ValueNode Graph. But got " << node->ToString();
if (op_node->isa<CNode>() || op_node->isa<Parameter>() || IsValueNode<FuncGraph>(op_node)) {
MS_LOG(EXCEPTION) << "Op node can not be CNode, Parameter or ValueNode Graph. But got " << op_node->ToString();
}
if (!IsValueNode<Primitive>(node)) {
MS_LOG(EXCEPTION) << "Op node is not primitive: " << node->ToString();
if (!IsValueNode<Primitive>(op_node)) {
MS_LOG(EXCEPTION) << "Op node is not primitive: " << op_node->ToString();
}
const PrimitivePtr &prim = GetValueNode<PrimitivePtr>(node);
const PrimitivePtr &prim = GetValueNode<PrimitivePtr>(op_node);
// set node parallel info
auto operator_info = node->user_data<parallel::OperatorInfo>();
auto operator_info = cnode->user_data<parallel::OperatorInfo>();
if (operator_info != nullptr) {
auto strategy = operator_info->strategy();
if (strategy != nullptr) {
@ -360,7 +363,7 @@ void ProtoExporter::GetOpNodeTypeAndAttrs(const FuncGraphPtr &, const AnfNodePtr
attr_proto->set_name(attr.first);
SetValueToProto(attr.second, attr_proto->mutable_value());
}
node_proto->set_scope(node->scope()->name());
node_proto->set_scope(op_node->scope()->name());
}
std::string ProtoExporter::GetOpNodeInputId(const FuncGraphPtr &, const AnfNodePtr &node,
@ -487,7 +490,7 @@ void ProtoExporter::ExportCNode(const FuncGraphPtr &func_graph, const CNodePtr &
if (op->isa<CNode>() || IsValueNode<FuncGraph>(op) || op->isa<Parameter>()) {
MS_LOG(DEBUG) << "Operator must be a primitive";
} else {
GetOpNodeTypeAndAttrs(func_graph, op, node_proto);
GetOpNodeTypeAndAttrs(func_graph, node, node_proto);
node_proto->set_name(std::to_string(apply_idx));
node_proto->set_scope(node->scope()->name());
node_proto->set_full_name(GetKernelNodeName(node));

57
mindspore/ccsrc/pipeline/jit/pipeline.cc
View File

@ -657,12 +657,6 @@ void GraphExecutorPy::SaveCompiledGraph(const std::string &phase) {
MS_LOG(INFO) << "Save compiled func graph(" << func_graph->ToString() << ") phase(" << phase << ")!";
info_[phase]->func_graph = func_graph;
#ifndef ENABLE_SECURITY
#ifdef ENABLE_D
profiler::ascend::DumpProfileParallelStrategy(func_graph);
#endif
#endif
if ((func_graph != nullptr) && func_graph->has_flag(parallel::AUTO_PARALLEL) &&
((parallel_mode == parallel::AUTO_PARALLEL) || (parallel_mode == parallel::SEMI_AUTO_PARALLEL))) {
MS_LOG(DEBUG) << "Save model parallel parameter layout graph!";
@ -919,6 +913,30 @@ void CheckInterpretNodeLineInfos() {
InterpretNodeRecorder::GetInstance().Clear();
}
#ifdef ENABLE_DUMP_IR
void RDRRecordGraph(const size_t action_index, const size_t action_size, const std::string &filename,
const FuncGraphPtr graph) {
if (mindspore::RecorderManager::Instance().RdrEnable()) {
MS_LOG(INFO) << "Recording FuncGraph in pipeline using RDR.";
if (graph != nullptr) {
auto graph_clone = BasicClone(graph);
if (graph_clone != nullptr) {
DumpGraphParams dump_params = {false, static_cast<int>(kTopStack)};
if (action_index == action_size) {
dump_params.dump_mode = static_cast<int>(kWholeStack);
}
(void)mindspore::RDR::RecordAnfGraph(SUBMODULE_ID, filename, graph_clone, dump_params, ".ir");
} else {
MS_LOG(WARNING) << "Clone FuncGraph failed in pipeline, no FuncGraph recording in RDR.";
}
} else {
MS_LOG(WARNING) << "Pipeline Resource has no FuncGraph, no FuncGraph recording in RDR";
}
MS_LOG(INFO) << "Recording FuncGraph in pipeline end.";
}
}
#endif
void Pipeline::Run(const std::string &phase) {
MS_LOG(INFO) << "Pipeline run";
MS_EXCEPTION_IF_NULL(resource_);
@ -942,6 +960,12 @@ void Pipeline::Run(const std::string &phase) {
} else if (action.first == "validate") {
CheckInterpretNodeLineInfos();
CacheValidateFuncGraph(phase, resource_);
#ifndef ENABLE_SECURITY
#ifdef ENABLE_D
FuncGraphPtr graph = resource_->func_graph();
profiler::ascend::DumpProfileParallelStrategy(graph);
#endif
#endif
}
if (!result) {
MS_LOG(EXCEPTION) << "Pipeline running to end, failed in step:" << action.first;
@ -949,25 +973,8 @@ void Pipeline::Run(const std::string &phase) {
FuncGraphPtr graph = resource_->func_graph();
#ifdef ENABLE_DUMP_IR
if (mindspore::RecorderManager::Instance().RdrEnable()) {
MS_LOG(INFO) << "Recording FuncGraph in pipeline using RDR.";
std::string name = GetBaseNameForIR(SizeToLong(i), action.first);
if (graph != nullptr) {
auto graph_clone = BasicClone(graph);
if (graph_clone != nullptr) {
DumpGraphParams dump_params = {false, static_cast<int>(kTopStack)};
if (i == actions_.size()) {
dump_params.dump_mode = static_cast<int>(kWholeStack);
}
(void)mindspore::RDR::RecordAnfGraph(SUBMODULE_ID, name, graph_clone, dump_params, ".ir");
} else {
MS_LOG(WARNING) << "Clone FuncGraph failed in pipeline, no FuncGraph recording in RDR.";
}
} else {
MS_LOG(WARNING) << "Pipeline Resource has no FuncGraph, no FuncGraph recording in RDR";
}
MS_LOG(INFO) << "Recording FuncGraph in pipeline end.";
}
std::string filename = GetBaseNameForIR(SizeToLong(i), action.first);
RDRRecordGraph(i, actions_.size(), filename, graph);
if (MsContext::GetInstance()->get_param<bool>(MS_CTX_SAVE_GRAPHS_FLAG) && graph != nullptr) {
user_graph = graph;

1
mindspore/ccsrc/profiler/device/ascend/parallel_strategy_profiling.cc
View File

@ -39,6 +39,7 @@ bool IsProfilingParallelStrategyEnabled() {
auto ascend_profiler = AscendProfiler::GetInstance();
MS_EXCEPTION_IF_NULL(ascend_profiler);
if (!ascend_profiler->GetProfilingEnableFlag()) {
MS_LOG(INFO) << "Profiling parallel strategy is disabled.";
return false;
}