diff --git a/mindspore/ccsrc/runtime/graph_scheduler/actor/actor_common.h b/mindspore/ccsrc/runtime/graph_scheduler/actor/actor_common.h index fd07973abac..31e31df5796 100644 --- a/mindspore/ccsrc/runtime/graph_scheduler/actor/actor_common.h +++ b/mindspore/ccsrc/runtime/graph_scheduler/actor/actor_common.h @@ -23,6 +23,7 @@ #include #include #include +#include #include "utils/hash_map.h" #include "mindrt/include/actor/op_actor.h" #include "runtime/device/device_address.h" @@ -47,8 +48,14 @@ constexpr int kSuccess = 0; constexpr int kFailure = 1; enum class GraphExecutionStrategy { - kPipeline, // The actor running is triggered only by data. - kStep // The actor running need be triggered by control in addition. + kPipeline, // The actor running is triggered only by data. + kStep, // The actor running need be triggered by control in addition. + kPipelineWithExecutionOrder // The actor running is triggered by data with the persistent execution order. +}; +static const std::map kGraphExecutionStrategyStr = { + {GraphExecutionStrategy::kPipeline, "pipeline"}, + {GraphExecutionStrategy::kStep, "step"}, + {GraphExecutionStrategy::kPipelineWithExecutionOrder, "pipeline_with_execution_order"}, }; const char kDataPrepareActorNameSuffix[] = "_DataPrepareActor"; diff --git a/mindspore/ccsrc/runtime/graph_scheduler/control_node_parser.h b/mindspore/ccsrc/runtime/graph_scheduler/control_node_parser.h index d29d6b79dde..b4ff9dd3130 100644 --- a/mindspore/ccsrc/runtime/graph_scheduler/control_node_parser.h +++ b/mindspore/ccsrc/runtime/graph_scheduler/control_node_parser.h @@ -25,7 +25,6 @@ #include #include #include -#include #include #include "utils/hash_map.h" #include "runtime/hardware/device_context.h" @@ -87,7 +86,8 @@ using FrontNodeToKernelGraph = mindspore::HashMap; using FuncGraphCallRelation = mindspore::HashMap>>; using CallNodeToFuncGraph = mindspore::HashMap>; using KernelGraphToDeviceContext = mindspore::HashMap; -using GroupNameToCommuNodes = std::unordered_map>; +using GroupNameToCommuNodes = + mindspore::HashMap, std::vector>>; // In the control flow, heterogeneous kernel graphs need to be reconnected in the same group, and the kernel graph // group info is used to store the inputs and outputs of the group. // Need stack indicates whether a stack actor needs to be created for the group. diff --git a/mindspore/ccsrc/runtime/graph_scheduler/graph_compiler.h b/mindspore/ccsrc/runtime/graph_scheduler/graph_compiler.h index 3942a412d85..b61ac43033e 100644 --- a/mindspore/ccsrc/runtime/graph_scheduler/graph_compiler.h +++ b/mindspore/ccsrc/runtime/graph_scheduler/graph_compiler.h @@ -88,7 +88,7 @@ struct BACKEND_EXPORT GraphCompilerInfo { size_t outputs_num_; std::string name_; bool need_erase_; - GraphExecutionStrategy strategy_; + mutable GraphExecutionStrategy strategy_; }; class GraphCompiler { diff --git a/mindspore/ccsrc/runtime/graph_scheduler/graph_scheduler.cc b/mindspore/ccsrc/runtime/graph_scheduler/graph_scheduler.cc index aaaa443bbfc..149965e588e 100644 --- a/mindspore/ccsrc/runtime/graph_scheduler/graph_scheduler.cc +++ b/mindspore/ccsrc/runtime/graph_scheduler/graph_scheduler.cc @@ -492,11 +492,13 @@ ActorSet *GraphScheduler::Transform(const GraphCompilerInfo &graph_compiler_info } scheduler_->graph_output_to_actor_.clear(); scheduler_->copy_actors_.clear(); + scheduler_->execution_order_running_ = false; } }; // cppcheck-suppress unreadVariable ScopeCleaner cleaner(this); - MS_LOG(INFO) << "Graph(" << graph_compiler_info.name_ << ") transforms actor begin."; + MS_LOG(INFO) << "Graph(" << graph_compiler_info.name_ + << ") transforms actor begin, strategy:" << kGraphExecutionStrategyStr.at(graph_compiler_info.strategy_); if (graph_compiler_info.graphs_.size() == 0) { MS_LOG(EXCEPTION) << "The number of graphs is zero."; } @@ -504,6 +506,10 @@ ActorSet *GraphScheduler::Transform(const GraphCompilerInfo &graph_compiler_info MS_LOG(EXCEPTION) << "The number of graphs is not equal to the number of device contexts."; } + if (graph_compiler_info.strategy_ == GraphExecutionStrategy::kPipelineWithExecutionOrder) { + execution_order_running_ = true; + graph_compiler_info.strategy_ = GraphExecutionStrategy::kPipeline; + } PersistDeviceTensor(graph_compiler_info); const auto &actor_set = Build(graph_compiler_info); MS_EXCEPTION_IF_NULL(actor_set); @@ -761,8 +767,10 @@ void GraphScheduler::Link(ActorSet *actor_set, const GraphCompilerInfo &graph_co std::vector communication_nodes; const auto &group_name = (parser->IsInited() ? parser->FetchGroupNameByKernelGraph(graph) : default_group_name); LinkDataArrowInNonSinkMode(graph, graph_compiler_info, &auto_monad_actors, &communication_nodes); - group_name_to_communication_nodes[group_name].insert(group_name_to_communication_nodes[group_name].end(), - communication_nodes.begin(), communication_nodes.end()); + group_name_to_communication_nodes[group_name].first.insert( + group_name_to_communication_nodes[group_name].first.end(), communication_nodes.begin(), + communication_nodes.end()); + (void)group_name_to_communication_nodes[group_name].second.emplace_back(graph); } } @@ -1645,9 +1653,16 @@ void GraphScheduler::LinkGlobalControlArrow(ActorSet *const actor_set, const std::vector &auto_monad_actors, const GraphCompilerInfo &graph_compiler_info) { MS_EXCEPTION_IF_NULL(actor_set); + // Link the control arrow by the execution order. + if (execution_order_running_) { + for (auto &graph : graph_compiler_info.graphs_) { + LinkControlArrowByExecutionOrder(graph); + } + } + for (const auto &communication_nodes : communication_node_groups) { // Link the control arrows by the communication nodes to ensure communication nodes running order. - LinkControlArrowByCommunicationNode(communication_nodes.second, graph_compiler_info); + LinkControlArrowByCommunicationNode(communication_nodes.second.first, communication_nodes.second.second); } // Auto monad actor may modify the device tensor store. @@ -1775,8 +1790,20 @@ void GraphScheduler::LinkControlArrowForCustomActor(ActorSet *const actor_set, } } +void GraphScheduler::LinkControlArrowByExecutionOrder(const KernelGraphPtr &graph) { + MS_EXCEPTION_IF_NULL(graph); + auto &execution_order = graph->execution_order(); + for (size_t i = 1; i < execution_order.size(); ++i) { + auto from_actor = FetchActor(execution_order[i - 1]->fullname_with_scope()); + auto to_actor = FetchActor(execution_order[i]->fullname_with_scope()); + if ((from_actor != nullptr) && (to_actor != nullptr)) { + AddControlArrow(from_actor, to_actor); + } + } +} + void GraphScheduler::LinkControlArrowByCommunicationNode(const std::vector &communication_nodes, - const GraphCompilerInfo &graph_compiler_info) { + const std::vector &graphs) { const size_t kCommunicationNodesMinNum = 2; if (communication_nodes.size() < kCommunicationNodesMinNum) { return; @@ -1793,15 +1820,9 @@ void GraphScheduler::LinkControlArrowByCommunicationNode(const std::vectorexecution_order(); - for (size_t i = 1; i < execution_order.size(); ++i) { - auto from_actor = FetchActor(execution_order[i - 1]->fullname_with_scope()); - auto to_actor = FetchActor(execution_order[i]->fullname_with_scope()); - if ((from_actor != nullptr) && (to_actor != nullptr)) { - AddControlArrow(from_actor, to_actor); - } + if (!execution_order_running_) { + for (auto &graph : graphs) { + LinkControlArrowByExecutionOrder(graph); } } } @@ -2279,7 +2300,10 @@ void GraphScheduler::DumpActor(const ActorSet *actor_set, const GraphCompilerInf auto first_graph_id = kernel_graphs.front()->graph_id(); MS_EXCEPTION_IF_NULL(kernel_graphs.back()); auto last_graph_id = kernel_graphs.back()->graph_id(); - std::string strategy = (graph_compiler_info.strategy_ == GraphExecutionStrategy::kPipeline) ? "pipeline" : "step"; + std::string strategy = kGraphExecutionStrategyStr.at(graph_compiler_info.strategy_); + if (execution_order_running_) { + strategy = "pipeline_with_excution_order"; + } std::string save_name = "actor_set_" + strategy + "_kernel_graph_" + std::to_string(first_graph_id); if (last_graph_id != first_graph_id) { save_name = save_name + "-" + std::to_string(last_graph_id); diff --git a/mindspore/ccsrc/runtime/graph_scheduler/graph_scheduler.h b/mindspore/ccsrc/runtime/graph_scheduler/graph_scheduler.h index 930dfa3b355..98f93f60469 100644 --- a/mindspore/ccsrc/runtime/graph_scheduler/graph_scheduler.h +++ b/mindspore/ccsrc/runtime/graph_scheduler/graph_scheduler.h @@ -166,9 +166,10 @@ class BACKEND_EXPORT GraphScheduler { const std::vector &auto_monad_actors, const GraphCompilerInfo &graph_compiler_info); void LinkControlArrowForCustomActor(ActorSet *const actor_set, const GraphCompilerInfo &graph_compiler_info); + void LinkControlArrowByExecutionOrder(const KernelGraphPtr &graph); // Link the control arrows by the communication nodes in the kernel graph to ensure communication nodes running order. void LinkControlArrowByCommunicationNode(const std::vector &communication_nodes, - const GraphCompilerInfo &graph_compiler_info); + const std::vector &graphs); void LinkDeviceTensorStoreForAutoMonadActor(const std::vector &auto_monad_actors); void LinkControlArrowForDataPrepareActor(DataPrepareActor *data_prepare_actor, const ActorSet *actor_set, const ControlNodeParserPtr &parser); @@ -222,6 +223,9 @@ class BACKEND_EXPORT GraphScheduler { const AID *recorder_aid_{nullptr}; const AID *debug_aid_{nullptr}; + // Whether actor running by the persistent execution order. + bool execution_order_running_{false}; + bool init_{false}; }; } // namespace runtime