ssk015
/
mindspore2022
forked from huawei/mindspore2022
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

unified runtime support the execution order running

This commit is contained in:
limingqi107 2022-03-22 11:08:26 +08:00
parent 693af7f260
commit d3e6080a9b
5 changed files with 56 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
mindspore/ccsrc/runtime/graph_scheduler/actor/actor_common.h
View File

@ -23,6 +23,7 @@
#include <utility>
#include <thread>
#include <algorithm>
#include <map>
#include "utils/hash_map.h"
#include "mindrt/include/actor/op_actor.h"
#include "runtime/device/device_address.h"
@ -47,8 +48,14 @@ constexpr int kSuccess = 0;
constexpr int kFailure = 1;
enum class GraphExecutionStrategy {
kPipeline, // The actor running is triggered only by data.
kStep // The actor running need be triggered by control in addition.
kPipeline, // The actor running is triggered only by data.
kStep, // The actor running need be triggered by control in addition.
kPipelineWithExecutionOrder // The actor running is triggered by data with the persistent execution order.
};
static const std::map<GraphExecutionStrategy, std::string> kGraphExecutionStrategyStr = {
{GraphExecutionStrategy::kPipeline, "pipeline"},
{GraphExecutionStrategy::kStep, "step"},
{GraphExecutionStrategy::kPipelineWithExecutionOrder, "pipeline_with_execution_order"},
};
const char kDataPrepareActorNameSuffix[] = "_DataPrepareActor";

4
mindspore/ccsrc/runtime/graph_scheduler/control_node_parser.h
View File

@ -25,7 +25,6 @@
#include <map>
#include <stack>
#include <utility>
#include <unordered_map>
#include <algorithm>
#include "utils/hash_map.h"
#include "runtime/hardware/device_context.h"
@ -87,7 +86,8 @@ using FrontNodeToKernelGraph = mindspore::HashMap<AnfNodePtr, KernelGraphPtr>;
using FuncGraphCallRelation = mindspore::HashMap<FuncGraphPtr, std::vector<std::set<FuncGraphPtr>>>;
using CallNodeToFuncGraph = mindspore::HashMap<AnfNodePtr, std::set<FuncGraphPtr>>;
using KernelGraphToDeviceContext = mindspore::HashMap<KernelGraphPtr, DeviceContext *>;
using GroupNameToCommuNodes = std::unordered_map<std::string, std::vector<CNodePtr>>;
using GroupNameToCommuNodes =
mindspore::HashMap<std::string, std::pair<std::vector<CNodePtr>, std::vector<KernelGraphPtr>>>;
// In the control flow, heterogeneous kernel graphs need to be reconnected in the same group, and the kernel graph
// group info is used to store the inputs and outputs of the group.
// Need stack indicates whether a stack actor needs to be created for the group.

2
mindspore/ccsrc/runtime/graph_scheduler/graph_compiler.h
View File

@ -88,7 +88,7 @@ struct BACKEND_EXPORT GraphCompilerInfo {
size_t outputs_num_;
std::string name_;
bool need_erase_;
GraphExecutionStrategy strategy_;
mutable GraphExecutionStrategy strategy_;
};
class GraphCompiler {

54
mindspore/ccsrc/runtime/graph_scheduler/graph_scheduler.cc
View File

@ -492,11 +492,13 @@ ActorSet *GraphScheduler::Transform(const GraphCompilerInfo &graph_compiler_info
}
scheduler_->graph_output_to_actor_.clear();
scheduler_->copy_actors_.clear();
scheduler_->execution_order_running_ = false;
}
};
// cppcheck-suppress unreadVariable
ScopeCleaner cleaner(this);
MS_LOG(INFO) << "Graph(" << graph_compiler_info.name_ << ") transforms actor begin.";
MS_LOG(INFO) << "Graph(" << graph_compiler_info.name_
<< ") transforms actor begin, strategy:" << kGraphExecutionStrategyStr.at(graph_compiler_info.strategy_);
if (graph_compiler_info.graphs_.size() == 0) {
MS_LOG(EXCEPTION) << "The number of graphs is zero.";
}
@ -504,6 +506,10 @@ ActorSet *GraphScheduler::Transform(const GraphCompilerInfo &graph_compiler_info
MS_LOG(EXCEPTION) << "The number of graphs is not equal to the number of device contexts.";
}
if (graph_compiler_info.strategy_ == GraphExecutionStrategy::kPipelineWithExecutionOrder) {
execution_order_running_ = true;
graph_compiler_info.strategy_ = GraphExecutionStrategy::kPipeline;
}
PersistDeviceTensor(graph_compiler_info);
const auto &actor_set = Build(graph_compiler_info);
MS_EXCEPTION_IF_NULL(actor_set);
@ -761,8 +767,10 @@ void GraphScheduler::Link(ActorSet *actor_set, const GraphCompilerInfo &graph_co
std::vector<CNodePtr> communication_nodes;
const auto &group_name = (parser->IsInited() ? parser->FetchGroupNameByKernelGraph(graph) : default_group_name);
LinkDataArrowInNonSinkMode(graph, graph_compiler_info, &auto_monad_actors, &communication_nodes);
group_name_to_communication_nodes[group_name].insert(group_name_to_communication_nodes[group_name].end(),
communication_nodes.begin(), communication_nodes.end());
group_name_to_communication_nodes[group_name].first.insert(
group_name_to_communication_nodes[group_name].first.end(), communication_nodes.begin(),
communication_nodes.end());
(void)group_name_to_communication_nodes[group_name].second.emplace_back(graph);
}
}
@ -1645,9 +1653,16 @@ void GraphScheduler::LinkGlobalControlArrow(ActorSet *const actor_set,
const std::vector<AbstractActor *> &auto_monad_actors,
const GraphCompilerInfo &graph_compiler_info) {
MS_EXCEPTION_IF_NULL(actor_set);
// Link the control arrow by the execution order.
if (execution_order_running_) {
for (auto &graph : graph_compiler_info.graphs_) {
LinkControlArrowByExecutionOrder(graph);
}
}
for (const auto &communication_nodes : communication_node_groups) {
// Link the control arrows by the communication nodes to ensure communication nodes running order.
LinkControlArrowByCommunicationNode(communication_nodes.second, graph_compiler_info);
LinkControlArrowByCommunicationNode(communication_nodes.second.first, communication_nodes.second.second);
}
// Auto monad actor may modify the device tensor store.
@ -1775,8 +1790,20 @@ void GraphScheduler::LinkControlArrowForCustomActor(ActorSet *const actor_set,
}
}
void GraphScheduler::LinkControlArrowByExecutionOrder(const KernelGraphPtr &graph) {
MS_EXCEPTION_IF_NULL(graph);
auto &execution_order = graph->execution_order();
for (size_t i = 1; i < execution_order.size(); ++i) {
auto from_actor = FetchActor(execution_order[i - 1]->fullname_with_scope());
auto to_actor = FetchActor(execution_order[i]->fullname_with_scope());
if ((from_actor != nullptr) && (to_actor != nullptr)) {
AddControlArrow(from_actor, to_actor);
}
}
}
void GraphScheduler::LinkControlArrowByCommunicationNode(const std::vector<CNodePtr> &communication_nodes,
const GraphCompilerInfo &graph_compiler_info) {
const std::vector<KernelGraphPtr> &graphs) {
const size_t kCommunicationNodesMinNum = 2;
if (communication_nodes.size() < kCommunicationNodesMinNum) {
return;
@ -1793,15 +1820,9 @@ void GraphScheduler::LinkControlArrowByCommunicationNode(const std::vector<CNode
// Ensure all actors execute orderly to optimize the execution performance in the multi device scenario currently.
// Using the multi stream to optimize the performance in the future.
for (auto &graph : graph_compiler_info.graphs_) {
MS_EXCEPTION_IF_NULL(graph);
auto &execution_order = graph->execution_order();
for (size_t i = 1; i < execution_order.size(); ++i) {
auto from_actor = FetchActor(execution_order[i - 1]->fullname_with_scope());
auto to_actor = FetchActor(execution_order[i]->fullname_with_scope());
if ((from_actor != nullptr) && (to_actor != nullptr)) {
AddControlArrow(from_actor, to_actor);
}
if (!execution_order_running_) {
for (auto &graph : graphs) {
LinkControlArrowByExecutionOrder(graph);
}
}
}
@ -2279,7 +2300,10 @@ void GraphScheduler::DumpActor(const ActorSet *actor_set, const GraphCompilerInf
auto first_graph_id = kernel_graphs.front()->graph_id();
MS_EXCEPTION_IF_NULL(kernel_graphs.back());
auto last_graph_id = kernel_graphs.back()->graph_id();
std::string strategy = (graph_compiler_info.strategy_ == GraphExecutionStrategy::kPipeline) ? "pipeline" : "step";
std::string strategy = kGraphExecutionStrategyStr.at(graph_compiler_info.strategy_);
if (execution_order_running_) {
strategy = "pipeline_with_excution_order";
}
std::string save_name = "actor_set_" + strategy + "_kernel_graph_" + std::to_string(first_graph_id);
if (last_graph_id != first_graph_id) {
save_name = save_name + "-" + std::to_string(last_graph_id);

6
mindspore/ccsrc/runtime/graph_scheduler/graph_scheduler.h
View File

@ -166,9 +166,10 @@ class BACKEND_EXPORT GraphScheduler {
const std::vector<AbstractActor *> &auto_monad_actors,
const GraphCompilerInfo &graph_compiler_info);
void LinkControlArrowForCustomActor(ActorSet *const actor_set, const GraphCompilerInfo &graph_compiler_info);
void LinkControlArrowByExecutionOrder(const KernelGraphPtr &graph);
// Link the control arrows by the communication nodes in the kernel graph to ensure communication nodes running order.
void LinkControlArrowByCommunicationNode(const std::vector<CNodePtr> &communication_nodes,
const GraphCompilerInfo &graph_compiler_info);
const std::vector<KernelGraphPtr> &graphs);
void LinkDeviceTensorStoreForAutoMonadActor(const std::vector<AbstractActor *> &auto_monad_actors);
void LinkControlArrowForDataPrepareActor(DataPrepareActor *data_prepare_actor, const ActorSet *actor_set,
const ControlNodeParserPtr &parser);
@ -222,6 +223,9 @@ class BACKEND_EXPORT GraphScheduler {
const AID *recorder_aid_{nullptr};
const AID *debug_aid_{nullptr};
// Whether actor running by the persistent execution order.
bool execution_order_running_{false};
bool init_{false};
};
} // namespace runtime