yankf21
/
mindspore2022
forked from huawei/mindspore2022
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix cannot find backend node.

This commit is contained in:
gaoyong10 2021-07-08 23:28:28 +08:00
parent 607ffdf63a
commit c92d6d5fb5
7 changed files with 136 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
mindspore/ccsrc/runtime/framework/actor/output_actor.cc
View File

@ -25,6 +25,28 @@ TensorPtr CreateOutputTensor(const AnfNodePtr &output_node, size_t output_index,
MS_LOG(INFO) << "Create output tensor, output node: " << output_node->fullname_with_scope()
<< ", output index: " << output_index << ", output position: " << output_position;
// When there is a valuenode in the output, there is no device_address, and the valuenode is sent directly.
if (output_node->kernel_info() == nullptr && output_node->isa<ValueNode>()) {
const auto &node_value = output_node->cast<ValueNodePtr>()->value();
if (node_value->isa<ValueTuple>()) {
auto value_tuple = node_value->cast<ValueTuplePtr>();
MS_EXCEPTION_IF_NULL(value_tuple);
auto tuple_value = value_tuple->value();
if (output_index >= tuple_value.size()) {
MS_LOG(ERROR) << "Invalid index of value tuple node, size:" << tuple_value.size() << " index:" << output_index;
}
if (tuple_value[output_index]->isa<tensor::Tensor>()) {
return tuple_value[output_index]->cast<TensorPtr>();
} else if (tuple_value[output_index]->isa<Int64Imm>()) {
return std::make_shared<tensor::Tensor>(GetValue<int64_t>(tuple_value[output_index]),
tuple_value[output_index]->type());
}
} else if (node_value->isa<tensor::Tensor>()) {
return node_value->cast<TensorPtr>();
}
}
// Create host tensor, the output tensor should use the infer type, it will be handed correctly by tensor data sync
// when infer type is not equal to device type.
auto type_id = AnfAlgo::GetOutputInferDataType(output_node, output_index);

33
mindspore/ccsrc/runtime/framework/actor/switch_actor.cc
View File

@ -99,10 +99,7 @@ void SwitchActor::ParsePartialInput(const AnfNodePtr &node, const size_t branch_
}
auto func_graph = GetValueNode<FuncGraphPtr>(partial_inputs[kPartialFuncGraphPos]);
if (func_graph->output()->isa<ValueNode>()) {
AddInput(func_graph->output(), branch_id);
return;
} else if (AnfAlgo::CheckPrimitiveType(func_graph->output(), prim::kPrimPartial)) {
if (AnfAlgo::CheckPrimitiveType(func_graph->output(), prim::kPrimPartial)) {
// If the funcgraph called by the partial returns a partial node, the switch actor should call the funcgraph
// of the sub partial. Similarly, the input node should also be the input of the sub partial.
is_mulit_call_ = true;
@ -138,6 +135,9 @@ void SwitchActor::ParsePartialInput(const AnfNodePtr &node, const size_t branch_
for (size_t j = kPartialInputStartPos; j < partial_inputs.size(); ++j) {
AddInput(partial_inputs[j], branch_id);
}
} else if (IsValueNode<FuncGraph>(node)) {
const auto func_graph = GetValueNode<FuncGraphPtr>(node);
branch_func_graph_[branch_id] = func_graph;
} else {
AddInput(node, branch_id);
}
@ -308,6 +308,12 @@ void SwitchActor::AddInput(const AnfNodePtr &node, const size_t branch) {
for (size_t i = 0; i < call_output_num; ++i) {
AddInput({real_input.first, i}, branch);
}
} else if (real_input.first->isa<ValueNode>() && real_input.first->cast<ValueNodePtr>()->value()->isa<ValueTuple>()) {
const auto &value = real_input.first->cast<ValueNodePtr>()->value();
const auto &tuple_value = value->cast<ValueTuplePtr>();
for (size_t i = 0; i < tuple_value->value().size(); ++i) {
AddInput({real_input.first, i}, branch);
}
} else {
AddInput(real_input, branch);
}
@ -410,6 +416,11 @@ void SwitchActor::FetchInputDeviceTensor(OpContext<DeviceTensor> *context) {
auto device_tensor =
DeviceTensorStore::GetInstance().Fetch(device_tensor_store_key.second, device_context_->GetDeviceAddressType());
if (device_tensor == nullptr) {
// When the output of the graph is a valuenode, it does not need to be obtained from the store, and the valuenode
// is sent directly.
if (input_nodes_[device_tensor_store_key.first].first->isa<ValueNode>()) {
continue;
}
std::string error_info =
GetAID().Name() + " get device tensor store failed: " + device_tensor_store_key.second->DebugString() +
", device type:" + std::to_string(static_cast<int>(device_context_->GetDeviceAddressType()));
@ -457,6 +468,13 @@ void SwitchActor::SendOutput(OpContext<DeviceTensor> *context) {
size_t from_index = branch_inputs_pos_[index][result_arrow->from_output_index_];
MS_LOG(DEBUG) << "Switch actor:" << GetAID() << " send result addr:" << input_device_tensors_[from_index];
// When result is valuenode, send valuenode directly without device address.
if (input_device_tensors_[from_index] == nullptr && input_nodes_[from_index].first->isa<ValueNode>()) {
Async(result_arrow->to_op_id_, &OutputActor::CollectOutput, input_nodes_[from_index].first,
input_nodes_[from_index].second, result_arrow->to_input_index_, context);
continue;
}
bool is_send = false;
for (const auto &backend_node : backend_parameters_[from_index]) {
for (size_t j = 0; j < AnfAlgo::GetOutputTensorNum(backend_node.first); ++j) {
@ -490,6 +508,13 @@ void SwitchActor::SendOutput(OpContext<DeviceTensor> *context) {
MS_EXCEPTION_IF_NULL(data_arrow);
MS_EXCEPTION_IF_NULL(data);
data->data_ = input_device_tensors_[data_arrow->from_output_index_];
if (data->data_ == nullptr && branch_func_graph_[index] != nullptr &&
(!branch_func_graph_[index]->output()->isa<ValueNode>())) {
MS_LOG(WARNING) << "Switch actor:" + GetAID().Name() +
" input:" + std::to_string(data_arrow->from_output_index_) +
" device address is null, node:" +
AnfAlgo::GetNodeDebugString(input_nodes_[data_arrow->from_output_index_].first);
}
Async(data_arrow->to_op_id_, &OpActor::RunOpData, data.get(), context);
}

44
mindspore/ccsrc/runtime/framework/control_node_parser.cc
View File

@ -69,6 +69,9 @@ void FetchParameterBySwitchNode(const AnfNodePtr &switch_node, FuncGraphToParame
for (size_t i = kSwitchTrueBranchPos; i < kSwitchInputNum; ++i) {
const auto &partial_node = switch_inputs[i];
if (IsValueNode<FuncGraph>(partial_node)) {
continue;
}
const auto &func_graph = GetFuncGraphFromPartial(partial_node);
std::vector<AnfNodePtr> parameters;
const auto &partial_inputs = partial_node->cast<CNodePtr>()->inputs();
@ -516,6 +519,26 @@ FuncGraphPtr FetchFuncGraphInNode(const auto &node) {
}
} // namespace
AnfNodePtr FetchRealOutputByCallNode(const AnfNodePtr &node, std::set<AnfNodePtr> *call_nodes) {
const auto &real_node = AnfAlgo::VisitKernelWithReturnType(node, 0, false, {prim::kPrimTupleGetItem}).first;
if (!IsCallNode(real_node)) {
return real_node;
}
if ((*call_nodes).find(real_node) != (*call_nodes).end()) {
return nullptr;
}
(*call_nodes).insert(real_node);
const auto &func_graphs = FetchFuncGraphbyCallNode(real_node);
for (const auto &func_graph : func_graphs) {
const auto &output = FetchRealOutputByCallNode(func_graph->output(), call_nodes);
if (output != nullptr) {
return output;
}
}
return nullptr;
}
// Return true if the node has Ref abstract.
bool HasAbstractRef(const AnfNodePtr &node) {
if (node == nullptr) {
@ -602,6 +625,8 @@ std::vector<FuncGraphPtr> FetchFuncGraphbyCallNode(const AnfNodePtr &node) {
for (size_t i = kSwitchTrueBranchPos; i < cnode_inputs.size(); ++i) {
if (IsPrimitiveCNode(cnode_inputs[i], prim::kPrimPartial)) {
func_graphs.emplace_back(GetFuncGraphFromPartial(cnode_inputs[i]));
} else if (IsValueNode<FuncGraph>(cnode_inputs[i])) {
func_graphs.emplace_back(GetValueNode<FuncGraphPtr>(cnode_inputs[i]));
}
}
} else if (AnfAlgo::CheckPrimitiveType(cnode, prim::kPrimSwitchLayer) &&
@ -659,6 +684,11 @@ size_t FetchOutputSizebyCallNode(const AnfNodePtr &node, std::vector<AnfNodePtr>
break;
}
total_num += call_output_num;
} else if (inputs[i]->isa<ValueNode>() && inputs[i]->cast<ValueNodePtr>()->value()->isa<ValueTuple>()) {
auto value_tuple = inputs[i]->cast<ValueNodePtr>()->value()->cast<ValueTuplePtr>();
MS_EXCEPTION_IF_NULL(value_tuple);
auto tuple_value = value_tuple->value();
total_num += tuple_value.size();
} else if (!HasAbstractMonad(inputs[i])) {
++total_num;
}
@ -750,6 +780,9 @@ void ControlNodeParser::Parse(const std::vector<AnfNodePtr> &control_nodes, cons
RealToFormalNode formal_to_real_front_parameters;
for (const auto real_to_formal_front_parameter : real_to_formal_front_parameters) {
for (const auto formal_parameter : real_to_formal_front_parameter.second) {
MS_LOG(DEBUG) << "Control node parser front node pair, key:"
<< real_to_formal_front_parameter.first->DebugString()
<< " value:" << formal_parameter->DebugString();
formal_to_real_front_parameters[formal_parameter].emplace_back(real_to_formal_front_parameter.first);
}
}
@ -1110,10 +1143,6 @@ void ControlNodeParser::FetchFuncGraphCallNum(const std::vector<AnfNodePtr> &con
for (const auto &func_graph : func_graphs) {
MS_EXCEPTION_IF_NULL(func_graph);
if (func_graph->output()->isa<ValueNode>()) {
continue;
}
if (func_graph_to_call_num_.find(func_graph) == func_graph_to_call_num_.end()) {
func_graph_to_call_num_[func_graph] = 1;
} else {
@ -1147,13 +1176,6 @@ void ControlNodeParser::CreateBranchIDForFuncGraph(const std::vector<AnfNodePtr>
for (const auto &control_node : control_nodes) {
// Root funcgraph does not need to create a gather actor.
if (AnfAlgo::CheckPrimitiveType(control_node, prim::kPrimReturn)) {
const auto &cnode = control_node->cast<CNodePtr>();
const auto &inputs = cnode->inputs();
// If the output of funcgraph is a value node, no need to create gather actor.
if (inputs[kReturnInputPos]->isa<ValueNode>()) {
continue;
}
auto func_graph = control_node->func_graph();
func_graph_to_branch_id_[func_graph] = branch_id++;
}

3
mindspore/ccsrc/runtime/framework/control_node_parser.h
View File

@ -52,7 +52,8 @@ bool IsCallNode(const AnfNodePtr &node);
// Check if the call node is the input of another call node.
bool IsSubCallNode(const AnfNodePtr &node);
// Recursive interface, find the real output of funcgraph called by call node.
AnfNodePtr FetchRealOutputByCallNode(const AnfNodePtr &node, std::set<AnfNodePtr> *call_nodes);
// Check whether the parameter is a weight. In the control flow, weight is passed to the subgraph, and in the subgraph,
// it is determined whether it is a weight.
bool HasAbstractRef(const AnfNodePtr &node);

58
mindspore/ccsrc/runtime/framework/graph_scheduler.cc
View File

@ -1244,8 +1244,7 @@ std::vector<GatherActorPtr> GraphScheduler::BuildGatherActor(const GraphCompiler
}
// If the output of funcgraph is a value node, no need to create gather actor.
if (inputs[kReturnInputPos]->isa<ValueNode>() ||
AnfAlgo::CheckPrimitiveType(inputs[kReturnInputPos], prim::kPrimPartial)) {
if (AnfAlgo::CheckPrimitiveType(inputs[kReturnInputPos], prim::kPrimPartial)) {
continue;
}
@ -2206,10 +2205,10 @@ void GraphScheduler::LinkArrowByControlNode(const GraphCompilerInfo &graph_compi
MS_EXCEPTION_IF_NULL(actor);
auto gather_actor = dynamic_cast<GatherActor *>(actor);
for (const auto &input_with_index : gather_actor->data_nodes_) {
for (size_t i = 0; i < gather_actor->data_nodes_.size(); ++i) {
const auto &input_with_index = gather_actor->data_nodes_[i];
const auto &from_func_graph = kernel_graph->GetFuncGraph();
LinkDataArrowByControlNode(graph_compiler_info, input_with_index, from_func_graph, gather_actor,
gather_actor->FetchDataNodePosition(input_with_index));
LinkDataArrowByControlNode(graph_compiler_info, input_with_index, from_func_graph, gather_actor, i);
}
}
LinkBranchArrowForSwitchActor(graph_compiler_info, actor_set);
@ -2253,39 +2252,6 @@ void GraphScheduler::LinkDataArrowByCallInput(const KernelWithIndex &call_node_w
// Collect the output of each funcgraph.
for (const auto &func_graph : func_graphs) {
if (func_graph->output()->isa<ValueNode>()) {
if (AnfAlgo::CheckPrimitiveType(switch_node, prim::kPrimSwitch) ||
AnfAlgo::CheckPrimitiveType(switch_node, prim::kPrimSwitchLayer)) {
const auto &actor_name = switch_node->DebugString();
const auto &actor = FetchActor(actor_name);
MS_EXCEPTION_IF_NULL(actor);
auto switch_actor = dynamic_cast<SwitchActor *>(actor);
MS_EXCEPTION_IF_NULL(switch_actor);
const auto &output_with_index = KernelWithIndex(func_graph->output(), 0);
const auto &iter =
find(switch_actor->input_nodes_.begin(), switch_actor->input_nodes_.end(), output_with_index);
if (iter == switch_actor->input_nodes_.end()) {
MS_LOG(EXCEPTION) << "Invalid input node for switch actor:" << switch_actor->GetAID()
<< " node:" << AnfAlgo::GetNodeDebugString(func_graph->output());
}
size_t pos = iter - switch_actor->input_nodes_.begin();
// Add output for each branch of switch.
for (size_t i = 0; i < switch_actor->branch_inputs_pos_.size(); ++i) {
const auto poses = switch_actor->branch_inputs_pos_[i];
if (find(poses.begin(), poses.end(), pos) == poses.end()) {
continue;
}
auto op_arrow = std::make_shared<DataArrow>(pos, to_actor->GetAID(), to_index);
switch_actor->output_branch_arrows_[i].emplace_back(op_arrow);
}
} else {
MS_LOG(EXCEPTION) << "Invalid funcgraph:" << func_graph->ToString();
}
continue;
}
const auto actor_name = func_graph->get_return()->DebugString();
auto actor = FetchActor(actor_name);
MS_EXCEPTION_IF_NULL(actor);
@ -2430,7 +2396,7 @@ void GraphScheduler::LinkDataArrowForSwitchActor(const GraphCompilerInfo &graph_
// Link switch output.
for (size_t i = 0; i < actor->branch_func_graph_.size(); ++i) {
auto func_graph = actor->branch_func_graph_[i];
if (func_graph == nullptr || func_graph->output()->isa<ValueNode>()) {
if (func_graph == nullptr) {
continue;
}
@ -2534,6 +2500,18 @@ void GraphScheduler::LinkControlArrowForSwitchActor(std::vector<SwitchActorPtr>
// If there is no output from the switch actor branch, it means that the subgraph has no input,
// and need to connect a control arrow to the corresponding gather actor.
for (auto &switch_actor : (*switch_actors)) {
if (AnfAlgo::CheckPrimitiveType(switch_actor->node_, prim::kPrimReturn)) {
const auto &func_graph = switch_actor->node_->func_graph();
if (func_graph->output()->isa<ValueNode>()) {
const auto &actor_name = func_graph->ToString();
auto actor = FetchActor(actor_name);
MS_EXCEPTION_IF_NULL(actor);
auto gather_actor = dynamic_cast<GatherActor *>(actor);
gather_actor->output_control_arrows_.emplace_back(switch_actor->GetAID());
switch_actor->input_controls_num_++;
}
}
for (size_t i = 0; i < switch_actor->output_branch_arrows_.size(); ++i) {
const auto &arrows = switch_actor->output_branch_arrows_[i];
if (arrows.empty() && switch_actor->branch_func_graph_[i] != nullptr) {
@ -2590,7 +2568,7 @@ void GraphScheduler::LinkBranchArrowForSwitchActor(const GraphCompilerInfo &grap
auto switch_actor = dynamic_cast<SwitchActor *>(actor);
for (size_t i = 0; i < switch_actor->branch_func_graph_.size(); ++i) {
const auto &func_graph = switch_actor->branch_func_graph_[i];
if (func_graph == nullptr || func_graph->output()->isa<ValueNode>()) {
if (func_graph == nullptr) {
continue;
}

46
mindspore/ccsrc/vm/backend.cc
View File

@ -17,6 +17,7 @@
#include <algorithm>
#include <vector>
#include <set>
#include <map>
#include "vm/transform.h"
@ -835,9 +836,11 @@ void MindRTBackend::RunGraph(const ActorInfo &actor_info, const VectorRef &args,
actor_set->output_actor_->UpdateOutputDeviceAddress();
}
void MindRTBackend::ConstructOutputs(const AnfNodePtr &output_node,
const std::vector<tensor::TensorPtr> &output_tensors, size_t *output_position,
VectorRef *outputs) {
void MindRTBackend::ConstructOutputs(const AnfNodePtr &output, const std::vector<tensor::TensorPtr> &output_tensors,
size_t *output_position, VectorRef *outputs) {
std::set<AnfNodePtr> call_node;
const auto &output_node = runtime::FetchRealOutputByCallNode(output, &call_node);
// The makeTuple node need expand and recurse.
if (AnfAlgo::CheckPrimitiveType(output_node, prim::kPrimMakeTuple)) {
auto make_tuple = output_node->cast<CNodePtr>();
@ -916,21 +919,34 @@ std::unique_ptr<GraphCompilerInfo> MindRTBackend::ConstructGraphCompilerInfo(con
size_t outputs_num = 0;
const auto &root_output =
AnfAlgo::VisitKernelWithReturnType(root_graph->output(), 0, false, {prim::kPrimTupleGetItem}).first;
size_t position = 0;
auto outputs = AnfAlgo::GetAllOutputWithIndex(root_output);
if (runtime::IsCallNode(root_output)) {
std::vector<AnfNodePtr> call_nodes;
size_t call_output_num = runtime::FetchOutputSizebyCallNode(root_output, &call_nodes);
for (size_t i = 0; i < call_output_num; ++i) {
outputs.push_back({root_output, i});
std::vector<AnfNodePtr> tuple_outputs;
if (AnfAlgo::CheckPrimitiveType(root_output, prim::kPrimMakeTuple)) {
const auto inputs = root_output->cast<CNodePtr>()->inputs();
for (size_t i = 1; i < inputs.size(); ++i) {
tuple_outputs.emplace_back(inputs[i]);
}
} else {
tuple_outputs.emplace_back(root_output);
}
outputs_num = outputs.size();
for (const auto &output : outputs) {
if (outputs_order.count(output) == 0) {
outputs_order[output] = {position++};
} else {
outputs_order[output].emplace_back(position++);
for (const auto tuple_output : tuple_outputs) {
auto outputs = AnfAlgo::GetAllOutputWithIndex(tuple_output);
if (runtime::IsCallNode(tuple_output)) {
std::vector<AnfNodePtr> call_nodes;
size_t call_output_num = runtime::FetchOutputSizebyCallNode(tuple_output, &call_nodes);
for (size_t i = 0; i < call_output_num; ++i) {
outputs.push_back({tuple_output, i});
}
}
outputs_num += outputs.size();
for (const auto &output : outputs) {
if (outputs_order.count(output) == 0) {
outputs_order[output] = {position++};
} else {
outputs_order[output].emplace_back(position++);
}
}
}

2
mindspore/ccsrc/vm/backend.h
View File

@ -130,7 +130,7 @@ class MindRTBackend : public Backend {
void CompileGraph(const FuncGraphPtr &func_graph);
// Restore the outputs tuple by the origin funcGraph output node and output tensors.
void ConstructOutputs(const AnfNodePtr &output_node, const std::vector<tensor::TensorPtr> &output_tensors,
void ConstructOutputs(const AnfNodePtr &output, const std::vector<tensor::TensorPtr> &output_tensors,
size_t *output_position, VectorRef *outputs);
// Construct the GraphCompilerInfo by the compilation results of graph, used in Graph mode.