From 12b4940e8ad3f95bce1b9c766cb1ed60fd99678e Mon Sep 17 00:00:00 2001 From: hwjiaorui Date: Mon, 23 Aug 2021 17:14:52 +0800 Subject: [PATCH] log rectification --- .../tbe_kernel_broadcast_selecter.cc | 8 +- .../tbe_kernel_select/tbe_kernel_select.cc | 2 +- .../device/ascend/ascend_stream_assign.cc | 83 ++++--------------- .../device/ascend/ascend_stream_assign.h | 2 - .../ccsrc/runtime/device/kernel_adjust.cc | 6 +- mindspore/ccsrc/runtime/device/kernel_info.cc | 4 +- .../ccsrc/runtime/device/kernel_runtime.cc | 2 +- .../ccsrc/runtime/device/launch_kernel.cc | 3 +- mindspore/ccsrc/runtime/device/launch_mul.cc | 3 +- .../ccsrc/runtime/device/memory_manager.cc | 3 +- 10 files changed, 33 insertions(+), 83 deletions(-) diff --git a/mindspore/ccsrc/backend/kernel_compiler/tbe/tbe_kernel_select/tbe_kernel_broadcast_selecter.cc b/mindspore/ccsrc/backend/kernel_compiler/tbe/tbe_kernel_select/tbe_kernel_broadcast_selecter.cc index eea46ecd768..ad3e11f3c66 100644 --- a/mindspore/ccsrc/backend/kernel_compiler/tbe/tbe_kernel_select/tbe_kernel_broadcast_selecter.cc +++ b/mindspore/ccsrc/backend/kernel_compiler/tbe/tbe_kernel_select/tbe_kernel_broadcast_selecter.cc @@ -36,8 +36,12 @@ bool TbeKernelBroadCastSelecter::GetShapeInfo(SupportFormat *support_format) { if (AnfAlgo::HasNodeAttr(kAttrDynInputSizes, cnode_ptr_)) { auto dynamic_size_vec = AnfAlgo::GetNodeAttr>(cnode_ptr_, kAttrDynInputSizes); constexpr int64_t DYNAMIC_INPUT_NUM = 2; - if (dynamic_size_vec.empty() || dynamic_size_vec[0] < DYNAMIC_INPUT_NUM) { - MS_LOG(EXCEPTION) << "dynamic attr set error, please check."; + if (dynamic_size_vec.empty()) { + MS_LOG(EXCEPTION) << "Node [" << AnfAlgo::GetCNodeName(cnode_ptr_) << "]'s attr [dyn_input_sizes] is empty."; + } + if (dynamic_size_vec[0] < DYNAMIC_INPUT_NUM) { + MS_LOG(EXCEPTION) << "Node [" << AnfAlgo::GetCNodeName(cnode_ptr_) + << "]'s attr [dyn_input_sizes] value less than " << DYNAMIC_INPUT_NUM; } auto dynamic_input_shape0_ = AnfAlgo::GetPrevNodeOutputInferShape(cnode_ptr_, kInputIndex_0); PadScalarShape(&dynamic_input_shape0_); diff --git a/mindspore/ccsrc/backend/kernel_compiler/tbe/tbe_kernel_select/tbe_kernel_select.cc b/mindspore/ccsrc/backend/kernel_compiler/tbe/tbe_kernel_select/tbe_kernel_select.cc index f3ef5a95733..84ce1b4ca8e 100644 --- a/mindspore/ccsrc/backend/kernel_compiler/tbe/tbe_kernel_select/tbe_kernel_select.cc +++ b/mindspore/ccsrc/backend/kernel_compiler/tbe/tbe_kernel_select/tbe_kernel_select.cc @@ -96,7 +96,7 @@ void TbeKernelSelect::GetCommonPatternKernelInfo(const OpInfo &op_info) { size_t real_output_tensor_num = AnfAlgo::GetOutputTensorNum(cnode_ptr_); const auto outputs_info = op_info.outputs_ptr(); if (inputs_info.empty() && outputs_info.empty()) { - MS_LOG(EXCEPTION) << "op info input & output is null, please check."; + MS_LOG(EXCEPTION) << AnfAlgo::GetCNodeName(cnode_ptr_) << "'s op info input & output is null, please check."; } // create kernel build info from opinfo size_t kernel_build_info_num = diff --git a/mindspore/ccsrc/runtime/device/ascend/ascend_stream_assign.cc b/mindspore/ccsrc/runtime/device/ascend/ascend_stream_assign.cc index a2850cdc33d..e62fb5fb6f5 100644 --- a/mindspore/ccsrc/runtime/device/ascend/ascend_stream_assign.cc +++ b/mindspore/ccsrc/runtime/device/ascend/ascend_stream_assign.cc @@ -1159,63 +1159,6 @@ bool AscendStreamAssign::CheckStreamSwitch(const CNodePtr &switch_ptr) { return true; } -void AscendStreamAssign::UpdateStreamSwitch(const NotNull &graph_ptr, const CNodePtr &switch_ptr, - vector *orders) { - if (!CheckStreamSwitch(switch_ptr)) { - orders->emplace_back(switch_ptr); - return; - } - - auto kind = AnfAlgo::GetNodeAttr(switch_ptr, kAttrStreamSwitchKind); - if (kind == kIndependentStreamSwitch) { - bool independent_empty = independent_stream_map_.empty(); - // if independent empty: delete independent streamswitch - if (!independent_empty) { - for (const auto &item : independent_stream_map_) { - // first independent stream id is minimum and order by std map; - auto first_independent_stream = item.first; - AnfAlgo::SetNodeAttr(kAttrTrueBranchStream, MakeValue(first_independent_stream), switch_ptr); - orders->emplace_back(switch_ptr); - break; - } - } else { - MS_LOG(ERROR) << "Independent stream switch exit, but independent stream is empty"; - } - - // update processed stream - independent_stream_activated_ = true; - for (const auto &item : independent_stream_map_) { - processed_streams_.emplace(item.first); - } - } else if (kind == kFpBpStreamSwitch) { - if (hcom_stream_map_.empty()) { - orders->emplace_back(switch_ptr); - return; - } - if (!AnfAlgo::HasNodeAttr(kAttrTrueBranchStream, switch_ptr)) { - // FpBp StreamSwitch has no true branch attr - orders->emplace_back(switch_ptr); - return; - } - auto true_stream_id = AnfAlgo::GetNodeAttr(switch_ptr, kAttrTrueBranchStream); - MS_LOG(INFO) << "Switch stream id:" << AnfAlgo::GetStreamId(switch_ptr) << "; active stream id:" << true_stream_id; - CNodePtr active_ptr = KernelAdjust::GetInstance().CreateStreamActiveOp(graph_ptr); - AnfAlgo::SetStreamId(true_stream_id, active_ptr.get()); - vector active_ids; - // active hcom stream - for (const auto &item : hcom_stream_map_) { - active_ids.emplace_back(item.first); - } - AnfAlgo::SetNodeAttr(kAttrActiveStreamList, MakeValue>(active_ids), active_ptr); - hcom_stream_activated_ = true; - for (const auto &item : hcom_stream_map_) { - processed_streams_.emplace(item.first); - } - orders->emplace_back(switch_ptr); - orders->emplace_back(active_ptr); - } -} - bool AscendStreamAssign::IsProcessedStream(uint32_t stream_id) { auto it = std::find(processed_streams_.begin(), processed_streams_.end(), stream_id); if (it != processed_streams_.end()) { @@ -1936,16 +1879,18 @@ void AscendStreamAssign::CheckEventAssign(const NotNull &graph_p } uint32_t assigned_event_num = resource_manager.get_cur_event_num(); if ((max_event_id != assigned_event_num - 1) || (event_map.size() != assigned_event_num)) { - MS_LOG(EXCEPTION) << "Event should be consecutive"; + MS_LOG(EXCEPTION) << "Event should be consecutive, however, assigned event num is: " << assigned_event_num + << ", max event id:" << max_event_id << ", event map is:" << event_map; } for (const auto &item : event_map) { if (item.second.size() != 2) { - MS_LOG(EXCEPTION) << "Send/recv should be in pair and share one event id"; + MS_LOG(EXCEPTION) << "Send/recv should be in pair and share one event id, invalid event id is:" << item.first + << ", event size is:" << item.second.size(); } auto first_name = AnfAlgo::GetCNodeName(item.second[0]); auto second_name = AnfAlgo::GetCNodeName(item.second[1]); if (!(first_name == kSendOpName && second_name == kRecvOpName)) { - MS_LOG(EXCEPTION) << "Send should be before recv"; + MS_LOG(EXCEPTION) << "Send should be before recv, invalid event id is:" << item.first; } } } @@ -2220,20 +2165,20 @@ void AscendStreamAssign::GetStreamActiveStreamRelation(const NotNull>(cur_cnode, kAttrActiveStreamList); if (kind == kHead) { - uint32_t active_current_node = GetStreamByActivedStream(cur_stream_id); - if (active_current_node == kInvalidStreamId) { - MS_LOG(EXCEPTION) << "No stream to active streamactive stream"; + uint32_t active_current_stream_id = GetStreamByActivedStream(cur_stream_id); + if (active_current_stream_id == kInvalidStreamId) { + MS_LOG(EXCEPTION) << "No stream to active streamactive stream: " << cur_stream_id; } for (const auto &item : active_list) { - if (item <= active_current_node) { + if (item <= active_current_stream_id) { MS_LOG(WARNING) << "Activated stream is less than activing stream"; continue; } - auto it = - std::find(stream_relations_[active_current_node].begin(), stream_relations_[active_current_node].end(), item); - if (it == stream_relations_[active_current_node].end()) { - stream_relations_[active_current_node].emplace_back(item); + auto it = std::find(stream_relations_[active_current_stream_id].begin(), + stream_relations_[active_current_stream_id].end(), item); + if (it == stream_relations_[active_current_stream_id].end()) { + stream_relations_[active_current_stream_id].emplace_back(item); } } } @@ -2277,7 +2222,7 @@ StreamActiveKind AscendStreamAssign::GetStreamActiveKind(const NotNull &independent_streams); void ActiveOtherGraphParallel(const NotNull &graph_ptr, std::map> other_graph); - void UpdateStreamSwitch(const NotNull &graph_ptr, const CNodePtr &switch_ptr, - vector *orders); bool CheckStreamSwitch(const CNodePtr &switch_ptr); void InsertEventForIndependentParallel(const NotNull &graph_ptr); void InsertCtrlForIndependentParallel(const NotNull &graph_ptr); diff --git a/mindspore/ccsrc/runtime/device/kernel_adjust.cc b/mindspore/ccsrc/runtime/device/kernel_adjust.cc index 851aefbbe72..7e3184a9a24 100644 --- a/mindspore/ccsrc/runtime/device/kernel_adjust.cc +++ b/mindspore/ccsrc/runtime/device/kernel_adjust.cc @@ -425,7 +425,7 @@ void KernelAdjust::InsertSwitchLoop(const std::shared_ptr const std::vector &orders = kernel_graph_ptr->execution_order(); if (orders.empty()) { - MS_LOG(EXCEPTION) << "graph execution order is empty"; + MS_LOG(EXCEPTION) << "graph " << kernel_graph_ptr->graph_id() << " execution order is empty"; } std::vector exec_order; @@ -571,7 +571,7 @@ CNodePtr KernelAdjust::CreateStreamSwitchOp(const std::shared_ptr new_cnode_list; std::vector cnode_ptr_list = kernel_graph_ptr->execution_order(); if (cnode_ptr_list.empty()) { - MS_LOG(ERROR) << "No CNode in graph"; + MS_LOG(ERROR) << "No CNode in graph " << kernel_graph_ptr->graph_id(); return; } for (const auto &cnode_ptr : cnode_ptr_list) { diff --git a/mindspore/ccsrc/runtime/device/kernel_info.cc b/mindspore/ccsrc/runtime/device/kernel_info.cc index ae47a00233e..9d8ae77f06a 100644 --- a/mindspore/ccsrc/runtime/device/kernel_info.cc +++ b/mindspore/ccsrc/runtime/device/kernel_info.cc @@ -24,7 +24,7 @@ kernel::KernelBuildInfoPtr KernelInfo::GetMutableSelectKernelBuildInfo() const { const DeviceAddress *KernelInfo::GetOutputAddr(size_t index) const { if (index >= output_address_list_.size()) { - MS_LOG(ERROR) << "Index [" << index << "] out of range"; + MS_LOG(ERROR) << "Index [" << index << "] out of range 0~" << output_address_list_.size() - 1; return nullptr; } return output_address_list_[index].get(); @@ -101,7 +101,7 @@ bool KernelInfo::SetWorkspaceAddr(const DeviceAddressPtr &output_address, size_t } } if (index >= workspace_address_list_.size()) { - MS_LOG(ERROR) << "Index" << index << " out of range"; + MS_LOG(ERROR) << "Index [" << index << "] out of range"; return false; } workspace_address_list_[index] = output_address; diff --git a/mindspore/ccsrc/runtime/device/kernel_runtime.cc b/mindspore/ccsrc/runtime/device/kernel_runtime.cc index 0cf06d56de4..79a3ef475b9 100644 --- a/mindspore/ccsrc/runtime/device/kernel_runtime.cc +++ b/mindspore/ccsrc/runtime/device/kernel_runtime.cc @@ -582,7 +582,7 @@ DeviceAddressPtr KernelRuntime::PreAssignCNodeMemory(const AnfNodePtr &anf_node, MS_EXCEPTION_IF_NULL(kernel_mod); auto output_sizes = kernel_mod->GetOutputSizeList(); if (output_sizes.size() <= index) { - MS_LOG(EXCEPTION) << "Previous node output size < node index"; + MS_LOG(EXCEPTION) << "Previous node output size " << output_sizes.size() << " <= node index " << index; } std::string output_format = AnfAlgo::GetOutputFormat(anf_node, index); auto output_type = AnfAlgo::GetOutputDeviceDataType(anf_node, index); diff --git a/mindspore/ccsrc/runtime/device/launch_kernel.cc b/mindspore/ccsrc/runtime/device/launch_kernel.cc index 74f8e435faf..64d568d65c1 100644 --- a/mindspore/ccsrc/runtime/device/launch_kernel.cc +++ b/mindspore/ccsrc/runtime/device/launch_kernel.cc @@ -44,7 +44,8 @@ std::vector LaunchKernel::ObtainKernelInputs(const std::vect const std::vector &inputs_addr) { std::vector kernel_inputs; if (inputs_list.size() != inputs_addr.size()) { - MS_LOG(ERROR) << "input_list size should equal to input_addr_ size"; + MS_LOG(ERROR) << "input_list size should equal to input_addr_ size, input_list size: " << inputs_list.size() + << ", input_addr_ size: " << inputs_addr.size(); } for (size_t i = 0; i < inputs_list.size(); ++i) { auto input_size = AlignSizeForLaunchKernel(inputs_list[i]); diff --git a/mindspore/ccsrc/runtime/device/launch_mul.cc b/mindspore/ccsrc/runtime/device/launch_mul.cc index ff3b157d1f4..75cddf3ae59 100644 --- a/mindspore/ccsrc/runtime/device/launch_mul.cc +++ b/mindspore/ccsrc/runtime/device/launch_mul.cc @@ -52,7 +52,8 @@ kernel::KernelMod *LaunchMul::ObtainLaunchMulKernelMod() { } // obtain kernel_mod if (mul_graph_->execution_order().size() != 1) { - MS_LOG(ERROR) << "the execution order of the mul graph should have only one node"; + MS_LOG(ERROR) << "the execution order of the mul graph should have only one node, however, it has " + << mul_graph_->execution_order().size() << " nodes."; } return AnfAlgo::GetKernelMod(mul_graph_->execution_order()[0]); } diff --git a/mindspore/ccsrc/runtime/device/memory_manager.cc b/mindspore/ccsrc/runtime/device/memory_manager.cc index f3a93686e3a..4a1ba8f7db8 100644 --- a/mindspore/ccsrc/runtime/device/memory_manager.cc +++ b/mindspore/ccsrc/runtime/device/memory_manager.cc @@ -177,7 +177,8 @@ bool MemoryManager::MallocContinuousMemFromMemPool(const DeviceAddressPtrList ad return false; } if (addr_list.size() != device_ptr_list.size()) { - MS_LOG(EXCEPTION) << "The size of device list is not equal to the size of address list."; + MS_LOG(EXCEPTION) << "The size of device list " << addr_list.size() << " is not equal to the size of address list " + << device_ptr_list.size(); } for (size_t i = 0; i < addr_list.size(); i++) { MS_EXCEPTION_IF_NULL(device_ptr_list[i]);