From 4e4be17648bab02aeb7222bc632b10156ccef790 Mon Sep 17 00:00:00 2001 From: Parastoo Ashtari Date: Tue, 29 Jun 2021 10:29:10 -0400 Subject: [PATCH] SetDebugger for MindRTBackend and clean debug_actor code --- mindspore/ccsrc/debug/CMakeLists.txt | 1 + mindspore/ccsrc/debug/debugger/debugger.cc | 3 - .../ccsrc/debug/debugger/debugger_utils.cc | 159 ++++++++++++++++++ .../ccsrc/debug/debugger/debugger_utils.h | 37 ++++ .../runtime/framework/actor/debug_actor.cc | 137 +-------------- mindspore/ccsrc/vm/backend.cc | 6 +- mindspore/ccsrc/vm/backend.h | 2 +- 7 files changed, 207 insertions(+), 138 deletions(-) create mode 100644 mindspore/ccsrc/debug/debugger/debugger_utils.cc create mode 100644 mindspore/ccsrc/debug/debugger/debugger_utils.h diff --git a/mindspore/ccsrc/debug/CMakeLists.txt b/mindspore/ccsrc/debug/CMakeLists.txt index 86e4c31d2ed..5a18012fcbd 100644 --- a/mindspore/ccsrc/debug/CMakeLists.txt +++ b/mindspore/ccsrc/debug/CMakeLists.txt @@ -42,6 +42,7 @@ if(ENABLE_DEBUGGER) "${CMAKE_CURRENT_SOURCE_DIR}/debugger/proto_exporter.cc" "${CMAKE_CURRENT_SOURCE_DIR}/debugger/tensor_summary.cc" "${CMAKE_CURRENT_SOURCE_DIR}/debug_services.cc" + "${CMAKE_CURRENT_SOURCE_DIR}/debugger/debugger_utils.cc" ) endif() if(NOT CMAKE_SYSTEM_NAME MATCHES "Windows") diff --git a/mindspore/ccsrc/debug/debugger/debugger.cc b/mindspore/ccsrc/debug/debugger/debugger.cc index cc4534d2c8e..c088378900a 100644 --- a/mindspore/ccsrc/debug/debugger/debugger.cc +++ b/mindspore/ccsrc/debug/debugger/debugger.cc @@ -236,9 +236,6 @@ bool Debugger::CheckDebuggerDumpEnabled() const { // see if dump is enabled if (device_target_ == kGPUDevice) { return device::KernelRuntime::DumpDataEnabled(); - } else if (MsContext::GetInstance()->get_param(MS_CTX_ENABLE_MINDRT)) { - auto &dump_json_parser = DumpJsonParser::GetInstance(); - return dump_json_parser.e2e_dump_enabled(); } return false; } diff --git a/mindspore/ccsrc/debug/debugger/debugger_utils.cc b/mindspore/ccsrc/debug/debugger/debugger_utils.cc new file mode 100644 index 00000000000..52f53e4370d --- /dev/null +++ b/mindspore/ccsrc/debug/debugger/debugger_utils.cc @@ -0,0 +1,159 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "debug/debugger/debugger_utils.h" +#include +#include +#include +#include +#include "debug/debugger/debugger.h" +#include "runtime/device/gpu/gpu_device_address.h" +#include "debug/data_dump/dump_json_parser.h" +#include "backend/session/anf_runtime_algorithm.h" +#include "backend/kernel_compiler/kernel.h" + +using mindspore::kernel::AddressPtr; +using mindspore::kernel::KernelLaunchInfo; +using AddressPtrList = std::vector; +using KernelGraph = mindspore::session::KernelGraph; +using AnfAlgo = mindspore::session::AnfRuntimeAlgorithm; + +namespace mindspore { + +static const size_t PARAMETER_OUTPUT_INDEX = 0; + +std::vector CheckRealOutput(const std::string &node_name, const size_t &output_size) { + // define a vector containing real output number + std::vector real_outputs; + // P.BatchNorm is used for training and inference + // can add the filter list for more operators here.... + if (node_name == "BatchNorm") { + MS_LOG(INFO) << "loading node named " << node_name; + real_outputs.insert(real_outputs.end(), {0, 3, 4}); + } else { + // by default, TensorLoader will load all outputs + for (size_t j = 0; j < output_size; ++j) { + real_outputs.push_back(j); + } + } + return real_outputs; +} +void LoadInputs(const CNodePtr &cnode, const KernelLaunchInfo *launch_info_, uint32_t exec_order_) { + // get inputs + auto kernel_inputs = launch_info_->inputs_; + auto input_size = AnfAlgo::GetInputTensorNum(cnode); + for (size_t j = 0; j < input_size; ++j) { + auto input_kernel = cnode->input(j + 1); + std::string input_kernel_name = input_kernel->fullname_with_scope(); + auto addr = kernel_inputs[j]; + auto type = AnfAlgo::GetOutputInferDataType(input_kernel, PARAMETER_OUTPUT_INDEX); + // For example, this happens with the Depend op + if (type == kMetaTypeNone) { + continue; + } +#ifdef ENABLE_GPU + auto format = kOpFormat_DEFAULT; + auto gpu_addr = std::make_unique(addr->addr, addr->size, format, type); + string input_tensor_name = input_kernel_name + ':' + "0"; + ShapeVector int_shapes = trans::GetRuntimePaddingShape(input_kernel, PARAMETER_OUTPUT_INDEX); + auto ret = gpu_addr->LoadMemToHost(input_tensor_name, exec_order_, format, int_shapes, type, 0, true); + if (!ret) { + MS_LOG(ERROR) << "LoadMemToHost:" + << ", tensor_name:" << input_tensor_name << ", host_format:" << format << ".!"; + } +#endif + } +} +void LoadOutputs(const CNodePtr &cnode, const KernelLaunchInfo *launch_info_, uint32_t exec_order_) { + // get outputs + auto kernel_outputs = launch_info_->outputs_; + auto output_size = AnfAlgo::GetOutputTensorNum(cnode); + auto node_name = AnfAlgo::GetCNodeName(cnode); + std::string kernel_name = cnode->fullname_with_scope(); + std::vector real_outputs = CheckRealOutput(node_name, output_size); + + for (int j : real_outputs) { + auto addr = kernel_outputs[j]; + auto type = AnfAlgo::GetOutputInferDataType(cnode, j); + // For example, this happens with the Depend op + if (type == kMetaTypeNone) { + continue; + } +#ifdef ENABLE_GPU + auto format = kOpFormat_DEFAULT; + auto gpu_addr = std::make_unique(addr->addr, addr->size, format, type); + string tensor_name = kernel_name + ':' + std::to_string(j); + ShapeVector int_shapes = trans::GetRuntimePaddingShape(cnode, j); + auto ret = gpu_addr->LoadMemToHost(tensor_name, exec_order_, format, int_shapes, type, j, false); + if (!ret) { + MS_LOG(ERROR) << "LoadMemToHost:" + << ", tensor_name:" << tensor_name << ", host_format:" << format << ".!"; + } +#endif + } +} + +bool CheckReadData(const CNodePtr &cnode) { + auto debugger = Debugger::GetInstance(); + if (!debugger) { + return false; + } + bool read_data = false; + auto &dump_json_parser = DumpJsonParser::GetInstance(); + bool dump_enabled = debugger->DumpDataEnabledIteration(); + std::string kernel_name = cnode->fullname_with_scope(); + if (dump_enabled) { + auto dump_mode = dump_json_parser.dump_mode(); + // dump the node if dump_mode is 0, which means all kernels, or if this kernel is in the kernels list + if ((dump_mode == 0) || ((dump_mode == 1) && dump_json_parser.NeedDump(kernel_name))) { + read_data = true; + } + } else if (debugger->debugger_enabled()) { + read_data = debugger->ReadNodeDataRequired(cnode); + } + return read_data; +} + +void ReadDataAndDump(const CNodePtr &cnode, const KernelLaunchInfo *launch_info_, uint32_t exec_order_) { + auto debugger = Debugger::GetInstance(); + if (!debugger) { + return; + } + auto &dump_json_parser = DumpJsonParser::GetInstance(); + bool dump_enabled = debugger->DumpDataEnabledIteration(); + if (debugger->debugger_enabled() || dump_json_parser.InputNeedDump()) { + LoadInputs(cnode, launch_info_, exec_order_); + } + if (debugger->debugger_enabled() || dump_json_parser.OutputNeedDump()) { + LoadOutputs(cnode, launch_info_, exec_order_); + } + // Dump kernel + if (dump_enabled) { + auto kernel_graph = std::dynamic_pointer_cast(cnode->func_graph()); + MS_EXCEPTION_IF_NULL(kernel_graph); + auto graph_id = kernel_graph->graph_id(); + debugger->DumpSingleNode(cnode, graph_id); + // Clear Dumped data when online debugger is not enabled + if (!debugger->debugger_enabled()) { + debugger->ClearCurrentData(); + } + } + // check if the node is last kernel + bool last_kernel = !AnfAlgo::IsInplaceNode(cnode, "skip"); + debugger->PostExecuteNode(cnode, last_kernel); +} + +} // namespace mindspore diff --git a/mindspore/ccsrc/debug/debugger/debugger_utils.h b/mindspore/ccsrc/debug/debugger/debugger_utils.h new file mode 100644 index 00000000000..54d13b64658 --- /dev/null +++ b/mindspore/ccsrc/debug/debugger/debugger_utils.h @@ -0,0 +1,37 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include "debug/debugger/debugger.h" +#include "backend/kernel_compiler/kernel.h" + +using mindspore::kernel::KernelLaunchInfo; + +namespace mindspore { + +std::vector CheckRealOutput(const std::string &node_name, const size_t &output_size); + +void LoadInputs(const CNodePtr &cnode, const KernelLaunchInfo *launch_info_, uint32_t exec_order_); + +void LoadOutputs(const CNodePtr &cnode, const KernelLaunchInfo *launch_info_, uint32_t exec_order_); + +bool CheckReadData(const CNodePtr &cnode); + +void ReadDataAndDump(const CNodePtr &cnode, const KernelLaunchInfo *launch_info_, uint32_t exec_order_); + +} // namespace mindspore diff --git a/mindspore/ccsrc/runtime/framework/actor/debug_actor.cc b/mindspore/ccsrc/runtime/framework/actor/debug_actor.cc index daa64162195..27c3bba1e81 100644 --- a/mindspore/ccsrc/runtime/framework/actor/debug_actor.cc +++ b/mindspore/ccsrc/runtime/framework/actor/debug_actor.cc @@ -21,138 +21,14 @@ #include "runtime/framework/actor/debug_aware_actor.h" #include "mindrt/include/async/async.h" #include "utils/log_adapter.h" -#ifdef ENABLE_GPU +#ifdef ENABLE_DEBUGGER #include "debug/debugger/debugger.h" -#include "runtime/device/gpu/gpu_device_address.h" - -using mindspore::kernel::AddressPtr; -using AddressPtrList = std::vector; -using KernelGraph = mindspore::session::KernelGraph; +#include "debug/debugger/debugger_utils.h" #endif + namespace mindspore { namespace runtime { -#ifdef ENABLE_GPU -static const size_t PARAMETER_OUTPUT_INDEX = 0; - -std::vector CheckRealOutput(const std::string &node_name, const size_t &output_size) { - // define a vector containing real output number - std::vector real_outputs; - // P.BatchNorm is used for training and inference - // can add the filter list for more operators here.... - if (node_name == "BatchNorm") { - MS_LOG(INFO) << "loading node named " << node_name; - real_outputs.insert(real_outputs.end(), {0, 3, 4}); - } else { - // by default, TensorLoader will load all outputs - for (size_t j = 0; j < output_size; ++j) { - real_outputs.push_back(j); - } - } - return real_outputs; -} -void LoadInputs(const CNodePtr &cnode, const KernelLaunchInfo *launch_info_, uint32_t exec_order_) { - // get inputs - auto kernel_inputs = launch_info_->inputs_; - auto input_size = AnfAlgo::GetInputTensorNum(cnode); - for (size_t j = 0; j < input_size; ++j) { - auto input_kernel = cnode->input(j + 1); - std::string input_kernel_name = input_kernel->fullname_with_scope(); - auto addr = kernel_inputs[j]; - auto type = AnfAlgo::GetOutputInferDataType(input_kernel, PARAMETER_OUTPUT_INDEX); - // For example, this happens with the Depend op - if (type == kMetaTypeNone) { - continue; - } - auto format = kOpFormat_DEFAULT; - auto gpu_addr = std::make_unique(addr->addr, addr->size, format, type); - string input_tensor_name = input_kernel_name + ':' + "0"; - ShapeVector int_shapes = trans::GetRuntimePaddingShape(input_kernel, PARAMETER_OUTPUT_INDEX); - auto ret = gpu_addr->LoadMemToHost(input_tensor_name, exec_order_, format, int_shapes, type, 0, true); - if (!ret) { - MS_LOG(ERROR) << "LoadMemToHost:" - << ", tensor_name:" << input_tensor_name << ", host_format:" << format << ".!"; - } - } -} -void LoadOutputs(const CNodePtr &cnode, const KernelLaunchInfo *launch_info_, uint32_t exec_order_) { - // get outputs - auto kernel_outputs = launch_info_->outputs_; - auto output_size = AnfAlgo::GetOutputTensorNum(cnode); - auto node_name = AnfAlgo::GetCNodeName(cnode); - std::string kernel_name = cnode->fullname_with_scope(); - std::vector real_outputs = CheckRealOutput(node_name, output_size); - - for (int j : real_outputs) { - auto addr = kernel_outputs[j]; - auto type = AnfAlgo::GetOutputInferDataType(cnode, j); - // For example, this happens with the Depend op - if (type == kMetaTypeNone) { - continue; - } - auto format = kOpFormat_DEFAULT; - auto gpu_addr = std::make_unique(addr->addr, addr->size, format, type); - string tensor_name = kernel_name + ':' + std::to_string(j); - ShapeVector int_shapes = trans::GetRuntimePaddingShape(cnode, j); - auto ret = gpu_addr->LoadMemToHost(tensor_name, exec_order_, format, int_shapes, type, j, false); - if (!ret) { - MS_LOG(ERROR) << "LoadMemToHost:" - << ", tensor_name:" << tensor_name << ", host_format:" << format << ".!"; - } - } -} - -bool CheckReadData(const CNodePtr &cnode) { - auto debugger = Debugger::GetInstance(); - if (!debugger) { - return false; - } - bool read_data = false; - auto &dump_json_parser = DumpJsonParser::GetInstance(); - bool dump_enabled = debugger->DumpDataEnabledIteration(); - std::string kernel_name = cnode->fullname_with_scope(); - if (dump_enabled) { - auto dump_mode = dump_json_parser.dump_mode(); - // dump the node if dump_mode is 0, which means all kernels, or if this kernel is in the kernels list - if ((dump_mode == 0) || ((dump_mode == 1) && dump_json_parser.NeedDump(kernel_name))) { - read_data = true; - } - } else if (debugger->debugger_enabled()) { - read_data = debugger->ReadNodeDataRequired(cnode); - } - return read_data; -} - -void ReadDataAndDump(const CNodePtr &cnode, const KernelLaunchInfo *launch_info_, uint32_t exec_order_) { - auto debugger = Debugger::GetInstance(); - if (!debugger) { - return; - } - auto &dump_json_parser = DumpJsonParser::GetInstance(); - bool dump_enabled = debugger->DumpDataEnabledIteration(); - if (debugger->debugger_enabled() || dump_json_parser.InputNeedDump()) { - LoadInputs(cnode, launch_info_, exec_order_); - } - if (debugger->debugger_enabled() || dump_json_parser.OutputNeedDump()) { - LoadOutputs(cnode, launch_info_, exec_order_); - } - // Dump kernel - if (dump_enabled) { - auto kernel_graph = std::dynamic_pointer_cast(cnode->func_graph()); - MS_EXCEPTION_IF_NULL(kernel_graph); - auto graph_id = kernel_graph->graph_id(); - debugger->DumpSingleNode(cnode, graph_id); - // Clear Dumped data when online debugger is not enabled - if (!debugger->debugger_enabled()) { - debugger->ClearCurrentData(); - } - } - // check if the node is last kernel - bool last_kernel = !AnfAlgo::IsInplaceNode(cnode, "skip"); - debugger->PostExecuteNode(cnode, last_kernel); -} -#endif - void DebugActor::Debug(const AnfNodePtr &node, const KernelLaunchInfo *launch_info_, const DeviceContext *device_context, OpContext *op_context, const AID *from_aid) { MS_EXCEPTION_IF_NULL(node); @@ -160,14 +36,12 @@ void DebugActor::Debug(const AnfNodePtr &node, const KernelLaunchInfo *launch_in MS_EXCEPTION_IF_NULL(op_context); MS_EXCEPTION_IF_NULL(from_aid); // todo debug. - MS_LOG(INFO) << "DebugActor is called"; -#ifdef ENABLE_GPU +#ifdef ENABLE_DEBUGGER if (node->isa()) { const auto &cnode = node->cast(); auto debugger = Debugger::GetInstance(); if (debugger) { std::string kernel_name = cnode->fullname_with_scope(); - MS_LOG(INFO) << "kernel_name is " << kernel_name; debugger->SetCurNode(kernel_name); bool read_data = CheckReadData(cnode); if (read_data) { @@ -185,8 +59,7 @@ void DebugActor::DebugOnStepEnd(OpContext *op_context, const AID * MS_EXCEPTION_IF_NULL(op_context); MS_EXCEPTION_IF_NULL(from_aid); // todo debug. - MS_LOG(INFO) << "DebugActor::DebugOnStepEnd is called"; -#ifdef ENABLE_GPU +#ifdef ENABLE_DEBUGGER auto debugger = Debugger::GetInstance(); if (debugger) { debugger->Debugger::UpdateStepNumGPU(); diff --git a/mindspore/ccsrc/vm/backend.cc b/mindspore/ccsrc/vm/backend.cc index 16989ad22a4..2c81f46de63 100644 --- a/mindspore/ccsrc/vm/backend.cc +++ b/mindspore/ccsrc/vm/backend.cc @@ -285,7 +285,9 @@ MindRTBackend::MindRTBackend(const std::string &backend_name, const std::string device::DeviceContextManager::GetInstance().GetOrCreateDeviceContext({device_name, device_id}); device_context->Initialize(); device_id_ = device_context->device_context_key().device_id_; - +#ifdef ENABLE_DEBUGGER + SetDebuggerInit(); +#endif runtime::GraphScheduler::GetInstance().Initialize(); } @@ -688,7 +690,7 @@ void MindRTBackend::ConstructOutputs(const AnfNodePtr &output_node, } #ifdef ENABLE_DEBUGGER -void MindRTBackend::SetDebugger() { +void MindRTBackend::SetDebuggerInit() { auto debugger_ = Debugger::GetInstance(); auto ms_context = MsContext::GetInstance(); MS_EXCEPTION_IF_NULL(ms_context); diff --git a/mindspore/ccsrc/vm/backend.h b/mindspore/ccsrc/vm/backend.h index e0300d52011..18a108a6c67 100644 --- a/mindspore/ccsrc/vm/backend.h +++ b/mindspore/ccsrc/vm/backend.h @@ -120,7 +120,7 @@ class MindRTBackend : public Backend { void RunGraph(const ActorInfo &actor_info, OpRunInfo *op_run_info, const std::vector *tensors_mask, const std::vector *input_tensors, VectorRef *outputs); #ifdef ENABLE_DEBUGGER - void SetDebugger() override; + void SetDebuggerInit(); #endif private: