diff --git a/.jenkins/check/config/filter_cpplint.txt b/.jenkins/check/config/filter_cpplint.txt index ce7a8bd7e16..3c82ccb356e 100644 --- a/.jenkins/check/config/filter_cpplint.txt +++ b/.jenkins/check/config/filter_cpplint.txt @@ -7,6 +7,8 @@ "mindspore/mindspore/core/abstract/utils.cc" "build/include_what_you_use" "mindspore/mindspore/ccsrc/backend/optimizer/ascend/buffer_fusion/ub_pattern_fusion.cc" "build/include_what_you_use" "mindspore/mindspore/ccsrc/runtime/device/ascend/profiling/profiling_callback_register.cc" "runtime/references" +"mindspore/mindspore/ccsrc/runtime/device/ascend/profiling/profiling_manager.h" "runtime/references" +"mindspore/mindspore/ccsrc/runtime/device/ascend/profiling/profiling_reporter.h" "runtime/references" "mindspore/mindspore/core/mindrt/src/actor/actormgr.h" "runtime/references" "mindspore/mindspore/core/mindrt/src/actor/actorpolicyinterface.h" "runtime/references" "mindspore/mindspore/core/mindrt/src/actor/actorthread.h" "runtime/references" diff --git a/mindspore/ccsrc/runtime/device/CMakeLists.txt b/mindspore/ccsrc/runtime/device/CMakeLists.txt index d0d6168da55..573a882f029 100644 --- a/mindspore/ccsrc/runtime/device/CMakeLists.txt +++ b/mindspore/ccsrc/runtime/device/CMakeLists.txt @@ -90,12 +90,6 @@ if(ENABLE_SECURITY) list(REMOVE_ITEM D_SRC_LIST "ascend/profiling/profiling_callback_register.cc") list(REMOVE_ITEM D_SRC_LIST "ascend/profiling/profiling_manager.cc") list(REMOVE_ITEM D_SRC_LIST "ascend/profiling/profiling_utils.cc") - list(REMOVE_ITEM D_SRC_LIST "ascend/profiling/reporter/desc_reporter.cc") - list(REMOVE_ITEM D_SRC_LIST "ascend/profiling/reporter/graph_desc_reporter.cc") - list(REMOVE_ITEM D_SRC_LIST "ascend/profiling/reporter/op_name_task_stream_reporter.cc") - list(REMOVE_ITEM D_SRC_LIST "ascend/profiling/reporter/point_reporter.cc") - list(REMOVE_ITEM D_SRC_LIST "ascend/profiling/reporter/profiling_desc.cc") - list(REMOVE_ITEM D_SRC_LIST "ascend/profiling/reporter/task_desc_reporter.cc") endif() set_property(SOURCE ${DEVICE_SRC_LIST} ${D_SRC_LIST} ${CPU_SRC_LIST} diff --git a/mindspore/ccsrc/runtime/device/ascend/ascend_kernel_runtime.cc b/mindspore/ccsrc/runtime/device/ascend/ascend_kernel_runtime.cc index f99245bcb9f..223b782afe9 100644 --- a/mindspore/ccsrc/runtime/device/ascend/ascend_kernel_runtime.cc +++ b/mindspore/ccsrc/runtime/device/ascend/ascend_kernel_runtime.cc @@ -60,7 +60,6 @@ #endif #include "runtime/device/ascend/executor/hccl_dynamic_kernel.h" #include "utils/config_manager.h" -#include "runtime/device/ascend/profiling/reporter/op_name_task_stream_reporter.h" #include "runtime/hccl_adapter/hccl_adapter.h" #ifdef ENABLE_TDTQUE #include "minddata/dataset/engine/tdt/tdt_handle.h" @@ -241,17 +240,6 @@ void AsyncDataDumpUninit() { } } } - -void AscendKernelRuntime::ReportProfilingData() { - auto context = MsContext::GetInstance(); - MS_EXCEPTION_IF_NULL(context); - if (ProfilingManager::GetInstance().IsProfilingStart() && - context->get_param(MS_CTX_EXECUTION_MODE) == kPynativeMode) { - // Save Profiling Framework data - OpNameTaskStreamReporter reporter(device_id_, "nonsink", stream_id_task_id_op_name_map_); - reporter.ReportData(); - } -} #endif void AscendKernelRuntime::ReleaseDeviceRes() { @@ -269,9 +257,7 @@ void AscendKernelRuntime::ReleaseDeviceRes() { return; } SetCurrentContext(); -#ifndef ENABLE_SECURITY - ReportProfilingData(); -#endif + // release ge runtime ClearGraphModelMap(); diff --git a/mindspore/ccsrc/runtime/device/ascend/ascend_kernel_runtime.h b/mindspore/ccsrc/runtime/device/ascend/ascend_kernel_runtime.h index 17372251312..dc0e3e8c5d8 100644 --- a/mindspore/ccsrc/runtime/device/ascend/ascend_kernel_runtime.h +++ b/mindspore/ccsrc/runtime/device/ascend/ascend_kernel_runtime.h @@ -101,7 +101,6 @@ class AscendKernelRuntime : public KernelRuntime { #ifndef ENABLE_SECURITY void DistributeDebugTask(const session::KernelGraph &graph, const NotNull> &model_handle); void LaunchDataDump(GraphId graph_id); - void ReportProfilingData(); #endif static CNodePtr GetErrorNodeName(uint32_t streamid, uint32_t taskid); static std::string GetDumpPath(); diff --git a/mindspore/ccsrc/runtime/device/ascend/profiling/prof_common.h b/mindspore/ccsrc/runtime/device/ascend/profiling/prof_common.h new file mode 100644 index 00000000000..69a295b997a --- /dev/null +++ b/mindspore/ccsrc/runtime/device/ascend/profiling/prof_common.h @@ -0,0 +1,179 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSPROFILER_PROF_COMMON_H_ +#define MSPROFILER_PROF_COMMON_H_ + +#include + +#define MSPROF_DATA_HEAD_MAGIC_NUM 0x5a5a +#define MSPROF_DATA_HASH_MIN_LEN_128 128 +#define MSPROF_GE_MODELLOAD_DATA_BYTES 104 +#define MSPROF_DATA_HASH_MIN_LEN_64 64 + +#define MSPROF_MIX_DATA_RESERVE_BYTES 7 +#define MSPROF_MIX_DATA_STRING_LEN 120 + +enum MsprofMixDataType { + MSPROF_MIX_DATA_HASH_ID = 0, + MSPROF_MIX_DATA_STRING = 1, +}; + +enum MsprofDataTag { + MSPROF_GE_DATA_TAG_MODEL_LOAD = 20, + MSPROF_GE_DATA_TAG_FUSION = 21, + MSPROF_GE_DATA_TAG_INFER = 22, + MSPROF_GE_DATA_TAG_TASK = 23, + MSPROF_GE_DATA_TAG_TENSOR = 24, + MSPROF_GE_DATA_TAG_STEP = 25, + MSPROF_GE_DATA_TAG_ID_MAP = 26, +}; + +struct MsprofMixData { + uint8_t type; + uint8_t rsv[MSPROF_MIX_DATA_RESERVE_BYTES]; + union { + uint64_t hashId; + char dataStr[MSPROF_MIX_DATA_STRING_LEN]; + } data; +}; +using MixData = struct MsprofMixData; + +struct MsprofGeProfModelLoadData { + uint16_t magicNumber = MSPROF_DATA_HEAD_MAGIC_NUM; + uint16_t dataTag = MSPROF_GE_DATA_TAG_MODEL_LOAD; + uint32_t modelId; + MixData modelName; + uint64_t startTime; + uint64_t endTime; + uint8_t reserve[MSPROF_GE_MODELLOAD_DATA_BYTES]; +}; + +#define MSPROF_FUSION_DATA_RESERVE_BYTES 14 +#define MSPROF_GE_FUSION_OP_NUM 8 +struct MsprofGeProfFusionData { + uint16_t magicNumber = MSPROF_DATA_HEAD_MAGIC_NUM; + uint16_t dataTag = MSPROF_GE_DATA_TAG_FUSION; + uint32_t modelId; + MixData fusionName; + uint64_t inputMemSize; + uint64_t outputMemSize; + uint64_t weightMemSize; + uint64_t workspaceMemSize; + uint64_t totalMemSize; + uint16_t fusionOpNum; + uint64_t fusionOp[MSPROF_GE_FUSION_OP_NUM]; + uint8_t reserve[MSPROF_FUSION_DATA_RESERVE_BYTES]; +}; + +#define MSPROF_GE_INFER_DATA_RESERVE_BYTES 64 +struct MsprofGeProfInferData { + uint16_t magicNumber; + uint16_t dataTag; + uint32_t modelId; + MixData modelName; + uint32_t requestId; + uint32_t threadId; + uint64_t inputDataStartTime; + uint64_t inputDataEndTime; + uint64_t inferStartTime; + uint64_t inferEndTime; + uint64_t outputDataStartTime; + uint64_t outputDataEndTime; + uint8_t reserve[MSPROF_GE_INFER_DATA_RESERVE_BYTES]; +}; + +#define MSPROF_GE_TASK_DATA_RESERVE_BYTES 16 +#define MSPROF_GE_OP_TYPE_LEN 56 +enum MsprofGeTaskType { MSPROF_GE_TASK_TYPE_AI_CORE = 0, MSPROF_GE_TASK_TYPE_AI_CPU, MSPROF_GE_TASK_TYPE_AIV }; +enum MsprofGeShapeType { MSPROF_GE_SHAPE_TYPE_STATIC = 0, MSPROF_GE_SHAPE_TYPE_DYNAMIC }; + +struct MsprofGeOpType { + uint8_t type; + uint8_t rsv[MSPROF_MIX_DATA_RESERVE_BYTES]; + union { + uint64_t hashId; + char dataStr[MSPROF_GE_OP_TYPE_LEN]; + } data; +}; +using GeOpType = struct MsprofGeOpType; + +struct MsprofGeProfTaskData { + uint16_t magicNumber = MSPROF_DATA_HEAD_MAGIC_NUM; + uint16_t dataTag = MSPROF_GE_DATA_TAG_TASK; + uint32_t taskType; + MixData opName; + GeOpType opType; + uint64_t curIterNum; + uint64_t timeStamp; + uint32_t shapeType; + uint32_t blockDims; + uint32_t modelId; + uint32_t streamId; + uint32_t taskId; + uint32_t threadId; + uint8_t reserve[MSPROF_GE_TASK_DATA_RESERVE_BYTES]; +}; + +#define MSPROF_GE_TENSOR_DATA_RESERVE_BYTES 4 +#define MSPROF_GE_TENSOR_DATA_SHAPE_LEN 8 +#define MSPROF_GE_TENSOR_DATA_NUM 5 +enum MsprofGeTensorType { MSPROF_GE_TENSOR_TYPE_INPUT = 0, MSPROF_GE_TENSOR_TYPE_OUTPUT }; +struct MsprofGeTensorData { + uint32_t tensorType; + uint32_t format; + uint32_t dataType; + uint32_t shape[MSPROF_GE_TENSOR_DATA_SHAPE_LEN]; +}; +using GeTensorData = struct MsprofGeTensorData; + +struct MsprofGeProfTensorData { + uint16_t magicNumber = MSPROF_DATA_HEAD_MAGIC_NUM; + uint16_t dataTag = MSPROF_GE_DATA_TAG_TENSOR; + uint32_t modelId; + uint64_t curIterNum; + uint32_t streamId; + uint32_t taskId; + uint8_t tensorNum; + GeTensorData tensorData[MSPROF_GE_TENSOR_DATA_NUM]; + uint8_t reserve[MSPROF_GE_TENSOR_DATA_RESERVE_BYTES]; +}; + +#define MSPROF_GE_STEP_DATA_RESERVE_BYTES 27 +struct MsprofGeProfStepData { + uint16_t magicNumber = MSPROF_DATA_HEAD_MAGIC_NUM; + uint16_t dataTag = MSPROF_GE_DATA_TAG_STEP; + uint32_t modelId; + uint32_t streamId; + uint32_t taskId; + uint64_t timeStamp; + uint64_t curIterNum; + uint32_t threadId; + uint8_t tag; + uint8_t reserve[MSPROF_GE_STEP_DATA_RESERVE_BYTES]; +}; + +#define MSRPOF_GE_ID_MAP_DATA_RESERVE_BYTES 6 +struct MsprofGeProfIdMapData { + uint16_t magicNumber = MSPROF_DATA_HEAD_MAGIC_NUM; + uint16_t dataTag = MSPROF_GE_DATA_TAG_ID_MAP; + uint32_t graphId; + uint32_t modelId; + uint32_t sessionId; + uint64_t timeStamp; + uint16_t mode; + uint8_t reserve[MSRPOF_GE_ID_MAP_DATA_RESERVE_BYTES]; +}; +#endif diff --git a/mindspore/ccsrc/runtime/device/ascend/profiling/profiling_manager.cc b/mindspore/ccsrc/runtime/device/ascend/profiling/profiling_manager.cc index 9201aa1b501..8f6e73d43cc 100644 --- a/mindspore/ccsrc/runtime/device/ascend/profiling/profiling_manager.cc +++ b/mindspore/ccsrc/runtime/device/ascend/profiling/profiling_manager.cc @@ -241,6 +241,24 @@ Status ProfCtrlSwitchHandle(void *data) { } Status ProfCommandHandle(ProfCommandHandleType type) { return ProfilingManager::GetInstance().ProfCommandHandle(type); } + +void ProfilingManager::QueryHashId(const int32_t &device_id, const std::string &src_str, uint64_t &hash_id) { + // when some profiling data size exceeds the specified size, query its hashId instead. + MsprofHashData hash_data{}; + hash_data.deviceId = device_id; + hash_data.dataLen = src_str.size(); + hash_data.data = reinterpret_cast(const_cast(src_str.c_str())); + + const int32_t ret = prof_cb_.msprofReporterCallback( + static_cast(MsprofReporterModuleId::MSPROF_MODULE_FRAMEWORK), + static_cast(MsprofReporterCallbackType::MSPROF_REPORTER_HASH), &hash_data, sizeof(MsprofHashData)); + if (ret != 0) { + MS_LOG(EXCEPTION) << "[Profiling] Query hash id of long string failed, src string is " << src_str.c_str(); + } + + hash_id = hash_data.hashId; +} + } // namespace ascend } // namespace device } // namespace mindspore diff --git a/mindspore/ccsrc/runtime/device/ascend/profiling/profiling_manager.h b/mindspore/ccsrc/runtime/device/ascend/profiling/profiling_manager.h index caa2e683bea..0c937881d68 100644 --- a/mindspore/ccsrc/runtime/device/ascend/profiling/profiling_manager.h +++ b/mindspore/ccsrc/runtime/device/ascend/profiling/profiling_manager.h @@ -64,6 +64,7 @@ class ProfilingManager { Status PluginInit() const; void PluginUnInit() const; Status CallMsprofReport(NotNull reporter_data) const; + void QueryHashId(const int32_t &device_id, const std::string &src_str, uint64_t &hash_id); const struct MsprofCallback &GetMsprofCallback() { return prof_cb_; } void SetMsprofCtrlCallback(MsprofCtrlCallback func) { prof_cb_.msprofCtrlCallback = func; } void SetMsprofReporterCallback(MsprofReporterCallback func) { prof_cb_.msprofReporterCallback = func; } diff --git a/mindspore/ccsrc/runtime/device/ascend/profiling/profiling_reporter.cc b/mindspore/ccsrc/runtime/device/ascend/profiling/profiling_reporter.cc new file mode 100644 index 00000000000..1a16a3e0e8b --- /dev/null +++ b/mindspore/ccsrc/runtime/device/ascend/profiling/profiling_reporter.cc @@ -0,0 +1,256 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include "runtime/device/ascend/profiling/profiling_reporter.h" +#include "backend/kernel_compiler/kernel.h" +#include "backend/kernel_compiler/ascend_kernel_mod.h" +#include "utils/utils.h" + +namespace mindspore { +namespace device { +namespace ascend { +static std::map KernelType2TaskTypeEnum{{TBE_KERNEL, MSPROF_GE_TASK_TYPE_AI_CORE}, + {AKG_KERNEL, MSPROF_GE_TASK_TYPE_AI_CORE}, + {AICPU_KERNEL, MSPROF_GE_TASK_TYPE_AI_CPU}}; + +// 0 means unknown format +static std::map OpFormat2Index{{kOpFormat_DEFAULT, 1}, + {kOpFormat_NC1KHKWHWC0, 2}, + {kOpFormat_ND, 3}, + {kOpFormat_NCHW, 4}, + {kOpFormat_NHWC, 5}, + {kOpFormat_HWCN, 6}, + {kOpFormat_NC1HWC0, 7}, + {kOpFormat_FRAC_Z, 8}, + {kOpFormat_C1HWNCoC0, 9}, + {kOpFormat_FRAC_NZ, 10}, + {kOpFormat_NC1HWC0_C04, 11}, + {kOpFormat_FRACTAL_Z_C04, 12}, + {kOpFormat_NDHWC, 13}, + {kOpFormat_FRACTAL_ZN_LSTM, 14}, + {kOpFormat_FRACTAL_ZN_RNN, 15}, + {kOpFormat_ND_RNN_BIAS, 16}, + {kOpFormat_NDC1HWC0, 17}, + {kOpFormat_NCDHW, 18}, + {kOpFormat_FRACTAL_Z_3D, 19}, + {kOpFormat_DHWNC, 20}, + {kOpFormat_DHWCN, 21}}; + +bool ProfilingReporter::CheckStreamTaskValid() { + if (cnode_list_.size() != stream_ids_.size() || cnode_list_.size() != task_ids_.size()) { + MS_LOG(ERROR) << "CNode size is not equal stream size or not equal task size, " + "can not support to report profiling data. CNode size is " + << cnode_list_.size() << ", stream size is " << stream_ids_.size() << ", task size is " + << task_ids_.size(); + return false; + } + return true; +} + +void ProfilingReporter::ReportTasks() { + MS_LOG(INFO) << "Profiling start to report tasks."; + if (!CheckStreamTaskValid()) { + return; + } + size_t task_index = 0; + for (const auto &node : cnode_list_) { + MS_EXCEPTION_IF_NULL(node); + KernelType kernel_type = AnfAlgo::GetKernelType(node); + // Note: some kernel stream id or task id will conflict, such as RT_KERNEL, + // and CPU_KERNEL does not have stream id, task id. + if (kernel_type != TBE_KERNEL && kernel_type != AKG_KERNEL && kernel_type != AICPU_KERNEL && + kernel_type != HCCL_KERNEL) { + MS_LOG(INFO) << "This node is not TBE_KERNEL, AKG_KERNEL, AICPU_KERNEL, HCCL_KERNEL, will skip, node name:" + << node->fullname_with_scope(); + ++task_index; + continue; + } + auto stream_id = stream_ids_[task_index]; + auto task_id = task_ids_[task_index]; + (void)ReportTask(node, stream_id, task_id, kernel_type); + (void)ReportNode(node, stream_id, task_id, MSPROF_GE_TENSOR_TYPE_INPUT); + (void)ReportNode(node, stream_id, task_id, MSPROF_GE_TENSOR_TYPE_OUTPUT); + + ++task_index; + } + MS_LOG(INFO) << "Profiling report task data finish."; +} + +void ProfilingReporter::ReportStepPoint(const std::vector> &points) { + MS_LOG(INFO) << "Profiling start to report step point data."; + if (!CheckStreamTaskValid()) { + return; + } + + ConstructNodeNameIndexMap(); + for (const auto &point : points) { + MsprofGeProfStepData step_point{}; + step_point.modelId = graph_id_; + auto op_name = point->op_name(); + step_point.streamId = GetStreamId(op_name); + step_point.taskId = GetTaskId(op_name); + step_point.timeStamp = 0; + step_point.curIterNum = 0; + step_point.threadId = 0; + step_point.tag = point->tag(); + (void)ReportData(device_id_, reinterpret_cast(&step_point), sizeof(step_point), "step_info"); + MS_LOG(WARNING) << "ReportStepPoint, graph_id: " << graph_id_ << ", op_name: " << point->op_name() + << ", streamId: " << GetStreamId(op_name) << ", task Id: " << GetTaskId(op_name); + } +} + +uint32_t ProfilingReporter::GetStreamId(const string &node_name) { + auto index = node_name_index_map_[node_name]; + return stream_ids_[index]; +} + +uint32_t ProfilingReporter::GetTaskId(const string &node_name) { + auto index = node_name_index_map_[node_name]; + return task_ids_[index]; +} + +void ProfilingReporter::ReportData(int32_t device_id, unsigned char *data, size_t data_size, const string &tag_name) { + ReporterData report_data{}; + report_data.deviceId = device_id; + report_data.data = data; + report_data.dataLen = data_size; + auto ret = memcpy_s(report_data.tag, MSPROF_ENGINE_MAX_TAG_LEN + 1, tag_name.c_str(), tag_name.length()); + if (ret != 0) { + MS_LOG(EXCEPTION) << "Report data failed, tag is " << tag_name.c_str() << ", ret: " << ret; + } + + auto report_ret = ProfilingManager::GetInstance().CallMsprofReport(NOT_NULL(&report_data)); + if (report_ret != 0) { + MS_LOG(EXCEPTION) << "Report data failed, tag is " << tag_name.c_str() << ", ret: " << ret; + } +} + +void ProfilingReporter::ConstructNodeNameIndexMap() { + if (!node_name_index_map_.empty()) { + return; + } + + size_t task_index = 0; + for (const auto &node : cnode_list_) { + MS_EXCEPTION_IF_NULL(node); + node_name_index_map_.insert(pair(node->fullname_with_scope(), task_index)); + ++task_index; + } +} + +uint32_t ProfilingReporter::GetBlockDim(const CNodePtr &node) { + auto kernel_mod = AnfAlgo::GetKernelMod(node); + auto ascend_kernel_mod = dynamic_cast(kernel_mod); + MS_EXCEPTION_IF_NULL(ascend_kernel_mod); + return ascend_kernel_mod->block_dim(); +} + +void ProfilingReporter::ReportTask(const CNodePtr &node, const uint32_t stream_id, uint32_t task_id, + KernelType kernel_type) { + MsprofGeProfTaskData task_info{}; + task_info.taskType = static_cast(KernelType2TaskTypeEnum[kernel_type]); + (void)SetAlternativeValue(task_info.opName, MSPROF_MIX_DATA_STRING_LEN, node->fullname_with_scope(), device_id_); + (void)SetAlternativeValue(task_info.opType, MSPROF_GE_OP_TYPE_LEN, AnfAlgo::GetCNodeName(node), device_id_); + // Note: Currently, the profiler supports only static shapes. + task_info.shapeType = static_cast(MSPROF_GE_SHAPE_TYPE_STATIC); + task_info.blockDims = GetBlockDim(node); + // Note: Currently, all steps are hardcoded to 0. + task_info.curIterNum = 0; + task_info.modelId = graph_id_; + task_info.streamId = stream_id; + task_info.taskId = task_id; + task_info.timeStamp = 0; + task_info.threadId = 0; + + (void)ReportData(device_id_, reinterpret_cast(&task_info), sizeof(task_info), "task_desc_info"); +} + +void ProfilingReporter::ReportNode(const CNodePtr &node, uint32_t stream_id, uint32_t task_id, uint32_t tensor_type) { + const std::string tag_name = "tensor_data_info"; + + size_t total_size = 0; + if (tensor_type == MSPROF_GE_TENSOR_TYPE_INPUT) { + total_size = AnfAlgo::GetInputTensorNum(node); + } else { + total_size = AnfAlgo::GetOutputTensorNum(node); + } + + const size_t batch_size = total_size / MSPROF_GE_TENSOR_DATA_NUM; + for (size_t i = 0U; i < batch_size; i++) { + MsprofGeProfTensorData tensor_info{}; + (void)BuildProfTensorDataCommon(tensor_info, stream_id, task_id); + tensor_info.tensorNum = MSPROF_GE_TENSOR_DATA_NUM; + for (size_t j = 0U; j < MSPROF_GE_TENSOR_DATA_NUM; j++) { + size_t cur_index = i * MSPROF_GE_TENSOR_DATA_NUM + j; + MsprofGeTensorData tensor_data{}; + (void)BuildTensorData(tensor_data, node, cur_index, tensor_type); + tensor_info.tensorData[j] = tensor_data; + } + (void)ReportData(device_id_, reinterpret_cast(&tensor_info), sizeof(tensor_info), tag_name); + } + + size_t remain_size = total_size % MSPROF_GE_TENSOR_DATA_NUM; + if (remain_size == 0) { + return; + } + + MsprofGeProfTensorData tensor_info{}; + (void)BuildProfTensorDataCommon(tensor_info, stream_id, task_id); + tensor_info.tensorNum = remain_size; + for (size_t i = 0U; i < remain_size; ++i) { + MsprofGeTensorData tensor_data{}; + size_t cur_index = batch_size * MSPROF_GE_TENSOR_DATA_NUM + i; + (void)BuildTensorData(tensor_data, node, cur_index, tensor_type); + tensor_info.tensorData[i] = tensor_data; + } + (void)ReportData(device_id_, reinterpret_cast(&tensor_info), sizeof(tensor_info), tag_name); +} + +void ProfilingReporter::BuildProfTensorDataCommon(MsprofGeProfTensorData &tensor_info, uint32_t stream_id, + uint32_t task_id) { + tensor_info.modelId = graph_id_; + tensor_info.streamId = stream_id; + tensor_info.taskId = task_id; + // Note: Currently, all steps are hardcoded to 0. + tensor_info.curIterNum = 0; +} + +void ProfilingReporter::BuildTensorData(MsprofGeTensorData &tensor_data, const CNodePtr &node, size_t index, + uint32_t tensor_type) { + tensor_data.tensorType = tensor_type; + std::vector shape; + string data_format; + if (tensor_type == MSPROF_GE_TENSOR_TYPE_INPUT) { + auto input_node_with_index = AnfAlgo::GetPrevNodeOutput(node, index); + auto input_node = input_node_with_index.first; + auto input_index = input_node_with_index.second; + shape = AnfAlgo::GetOutputDeviceShape(input_node, input_index); + data_format = AnfAlgo::GetOutputFormat(input_node, input_index); + tensor_data.dataType = static_cast(AnfAlgo::GetOutputDeviceDataType(input_node, input_index)); + } else { + shape = AnfAlgo::GetOutputDeviceShape(node, index); + data_format = AnfAlgo::GetOutputFormat(node, index); + tensor_data.dataType = static_cast(AnfAlgo::GetOutputDeviceDataType(node, index)); + } + + tensor_data.format = OpFormat2Index[data_format]; + auto shape_size = std::min(static_cast(MSPROF_GE_TENSOR_DATA_SHAPE_LEN), shape.size()); + std::copy(shape.begin(), shape.begin() + shape_size, tensor_data.shape); +} +} // namespace ascend +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/runtime/device/ascend/profiling/profiling_reporter.h b/mindspore/ccsrc/runtime/device/ascend/profiling/profiling_reporter.h new file mode 100644 index 00000000000..7a6f8a2abec --- /dev/null +++ b/mindspore/ccsrc/runtime/device/ascend/profiling/profiling_reporter.h @@ -0,0 +1,105 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_PROFILING_REPORTER_H +#define MINDSPORE_PROFILING_REPORTER_H +#include +#include +#include +#include + +#include "securec/include/securec.h" +#include "utils/log_adapter.h" +#include "backend/session/anf_runtime_algorithm.h" +#include "runtime/device/ascend/profiling/prof_common.h" +#include "runtime/device/ascend/profiling/profiling_manager.h" +#include "toolchain/prof_reporter.h" + +namespace mindspore { +namespace device { +namespace ascend { +using std::pair; +using std::string; +using std::vector; + +class StepPointDesc { + public: + StepPointDesc(string op_name, uint32_t tag) : op_name_(std::move(op_name)), tag_(tag) {} + ~StepPointDesc() = default; + + string op_name() { return op_name_; } + uint32_t tag() const { return tag_; } + + private: + string op_name_; + uint32_t tag_; +}; + +class ProfilingReporter { + public: + ProfilingReporter(int device_id, uint32_t graph_id, vector cnode_list, const vector &stream_ids, + const vector &task_ids) + : device_id_(device_id), + graph_id_(graph_id), + cnode_list_(std::move(cnode_list)), + stream_ids_(stream_ids), + task_ids_(task_ids) {} + ~ProfilingReporter() = default; + + void ReportTasks(); + void ReportStepPoint(const vector> &points); + + private: + uint32_t device_id_; + uint32_t graph_id_; + vector cnode_list_; + vector stream_ids_; + vector task_ids_; + map node_name_index_map_; + + bool CheckStreamTaskValid(); + static uint32_t GetBlockDim(const CNodePtr &node); + void ConstructNodeNameIndexMap(); + uint32_t GetStreamId(const string &node_name); + uint32_t GetTaskId(const string &node_name); + + void ReportData(int32_t device_id, unsigned char *data, size_t data_size, const std::string &tag_name); + void ReportTask(const CNodePtr &node, uint32_t stream_id, uint32_t task_id, KernelType kernel_type); + void ReportNode(const CNodePtr &node, uint32_t stream_id, uint32_t task_id, uint32_t tensor_type); + void BuildProfTensorDataCommon(MsprofGeProfTensorData &tensor_info, uint32_t stream_id, uint32_t task_id); + void BuildTensorData(MsprofGeTensorData &tensor_data, const CNodePtr &node, size_t index, uint32_t tensor_type); + + template + void SetAlternativeValue(T &property, const size_t property_size, const string &value, const int32_t &device_id) { + if (value.size() < property_size) { + property.type = static_cast(MSPROF_MIX_DATA_HASH_ID); + const auto ret = strncpy_s(property.data.dataStr, property_size, value.c_str(), value.size()); + if (ret != 0) { + MS_LOG(ERROR) << "[Profiling] strncpy_s value " << value.c_str() << " error!"; + return; + } + } else { + property.type = static_cast(MSPROF_MIX_DATA_STRING); + uint64_t hash_id; + (void)ProfilingManager::GetInstance().QueryHashId(device_id, value, hash_id); + property.data.hashId = hash_id; + } + } +}; +} // namespace ascend +} // namespace device +} // namespace mindspore +#endif // MINDSPORE_PROFILING_REPORTER_H diff --git a/mindspore/ccsrc/runtime/device/ascend/profiling/profiling_utils.cc b/mindspore/ccsrc/runtime/device/ascend/profiling/profiling_utils.cc index 28aa4d2882d..82ebe8cb687 100644 --- a/mindspore/ccsrc/runtime/device/ascend/profiling/profiling_utils.cc +++ b/mindspore/ccsrc/runtime/device/ascend/profiling/profiling_utils.cc @@ -15,16 +15,13 @@ */ #include -#include "runtime/device/ascend/profiling/reporter/graph_desc_reporter.h" #include "runtime/device/ascend/profiling/profiling_utils.h" #include "backend/kernel_compiler/kernel.h" #include "runtime/device/ascend/profiling/profiling_manager.h" #include "backend/session/anf_runtime_algorithm.h" #include "utils/ms_utils.h" #include "utils/utils.h" -#include "runtime/device/ascend/profiling/reporter/task_desc_reporter.h" #include "utils/ms_context.h" -#include "runtime/device/ascend/profiling/reporter/point_reporter.h" #include "nlohmann/json.hpp" #include "base/core_ops.h" #include "profiler/device/profiling.h" @@ -271,12 +268,14 @@ NotNull ProfilingUtils::CreateProfilingCNode(const ProfilingContent &p } void ProfilingUtils::SaveProfilingPoint(uint32_t graph_id, const std::string &node_name, uint32_t point_id) { - std::shared_ptr prof_desc_ptr = std::make_shared(node_name, point_id); + MS_LOG(INFO) << "Save profiling point, graph id" << graph_id << ", node name: " << node_name + << ", point_id: " << point_id; + std::shared_ptr point_desc_ptr = std::make_shared(node_name, point_id); auto iter = graph_point_.find(graph_id); if (iter == graph_point_.end()) { - graph_point_.emplace(graph_id, std::vector>{prof_desc_ptr}); + graph_point_.emplace(graph_id, std::vector>{point_desc_ptr}); } else { - iter->second.emplace_back(prof_desc_ptr); + iter->second.emplace_back(point_desc_ptr); } } @@ -383,7 +382,8 @@ void ProfilingUtils::SetGraphProfilingCNode(uint32_t graph_id, const std::vector bool ProfilingUtils::ValidComputeGraph(const session::KernelGraph &kernel_graph) { for (const auto &node : kernel_graph.execution_order()) { - if (AnfAlgo::GetKernelType(node) == TBE_KERNEL || AnfAlgo::GetKernelType(node) == AKG_KERNEL) { + auto kernel_type = AnfAlgo::GetKernelType(node); + if (kernel_type == TBE_KERNEL || kernel_type == AKG_KERNEL || kernel_type == AICPU_KERNEL) { return true; } } @@ -400,32 +400,26 @@ void ProfilingUtils::ReportProfilingData(const std::vector &task_ids, uint32_t graph_id) { auto ret = graph_profiling_cnode_.find(graph_id); if (ret == graph_profiling_cnode_.end()) { - MS_LOG(ERROR) << "Graph id not found"; + MS_LOG(ERROR) << "Graph id not found in graph_profiling_cnode_, graph id is " << graph_id + << ", will not report this graph profiling data."; return; } auto context = MsContext::GetInstance(); MS_EXCEPTION_IF_NULL(context); - TaskDescReporter task_reporter(context->get_param(MS_CTX_DEVICE_ID), "vm_task_desc_info", ret->second); - task_reporter.set_task_ids(task_ids); - task_reporter.set_stream_ids(stream_ids); - task_reporter.ReportData(); - GraphDescReporter graph_reporter(context->get_param(MS_CTX_DEVICE_ID), "vm_graph_desc_info", ret->second); - graph_profiling_cnode_.erase(ret); - graph_reporter.ReportData(); + auto device_id = context->get_param(MS_CTX_DEVICE_ID); + ProfilingReporter reporter(device_id, graph_id, ret->second, stream_ids, task_ids); + reporter.ReportTasks(); // Report profiling point auto point_iter = graph_point_.find(graph_id); if (point_iter == graph_point_.end()) { - MS_LOG(ERROR) << "Graph id not found in graph_point"; + MS_LOG(ERROR) << "Graph id not found in graph_point, will not report this graph step point data, graph id is: " + << graph_id; return; } - PointReporter point_reporter(context->get_param(MS_CTX_DEVICE_ID), "vm_point"); - for (const auto &point : point_iter->second) { - point_reporter.AddReportData(point); - } - point_reporter.ReportData(); + reporter.ReportStepPoint(point_iter->second); } void ProfilingUtils::SetReportProfilingData(const std::vector &task_ids, diff --git a/mindspore/ccsrc/runtime/device/ascend/profiling/profiling_utils.h b/mindspore/ccsrc/runtime/device/ascend/profiling/profiling_utils.h index 401e081d02b..73fa747f931 100644 --- a/mindspore/ccsrc/runtime/device/ascend/profiling/profiling_utils.h +++ b/mindspore/ccsrc/runtime/device/ascend/profiling/profiling_utils.h @@ -24,7 +24,7 @@ #include #include "backend/session/kernel_graph.h" #include "utils/contract.h" -#include "runtime/device/ascend/profiling/reporter/profiling_desc.h" +#include "runtime/device/ascend/profiling/profiling_reporter.h" namespace mindspore { namespace device { @@ -117,7 +117,7 @@ class ProfilingUtils { // graph id --> (kernel name list) inline static std::map> graph_profiling_cnode_; inline static std::map> graph_kernel_name_; - inline static std::map>> graph_point_; + inline static std::map>> graph_point_; inline static uint32_t custom_node_index_; inline static std::vector report_data_; }; diff --git a/mindspore/ccsrc/runtime/device/ascend/profiling/reporter/desc_reporter.cc b/mindspore/ccsrc/runtime/device/ascend/profiling/reporter/desc_reporter.cc deleted file mode 100644 index 7237e07f1af..00000000000 --- a/mindspore/ccsrc/runtime/device/ascend/profiling/reporter/desc_reporter.cc +++ /dev/null @@ -1,66 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include "runtime/device/ascend/profiling/reporter/desc_reporter.h" -#include "runtime/device/ascend/profiling/profiling_manager.h" -#include "utils/log_adapter.h" - -namespace { -constexpr size_t kReportMaxLen = 1024; -} -namespace mindspore { -namespace device { -namespace ascend { -DescReporter::~DescReporter() = default; - -void DescReporter::ReportByLine(const std::string &data, const std::string &file_name) const { - auto tot_size = data.size(); - size_t cur_size = 0; - while (cur_size < tot_size) { - size_t remain_size = tot_size - cur_size; - size_t report_size = std::min(remain_size, kReportMaxLen); - - ReporterData report_data{}; - report_data.deviceId = device_id_; - report_data.dataLen = report_size; - report_data.data = (unsigned char *)data.c_str() + cur_size; - auto ret = memcpy_s(report_data.tag, MSPROF_ENGINE_MAX_TAG_LEN + 1, file_name.c_str(), file_name.length()); - if (ret != 0) { - MS_LOG(EXCEPTION) << "Memcpy_s report data tag failed"; - } - auto report_ret = ProfilingManager::GetInstance().CallMsprofReport(NOT_NULL(&report_data)); - if (report_ret != 0) { - MS_LOG(EXCEPTION) << "Report data failed"; - } - if (report_size == 0) { - MS_LOG(WARNING) << "Report_size is 0"; - break; - } - cur_size += report_size; - } -} - -void DescReporter::ReportAllLine() { - for (const auto &desc : prof_desc_list_) { - MS_EXCEPTION_IF_NULL(desc); - auto data = desc->ToString(); - ReportByLine(data, file_name_); - } -} -} // namespace ascend -} // namespace device -} // namespace mindspore diff --git a/mindspore/ccsrc/runtime/device/ascend/profiling/reporter/desc_reporter.h b/mindspore/ccsrc/runtime/device/ascend/profiling/reporter/desc_reporter.h deleted file mode 100644 index 6b6ee9e9924..00000000000 --- a/mindspore/ccsrc/runtime/device/ascend/profiling/reporter/desc_reporter.h +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef MINDSPORE_CCSRC_RUNTIME_DEVICE_ASCEND_PROFILING_REPORTER_DESC_REPORTER_H_ -#define MINDSPORE_CCSRC_RUNTIME_DEVICE_ASCEND_PROFILING_REPORTER_DESC_REPORTER_H_ - -#include -#include -#include -#include -#include "toolchain/prof_reporter.h" -#include "runtime/device/ascend/profiling/reporter/profiling_desc.h" -#include "utils/contract.h" -#include "backend/session/kernel_graph.h" - -namespace mindspore { -namespace device { -namespace ascend { -class DescReporter { - public: - virtual ~DescReporter() = 0; - DescReporter(int device_id, std::string file_name) : device_id_(device_id), file_name_(std::move(file_name)) {} - - virtual void ReportData() = 0; - - protected: - void ReportByLine(const std::string &data, const std::string &file_name) const; - void ReportAllLine(); - - int device_id_; - std::string file_name_; - std::vector> prof_desc_list_; -}; -} // namespace ascend -} // namespace device -} // namespace mindspore -#endif // MINDSPORE_CCSRC_RUNTIME_DEVICE_ASCEND_PROFILING_REPORTER_DESC_REPORTER_H_ diff --git a/mindspore/ccsrc/runtime/device/ascend/profiling/reporter/graph_desc_reporter.cc b/mindspore/ccsrc/runtime/device/ascend/profiling/reporter/graph_desc_reporter.cc deleted file mode 100644 index f00c91d7963..00000000000 --- a/mindspore/ccsrc/runtime/device/ascend/profiling/reporter/graph_desc_reporter.cc +++ /dev/null @@ -1,64 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "runtime/device/ascend/profiling/reporter/graph_desc_reporter.h" -#include "backend/session/anf_runtime_algorithm.h" - -namespace mindspore { -namespace device { -namespace ascend { -void GraphDescReporter::ReportData() { - for (const auto &node : cnode_list_) { - MS_EXCEPTION_IF_NULL(node); - if (AnfAlgo::GetKernelType(node) != TBE_KERNEL && AnfAlgo::GetKernelType(node) != AKG_KERNEL) { - MS_LOG(INFO) << "Skip non tbe kernel:" << node->fullname_with_scope(); - continue; - } - std::vector input_data_list; - std::vector output_data_list; - auto op_name = node->fullname_with_scope(); - auto op_type = AnfAlgo::GetCNodeName(node); - auto input_size = AnfAlgo::GetInputTensorNum(node); - for (size_t i = 0; i < input_size; ++i) { - auto input_node_with_index = AnfAlgo::GetPrevNodeOutput(node, i); - auto input_node = input_node_with_index.first; - auto input_index = input_node_with_index.second; - DataElement element{}; - element.index_ = i; - element.data_type_ = AnfAlgo::GetOutputDeviceDataType(input_node, input_index); - element.data_format_ = AnfAlgo::GetOutputFormat(input_node, input_index); - element.data_shape_ = AnfAlgo::GetOutputDeviceShape(input_node, input_index); - input_data_list.emplace_back(element); - } - - auto output_size = AnfAlgo::GetOutputTensorNum(node); - for (size_t i = 0; i < output_size; ++i) { - DataElement element{}; - element.index_ = i; - element.data_type_ = AnfAlgo::GetOutputDeviceDataType(node, i); - element.data_format_ = AnfAlgo::GetOutputFormat(node, i); - element.data_shape_ = AnfAlgo::GetOutputDeviceShape(node, i); - output_data_list.emplace_back(element); - } - - auto graph_desc = std::make_shared(op_name, op_type, input_data_list, output_data_list); - prof_desc_list_.emplace_back(graph_desc); - } - ReportAllLine(); -} -} // namespace ascend -} // namespace device -} // namespace mindspore diff --git a/mindspore/ccsrc/runtime/device/ascend/profiling/reporter/graph_desc_reporter.h b/mindspore/ccsrc/runtime/device/ascend/profiling/reporter/graph_desc_reporter.h deleted file mode 100644 index 2ae9ff45ff3..00000000000 --- a/mindspore/ccsrc/runtime/device/ascend/profiling/reporter/graph_desc_reporter.h +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef MINDSPORE_CCSRC_RUNTIME_DEVICE_ASCEND_PROFILING_REPORTER_GRAPH_DESC_REPORTER_H_ -#define MINDSPORE_CCSRC_RUNTIME_DEVICE_ASCEND_PROFILING_REPORTER_GRAPH_DESC_REPORTER_H_ - -#include -#include -#include -#include "runtime/device/ascend/profiling/reporter/desc_reporter.h" - -namespace mindspore { -namespace device { -namespace ascend { -class GraphDescReporter : public DescReporter { - public: - GraphDescReporter(uint32_t device_id, const std::string &file_name, std::vector cnode_list) - : DescReporter(device_id, file_name), cnode_list_(std::move(cnode_list)) {} - ~GraphDescReporter() override = default; - void ReportData() override; - - private: - std::vector cnode_list_; -}; -} // namespace ascend -} // namespace device -} // namespace mindspore -#endif // MINDSPORE_CCSRC_RUNTIME_DEVICE_ASCEND_PROFILING_REPORTER_GRAPH_DESC_REPORTER_H_ diff --git a/mindspore/ccsrc/runtime/device/ascend/profiling/reporter/op_name_task_stream_reporter.cc b/mindspore/ccsrc/runtime/device/ascend/profiling/reporter/op_name_task_stream_reporter.cc deleted file mode 100644 index 211ca99490a..00000000000 --- a/mindspore/ccsrc/runtime/device/ascend/profiling/reporter/op_name_task_stream_reporter.cc +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "runtime/device/ascend/profiling/reporter/op_name_task_stream_reporter.h" - -namespace mindspore { -namespace device { -namespace ascend { -void OpNameTaskStreamReporter::ReportData() { - MS_LOG(INFO) << "ReportData start"; - - std::map>> op_name_map; - for (auto &iter : stream_id_task_id_op_name_map_) { - auto pair = iter.first; - auto op_name = iter.second; - auto ret = op_name_map.find(op_name); - if (ret == op_name_map.end()) { - auto vect = std::vector>(1, pair); - auto emplace_ret = op_name_map.emplace(op_name, vect); - if (!emplace_ret.second) { - MS_LOG(WARNING) << "Duplicate op_name:" << op_name << " task_id:" << pair.first << " stream_id:" << pair.second; - } - } else { - ret->second.emplace_back(pair); - } - } - - for (const auto &iter : op_name_map) { - auto desc_ptr = std::make_shared(iter.first, iter.second); - prof_desc_list_.emplace_back(desc_ptr); - } - ReportAllLine(); - MS_LOG(INFO) << "ReportData end"; -} -} // namespace ascend -} // namespace device -} // namespace mindspore diff --git a/mindspore/ccsrc/runtime/device/ascend/profiling/reporter/op_name_task_stream_reporter.h b/mindspore/ccsrc/runtime/device/ascend/profiling/reporter/op_name_task_stream_reporter.h deleted file mode 100644 index 6fe142b1160..00000000000 --- a/mindspore/ccsrc/runtime/device/ascend/profiling/reporter/op_name_task_stream_reporter.h +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef MINDSPORE_MINDSPORE_CCSRC_RUNTIME_DEVICE_ASCEND_PROFILING_REPORTER_OP_NAME_TASK_STREAM_REPORTER_H_ -#define MINDSPORE_MINDSPORE_CCSRC_RUNTIME_DEVICE_ASCEND_PROFILING_REPORTER_OP_NAME_TASK_STREAM_REPORTER_H_ - -#include -#include -#include -#include "runtime/device/ascend/profiling/reporter/desc_reporter.h" - -namespace mindspore { -namespace device { -namespace ascend { -class OpNameTaskStreamReporter : public DescReporter { - public: - OpNameTaskStreamReporter(uint32_t device_id, const std::string &file_name, - std::map, std::string> stream_id_task_id_op_name_map) - : DescReporter(device_id, file_name), stream_id_task_id_op_name_map_(std::move(stream_id_task_id_op_name_map)) {} - ~OpNameTaskStreamReporter() override = default; - void ReportData() override; - - private: - std::map, std::string> stream_id_task_id_op_name_map_; -}; -} // namespace ascend -} // namespace device -} // namespace mindspore -#endif // MINDSPORE_MINDSPORE_CCSRC_RUNTIME_DEVICE_ASCEND_PROFILING_REPORTER_OP_NAME_TASK_STREAM_REPORTER_H_ diff --git a/mindspore/ccsrc/runtime/device/ascend/profiling/reporter/point_reporter.cc b/mindspore/ccsrc/runtime/device/ascend/profiling/reporter/point_reporter.cc deleted file mode 100644 index 42a1b4c286d..00000000000 --- a/mindspore/ccsrc/runtime/device/ascend/profiling/reporter/point_reporter.cc +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "runtime/device/ascend/profiling/reporter/point_reporter.h" - -namespace mindspore { -namespace device { -namespace ascend { -void PointReporter::ReportData() { ReportAllLine(); } - -void PointReporter::AddReportData(const std::shared_ptr &prof_desc) { - prof_desc_list_.emplace_back(prof_desc); -} -} // namespace ascend -} // namespace device -} // namespace mindspore diff --git a/mindspore/ccsrc/runtime/device/ascend/profiling/reporter/point_reporter.h b/mindspore/ccsrc/runtime/device/ascend/profiling/reporter/point_reporter.h deleted file mode 100644 index a5c806020d6..00000000000 --- a/mindspore/ccsrc/runtime/device/ascend/profiling/reporter/point_reporter.h +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef MINDSPORE_CCSRC_RUNTIME_DEVICE_ASCEND_PROFILING_REPORTER_POINT_REPORTER_H_ -#define MINDSPORE_CCSRC_RUNTIME_DEVICE_ASCEND_PROFILING_REPORTER_POINT_REPORTER_H_ - -#include -#include -#include "runtime/device/ascend/profiling/reporter/desc_reporter.h" - -namespace mindspore { -namespace device { -namespace ascend { -class PointReporter : public DescReporter { - public: - PointReporter(uint32_t device_id, const std::string &file_name) : DescReporter(device_id, file_name) {} - ~PointReporter() override = default; - void ReportData() override; - void AddReportData(const std::shared_ptr &prof_desc); -}; -} // namespace ascend -} // namespace device -} // namespace mindspore -#endif // MINDSPORE_CCSRC_RUNTIME_DEVICE_ASCEND_PROFILING_REPORTER_POINT_REPORTER_H_ diff --git a/mindspore/ccsrc/runtime/device/ascend/profiling/reporter/profiling_desc.cc b/mindspore/ccsrc/runtime/device/ascend/profiling/reporter/profiling_desc.cc deleted file mode 100644 index 9f7b2ee9e5e..00000000000 --- a/mindspore/ccsrc/runtime/device/ascend/profiling/reporter/profiling_desc.cc +++ /dev/null @@ -1,99 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "runtime/device/ascend/profiling/reporter/profiling_desc.h" -#include -#include - -namespace mindspore { -namespace device { -namespace ascend { -std::string TaskDesc::ToString() { - std::string out = op_name_; - out.append(" ") - .append(std::to_string(block_dim_)) - .append(" ") - .append(std::to_string(task_id_)) - .append(" ") - .append(std::to_string(stream_id_)) - .append("\n"); - return out; -} - -std::string GraphDesc::ToString() { - std::string desc; - desc.append("op_name:").append(op_name_).append(" op_type:").append(op_type_); - int input_id = 0; - for (const auto &element : input_data_list_) { - desc.append(" input_id:") - .append(std::to_string(input_id++)) - .append(" input_format:") - .append(element.data_format_) - .append(" input_data_type:") - .append(std::to_string(element.data_type_)) - .append(" input_shape:") - .append(DataShapeToString(element.data_shape_)); - } - - input_id = 0; - for (const auto &element : output_data_list_) { - desc.append(" output_id:") - .append(std::to_string(input_id++)) - .append(" output_format:") - .append(element.data_format_) - .append(" output_data_type:") - .append(std::to_string(element.data_type_)) - .append(" output_shape:") - .append((DataShapeToString(element.data_shape_))); - } - - desc.append("\n"); - - return desc; -} - -std::string PointDesc::ToString() { - std::string desc; - desc.append(std::to_string(point_id_)).append(" ").append(op_name_).append("\n"); - return desc; -} - -std::string GraphDesc::DataShapeToString(const std::vector &shape) { - std::ostringstream oss; - oss << "\""; - if (!shape.empty()) { - std::copy(shape.begin(), shape.end() - 1, std::ostream_iterator(oss, ",")); - oss << shape.back(); - } - oss << "\""; - return oss.str(); -} - -std::string TaskStreamOpNameDesc::ToString() { - std::string desc = op_name_; - // op_name "task_id stream_id" "task_id stream_id" - for (auto pair : stream_id_task_id_pairs_) { - desc.append(" "); - desc.append(std::to_string(pair.first)); - desc.append("_"); - desc.append(std::to_string(pair.second)); - } - desc.append("\n"); - return desc; -} -} // namespace ascend -} // namespace device -} // namespace mindspore diff --git a/mindspore/ccsrc/runtime/device/ascend/profiling/reporter/profiling_desc.h b/mindspore/ccsrc/runtime/device/ascend/profiling/reporter/profiling_desc.h deleted file mode 100644 index 9d867c69032..00000000000 --- a/mindspore/ccsrc/runtime/device/ascend/profiling/reporter/profiling_desc.h +++ /dev/null @@ -1,100 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef MINDSPORE_CCSRC_RUNTIME_DEVICE_ASCEND_PROFILING_REPORTER_PROFILING_DESC_H_ -#define MINDSPORE_CCSRC_RUNTIME_DEVICE_ASCEND_PROFILING_REPORTER_PROFILING_DESC_H_ - -#include -#include -#include - -namespace mindspore { -namespace device { -namespace ascend { -class ProfDesc { - public: - explicit ProfDesc(std::string op_name) : op_name_(std::move(op_name)) {} - virtual ~ProfDesc() = default; - virtual std::string ToString() = 0; - - protected: - std::string op_name_; -}; - -class TaskDesc : public ProfDesc { - public: - TaskDesc(std::string op_name, uint32_t task_id, uint32_t block_dim, uint32_t stream_id) - : ProfDesc(std::move(op_name)), task_id_(task_id), block_dim_(block_dim), stream_id_(stream_id) {} - ~TaskDesc() override = default; - std::string ToString() override; - - private: - uint32_t task_id_; - uint32_t block_dim_; - uint32_t stream_id_; -}; - -struct DataElement { - size_t index_; - std::string data_format_; - int data_type_; - std::vector data_shape_; -}; - -class GraphDesc : public ProfDesc { - public: - GraphDesc(std::string op_name, std::string op_type, std::vector input_data_list, - std::vector output_data_list) - : ProfDesc(std::move(op_name)), - op_type_(std::move(op_type)), - input_data_list_(std::move(input_data_list)), - output_data_list_(std::move(output_data_list)) {} - ~GraphDesc() override = default; - std::string ToString() override; - - private: - std::string op_type_; - std::vector input_data_list_; - std::vector output_data_list_; - [[nodiscard]] static std::string DataShapeToString(const std::vector &shape); -}; - -class PointDesc : public ProfDesc { - public: - PointDesc(std::string op_name, uint32_t point_id) : ProfDesc(std::move(op_name)), point_id_(point_id) {} - ~PointDesc() override = default; - std::string ToString() override; - - private: - uint32_t point_id_; -}; - -class TaskStreamOpNameDesc : public ProfDesc { - public: - TaskStreamOpNameDesc(std::string op_name, std::vector> stream_id_task_id_pairs) - : ProfDesc(std::move(op_name)), stream_id_task_id_pairs_(std::move(stream_id_task_id_pairs)) {} - - ~TaskStreamOpNameDesc() override = default; - std::string ToString() override; - - private: - std::vector> stream_id_task_id_pairs_; -}; -} // namespace ascend -} // namespace device -} // namespace mindspore - -#endif // MINDSPORE_CCSRC_RUNTIME_DEVICE_ASCEND_PROFILING_REPORTER_PROFILING_DESC_H_ diff --git a/mindspore/ccsrc/runtime/device/ascend/profiling/reporter/task_desc_reporter.cc b/mindspore/ccsrc/runtime/device/ascend/profiling/reporter/task_desc_reporter.cc deleted file mode 100644 index 8df5fc19d1c..00000000000 --- a/mindspore/ccsrc/runtime/device/ascend/profiling/reporter/task_desc_reporter.cc +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "runtime/device/ascend/profiling/reporter/task_desc_reporter.h" -#include "backend/session/anf_runtime_algorithm.h" -#include "backend/kernel_compiler/ascend_kernel_mod.h" - -namespace mindspore { -namespace device { -namespace ascend { -void TaskDescReporter::ReportData() { - MS_LOG(INFO) << "cnode_list.size()=" << cnode_list_.size() << " task_ids_.size()=" << task_ids_.size(); - if (cnode_list_.size() != task_ids_.size()) { - MS_LOG(ERROR) << "cnode list size not equal task ids size"; - return; - } - - size_t task_index = 0; - for (const auto &node : cnode_list_) { - MS_EXCEPTION_IF_NULL(node); - if (AnfAlgo::GetKernelType(node) != TBE_KERNEL && AnfAlgo::GetKernelType(node) != AKG_KERNEL) { - MS_LOG(INFO) << "Skip non tbe kernel:" << node->fullname_with_scope(); - ++task_index; - continue; - } - auto kernel_mod = AnfAlgo::GetKernelMod(node); - auto ascend_kernel_mod = dynamic_cast(kernel_mod); - MS_EXCEPTION_IF_NULL(ascend_kernel_mod); - // Check task_id and stream_id valid - CheckStreamTaskValid(task_index, task_index); - auto desc_ptr = std::make_shared(node->fullname_with_scope(), task_ids_[task_index], - ascend_kernel_mod->block_dim(), stream_ids_[task_index]); - prof_desc_list_.emplace_back(desc_ptr); - ++task_index; - } - ReportAllLine(); -} - -void TaskDescReporter::CheckStreamTaskValid(size_t task_id, size_t stream_id) const { - if (task_id >= task_ids_.size() || stream_id >= stream_ids_.size()) { - MS_LOG(EXCEPTION) << "Index invalid. task_id:" << task_id << ", task_ids.size:" << task_ids_.size() - << ", stream_id:" << stream_id << ", stream_ids.size:" << stream_ids_.size(); - } -} -} // namespace ascend -} // namespace device -} // namespace mindspore diff --git a/mindspore/ccsrc/runtime/device/ascend/profiling/reporter/task_desc_reporter.h b/mindspore/ccsrc/runtime/device/ascend/profiling/reporter/task_desc_reporter.h deleted file mode 100644 index 30a3b7e662c..00000000000 --- a/mindspore/ccsrc/runtime/device/ascend/profiling/reporter/task_desc_reporter.h +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef MINDSPORE_CCSRC_RUNTIME_DEVICE_ASCEND_PROFILING_REPORTER_TASK_DESC_REPORTER_H_ -#define MINDSPORE_CCSRC_RUNTIME_DEVICE_ASCEND_PROFILING_REPORTER_TASK_DESC_REPORTER_H_ - -#include -#include -#include -#include "runtime/device/ascend/profiling/reporter/desc_reporter.h" - -namespace mindspore { -namespace device { -namespace ascend { -class TaskDescReporter : public DescReporter { - public: - TaskDescReporter(int device_id, const std::string &file_name, std::vector cnode_list) - : DescReporter(device_id, file_name), cnode_list_(std::move(cnode_list)) {} - ~TaskDescReporter() override = default; - void ReportData() override; - void set_task_ids(const std::vector &task_ids) { task_ids_ = task_ids; } - void set_stream_ids(const std::vector &stream_ids) { stream_ids_ = stream_ids; } - - private: - std::vector task_ids_; - std::vector stream_ids_; - void CheckStreamTaskValid(size_t task_id, size_t stream_id) const; - std::vector cnode_list_; -}; -} // namespace ascend -} // namespace device -} // namespace mindspore -#endif // MINDSPORE_CCSRC_RUNTIME_DEVICE_ASCEND_PROFILING_REPORTER_TASK_DESC_REPORTER_H_ diff --git a/mindspore/python/mindspore/profiler/common/struct_type.py b/mindspore/python/mindspore/profiler/common/struct_type.py new file mode 100644 index 00000000000..f41dfdbf3f1 --- /dev/null +++ b/mindspore/python/mindspore/profiler/common/struct_type.py @@ -0,0 +1,66 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Define a cpp type map python struct type.""" + +from enum import Enum + + +class StructType(Enum): + """Define cpp struct type, value is python struct type.""" + CHAR = 'c' + UCHAR = 'B' + UINT16 = 'H' + UINT32 = 'I' + UINT64 = 'Q' + + @classmethod + def format(cls, cpp_types): + """ + Given a Cpp type list, and return a python struct format string. + + Args: + cpp_types (list): The cpp type list that should be a member of StructType. + + Returns: + str, a python struct format string. + + Example: + >>> cpp_typs = [StructType.UINT16, StructType.UINT32, StructType.UINT32] + >>> ret = StructType.format(cpp_typs) + >>> print(ret) + ... 'HII' + """ + return ''.join([member.value for member in cpp_types]) + + @classmethod + def sizeof(cls, cpp_type): + """Given a Cpp type list or a StructType value, and return a python struct format size.""" + size_map = dict( + CHAR=1, + UCHAR=1, + UINT16=2, + UINT32=4, + UINT64=8 + ) + if isinstance(cpp_type, StructType): + return size_map[cpp_type.name] + + size = 0 + for member in cpp_type: + if isinstance(member, list): + size += cls.sizeof(member) + else: + size += size_map[member.name] + return size diff --git a/mindspore/python/mindspore/profiler/parser/framework_enum.py b/mindspore/python/mindspore/profiler/parser/framework_enum.py new file mode 100644 index 00000000000..48f6356e2fd --- /dev/null +++ b/mindspore/python/mindspore/profiler/parser/framework_enum.py @@ -0,0 +1,105 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Define framework enum data.""" + +from enum import Enum, IntEnum + + +class FileDataType(Enum): + """Define framework file data type.""" + STEP_INFO = 'step_info' + HASH_DIC = 'hash_dic' + TENSOR_DATA_INFO = 'tensor_data_info' + TASK_DESC_INFO = 'task_desc_info' + + @classmethod + def members(cls): + return {member.value for member in cls} + + +class VmDataType(IntEnum): + """Definition of vm data type.""" + NUMBER_TYPE_BEGIN = 30 + BOOL = 31 + INT = 32 + INT8 = 33 + INT16 = 34 + INT32 = 35 + INT64 = 36 + UINT = 37 + UINT8 = 38 + UINT16 = 39 + UINT32 = 40 + UINT64 = 41 + FLOAT = 42 + FLOAT16 = 43 + FLOAT32 = 44 + FLOAT64 = 45 + COMPLEX = 46 + NUMBER_TYPE_END = 47 + + @classmethod + def get_data_type_name(cls, num): + """ + Get the name of data type by enum number. + + Args: + num (int): Enum number. + + Returns: + str, the name of data type. + """ + data_type = cls._value2member_map_.get(num) + return 'UNKNOWN' if data_type is None else data_type.name + + +class VmFormat(IntEnum): + """Define mindspore data type.""" + UNKNOWN = 0 + DEFAULT = 1 + NC1KHKWHWC0 = 2 + ND = 3 + NCHW = 4 + NHWC = 5 + HWCN = 6 + NC1HWC0 = 7 + FRAC_Z = 8 + C1HWNCOC0 = 9 + FRAC_NZ = 10 + NC1HWC0_C04 = 11 + FRACTAL_Z_C04 = 12 + NDHWC = 13 + FRACTAL_ZN_LSTM = 14 + FRACTAL_ZN_RNN = 15 + ND_RNN_BIAS = 16 + NDC1HWC0 = 17 + NCDHW = 18 + FRACTAL_Z_3D = 19 + DHWNC = 20 + DHWCN = 21 + + @classmethod + def get_format_name(cls, num): + """ + Get the name of format by enum number. + + Args: + num (int): Enum number. + + Returns: + str, the name of data type. + """ + format_name = cls._value2member_map_.get(num) + return 'UNKNOWN' if format_name is None else format_name.name diff --git a/mindspore/python/mindspore/profiler/parser/framework_parser.py b/mindspore/python/mindspore/profiler/parser/framework_parser.py index 970c83bd72a..13e48c34221 100644 --- a/mindspore/python/mindspore/profiler/parser/framework_parser.py +++ b/mindspore/python/mindspore/profiler/parser/framework_parser.py @@ -14,156 +14,36 @@ # ============================================================================ """Thr parser for parsing framework files.""" import csv -import enum -import json import os import re -import stat +import struct +import json +from pathlib import Path +from typing import List +from collections import defaultdict +from collections import namedtuple -from mindspore.profiler.common.exceptions.exceptions import \ - ProfilerPathErrorException, ProfilerDirNotFoundException, \ - ProfilerFileNotFoundException, ProfilerDeviceIdMismatchException, \ - ProfilerRawFileException, ProfilerParamValueErrorException -from mindspore.profiler.common.validator.validate_path import \ - validate_and_normalize_path +from mindspore import log as logger +from mindspore.profiler.parser.framework_struct import TASK_DESC_STRUCT +from mindspore.profiler.parser.framework_struct import TENSOR_DATA_STRUCT +from mindspore.profiler.parser.framework_struct import STEP_INFO_STRUCT +from mindspore.profiler.parser.framework_enum import VmDataType, VmFormat, FileDataType +from mindspore.profiler.common.struct_type import StructType + +from mindspore.profiler.common.exceptions.exceptions import ProfilerDirNotFoundException +from mindspore.profiler.common.exceptions.exceptions import ProfilerFileNotFoundException +from mindspore.profiler.common.exceptions.exceptions import ProfilerParamValueErrorException -class VmDataType(enum.IntEnum): - """Definition of vm data type.""" - NUMBER_TYPE_BEGIN = 30 - NUMBER_TYPE_BOOL = 31 - NUMBER_TYPE_INT = 32 - NUMBER_TYPE_INT8 = 33 - NUMBER_TYPE_INT16 = 34 - NUMBER_TYPE_INT32 = 35 - NUMBER_TYPE_INT64 = 36 - NUMBER_TYPE_UINT = 37 - NUMBER_TYPE_UINT8 = 38 - NUMBER_TYPE_UINT16 = 39 - NUMBER_TYPE_UINT32 = 40 - NUMBER_TYPE_UINT64 = 41 - NUMBER_TYPE_FLOAT = 42 - NUMBER_TYPE_FLOAT16 = 43 - NUMBER_TYPE_FLOAT32 = 44 - NUMBER_TYPE_FLOAT64 = 45 - NUMBER_TYPE_COMPLEX = 46 - NUMBER_TYPE_END = 47 - - @classmethod - def get_data_type_name(cls, num): - """ - Get the name of data type by enum number. - - Args: - num (int): Enum number. - - Returns: - str, the name of data type. - """ - data_type = cls._value2member_map_.get(num) - return 'UNKNOWN' if data_type is None else data_type.name +FILE_DATA_STRUCT_DICT = { + FileDataType.STEP_INFO.value: STEP_INFO_STRUCT, + FileDataType.TENSOR_DATA_INFO.value: TENSOR_DATA_STRUCT, + FileDataType.TASK_DESC_INFO.value: TASK_DESC_STRUCT +} -class GeDataType(enum.IntEnum): - """Definition of ge data type.""" - DT_FLOAT = 0 - DT_FLOAT16 = 1 - DT_INT8 = 2 - DT_INT16 = 6 - DT_UINT16 = 7 - DT_UINT8 = 4 - DT_INT32 = 3 - DT_INT64 = 9 - DT_UINT32 = 8 - DT_UINT64 = 10 - DT_BOOL = 12 - DT_DOUBLE = 11 - DT_STRING = 13 - DT_DUAL_SUB_INT8 = 14 - DT_DUAL_SUB_UINT8 = 15 - DT_COMPLEX64 = 16 - DT_COMPLEX128 = 17 - DT_QINT8 = 18 - DT_QINT16 = 19 - DT_QINT32 = 20 - DT_QUINT8 = 21 - DT_QUINT16 = 22 - DT_RESOURCE = 23 - DT_STRING_REF = 24 - DT_DUAL = 25 - DT_UNDEFINED = 26 - - @classmethod - def get_data_type_name(cls, num): - """ - Get the name of data type by enum number. - - Args: - num (int): Enum number. - - Returns: - str, the name of data type. - """ - data_type = cls._value2member_map_.get(num) - return 'UNKNOWN' if data_type is None else data_type.name - - -class GeFormat(enum.IntEnum): - """Definition of ge format type.""" - FORMAT_NCHW = 0 - FORMAT_NHWC = 1 - FORMAT_ND = 2 - FORMAT_NC1HWC0 = 3 - FORMAT_FRACTAL_Z = 4 - FORMAT_NC1C0HWPAD = 5 - FORMAT_NHWC1C0 = 6 - FORMAT_FSR_NCHW = 7 - FORMAT_FRACTAL_DECONV = 8 - FORMAT_C1HWNC0 = 9 - FORMAT_FRACTAL_DECONV_TRANSPOSE = 10 - FORMAT_FRACTAL_DECONV_SP_STRIDE_TRANS = 11 - FORMAT_NC1HWC0_C04 = 12 - FORMAT_FRACTAL_Z_C04 = 13 - FORMAT_CHWN = 14 - FORMAT_FRACTAL_DECONV_SP_STRIDE8_TRANS = 15 - FORMAT_HWCN = 16 - FORMAT_NC1KHKWHWC0 = 17 - FORMAT_BN_WEIGHT = 18 - FORMAT_FILTER_HWCK = 19 - FORMAT_HASHTABLE_LOOKUP_LOOKUPS = 20 - FORMAT_HASHTABLE_LOOKUP_KEYS = 21 - FORMAT_HASHTABLE_LOOKUP_VALUE = 22 - FORMAT_HASHTABLE_LOOKUP_OUTPUT = 23 - FORMAT_HASHTABLE_LOOKUP_HITS = 24 - FORMAT_C1HWNCOC0 = 25 - FORMAT_MD = 26 - FORMAT_NDHWC = 27 - FORMAT_FRACTAL_ZZ = 28 - FORMAT_FRACTAL_NZ = 29 - FORMAT_NCDHW = 30 - FORMAT_DHWCN = 31 - FORMAT_NDC1HWC0 = 32 - FORMAT_FRACTAL_Z_3D = 33 - FORMAT_CN = 34 - FORMAT_NC = 35 - FORMAT_DHWNC = 36 - FORMAT_FRACTAL_Z_3D_TRANSPOSE = 37 - FORMAT_RESERVED = 38 - FORMAT_ALL = 39 - - @classmethod - def get_format_name(cls, num): - """ - Get the name of format type by enum number. - - Args: - num (int): Enum number. - - Returns: - str, the name of format type. - """ - format_type = cls._value2member_map_.get(num) - return 'UNKNOWN' if format_type is None else format_type.name +COL_NAMES = ['task_id', 'stream_id', 'block_dim', 'full_op_name', 'op_name', 'op_type', 'subgraph', 'op_info'] +OpData = namedtuple('OpData', field_names=COL_NAMES) class FrameworkParser: @@ -171,41 +51,30 @@ class FrameworkParser: Thr parser for parsing framework files. Args: - profiling_id (str): The profiling ID. - device_id (str): The device ID. + profiling_path (str): The profiling path which should be contain CANN profiling data. rank_id (str): The rank ID. output_path (str): The directory of the parsed file. Default: `./`. """ _regex_framework = r'Framework\.(?P.+)\.(?P\d).+' - _regex_framework_in_data = r'Framework\.(?P.+)\.' \ - r'(?P\d)\.(?P[a-zA-Z0-9]+).+' - _col_names = [ - 'task_id', 'stream_id', 'block_dim', 'full_op_name', 'op_name', - 'op_type', 'subgraph', 'op_info' - ] _graph_attr_name = [ 'input_format', 'input_data_type', 'input_shape', 'output_format', 'output_data_type', 'output_shape' ] + output_file_format = 'framework_raw_{rank_id}.csv' - # if the task id is less than the task id threshold, The combination of - # task id and Stream id represents one operator, else the task id represents - # one operator - _task_id_threshold = 25000 + def __init__(self, profiling_path, rank_id, output_path='./'): + self._profiling_path = profiling_path + self._output_path = output_path + self._rank_id = rank_id - def __init__(self, profiling_id, device_id, rank_id, output_path='./'): - self._raw_data_dir = output_path - self._profiling_path = self._get_raw_profiling_path(profiling_id) - self._backend_type = None - self._framework_path = {'graph': [], 'task': [], 'point': []} - self._search_file(profiling_id, device_id) - self._device_id = device_id - self._save_path = self._get_save_path(rank_id, output_path) + self._hash_dict = {} self._task_id_full_op_name_dict = {} - self._task_cache = {} self._point_info = {} - self._parse_task_files() - self._parse_point_files() + + @staticmethod + def _check_output_path(path): + if not os.path.exists(path) or not os.path.isdir(path): + raise ProfilerDirNotFoundException(path) @property def save_path(self): @@ -215,7 +84,7 @@ class FrameworkParser: Returns: str, the save path. """ - return self._save_path + return os.path.realpath(os.path.join(self._output_path, self.output_file_format.format(rank_id=self._rank_id))) @property def point_info(self): @@ -223,24 +92,11 @@ class FrameworkParser: The property of the framework point information. Returns: - dict, the framework point information. + dict, the framework point information, key is tag, value is op name. """ + # Note: In the multi-subgraph or multi-tag scenario, op name is overwritten. return self._point_info - def to_task_id_full_op_name_dict(self): - """ - Get the task id and full operator name dict. - - Returns: - dict, the task id and full operator name dict. - """ - return self._task_id_full_op_name_dict - - def parse(self): - """Parse the framework files.""" - self._parse_graph_files_and_save(self._task_cache) - del self._task_cache - def check_op_name(self, op_name, is_prefix=True): """ Check whether the operator name exists. @@ -264,318 +120,281 @@ class FrameworkParser: return True return False - def _get_raw_profiling_path(self, profiling_id): + def to_task_id_full_op_name_dict(self): """ - Get raw profiling path. - - Args: - profiling_id (str): The profiling ID. + Get the task id and full operator name dict. Returns: - str, the raw profiling path. - - Raises: - ProfilerPathErrorException: If the profiling path is invalid. - ProfilerDirNotFoundException: If the profiling dir is not found. + dict, the task id and full operator name dict. """ - profiling_path = os.path.join(self._raw_data_dir, profiling_id) - try: - profiling_path = validate_and_normalize_path(profiling_path) - except RuntimeError: - raise ProfilerPathErrorException('Profiling path is invalid.') - if not os.path.isdir(profiling_path): - raise ProfilerDirNotFoundException(profiling_path) - return profiling_path + return self._task_id_full_op_name_dict - def _search_file(self, profiling_id, device_id): + def parse(self): + """Parse the framework files.""" + framework_path_dict = self._search_file(self._profiling_path) + self._hash_dict = self._parse_hash_dic(framework_path_dict) + + all_file_data = self._parse_binary_data(framework_path_dict) + task_id_full_op_name_dict = self._construct_task_id_full_op_name_dict( + all_file_data[FileDataType.TASK_DESC_INFO.value]) + point_info = self._construct_point_info(task_id_full_op_name_dict, all_file_data[FileDataType.STEP_INFO.value]) + task_id_op_attr_dict = self._construct_task_id_op_attr_dict(all_file_data[FileDataType.TENSOR_DATA_INFO.value]) + + self._point_info = point_info + self._task_id_full_op_name_dict = task_id_full_op_name_dict + + all_op_data = self._construct_op_data_to_file(all_file_data[FileDataType.TASK_DESC_INFO.value], + task_id_op_attr_dict) + + self._write_framework_to_file(all_op_data, output_file=self.save_path) + + def _search_file(self, profiling_path): """ Search all framework files in raw profiling path. Args: - profiling_id (str): The profiling ID. - device_id (str): The device ID. + profiling_path (str): This profiling path should contain data dir. + + Return: + dict, return a dict container all framework file paths. Format is {FileDataType: [file paths]}. Raises: ProfilerFileNotFoundException: If the framework files are not found. """ - # first search in the JOB dir, and if not, search in the sub directory - # in the JOB - self._search_file_from_job_path(device_id, search_in_sub_path=False) - if self._backend_type is None: - self._search_file_from_job_path(device_id, search_in_sub_path=True) - self._search_file_from_data_path(profiling_id, device_id) + data_dir = os.path.join(profiling_path, 'data') + if not os.path.isdir(data_dir): + raise ProfilerDirNotFoundException(data_dir) - if self._backend_type is None: + framework_path_dict = defaultdict(list) + for file in Path(data_dir).glob(r'Framework*[0-9]'): + file_name = file.name + match = re.search(self._regex_framework, file_name) + if match is None: + logger.warning("Profiler does not support to analyse file(%s), this file name format is not %s, " + "skip this file.", file.resolve(), self._regex_framework) + continue + + if match['data_type'] not in FileDataType.members(): + logger.warning("Profiler does not support to analyse file(%s), this file data type is %s, " + "skip this file.", file.resolve(), match['data_type']) + continue + + framework_path_dict[match['data_type']].append(file.resolve()) + + empty_files = [data_type for data_type, files in framework_path_dict.items() if not files] + if not framework_path_dict or empty_files: + if empty_files: + logger.error("Can not find %s files when parse profiler framework file.", ','.join(empty_files)) raise ProfilerFileNotFoundException('Framework') - self._framework_path['graph'].sort() - self._framework_path['task'].sort() - def _search_file_from_job_path(self, device_id, search_in_sub_path=False): - """ - Search framework files from job path. - - Args: - device_id (str): The device ID. - search_in_sub_path (bool): `True` if search file in profiling dir, - else search in profiling sub dir. Default: False. - - Raises: - ProfilerRawFileException: If the framework file type is inconsistent. - ProfilerDeviceIdMismatchException: If the device id is mismatch - with framework in the raw dir. - """ - profiling_dir = os.path.join(self._profiling_path, 'data') \ - if search_in_sub_path else self._profiling_path - if not os.path.isdir(profiling_dir): - return - - files = os.listdir(profiling_dir) - for file in files: - pattern = re.search(self._regex_framework, file) - if not pattern or file.endswith('.done'): + for data_type in FileDataType.members(): + if data_type not in framework_path_dict: + logger.warning("Can not find %s file when parse profiler framework file.", data_type) continue - attrs = pattern.groupdict() + framework_path_dict[data_type].sort() - device_id_in_path = attrs.get('device_id') - if device_id_in_path != device_id: - raise ProfilerDeviceIdMismatchException() + return framework_path_dict - data_type = attrs.get('data_type') - data_type = data_type.replace("host.", "") - if data_type.startswith('vm_'): - if self._backend_type and self._backend_type != 'vm': - raise ProfilerRawFileException('Backend type is inconsistent.') - self._backend_type = 'vm' - _, data_type = data_type.split('_', 1) - else: - if self._backend_type and self._backend_type != 'ge': - raise ProfilerRawFileException('Backend type is inconsistent.') - self._backend_type = 'ge' - if data_type.startswith('graph_desc_info'): - self._framework_path['graph'].append( - os.path.join(profiling_dir, file) - ) - elif data_type.startswith('task_desc_info'): - self._framework_path['task'].append( - os.path.join(profiling_dir, file) - ) - elif data_type.startswith('point'): - self._framework_path['point'].append( - os.path.join(profiling_dir, file) - ) - - def _search_file_from_data_path(self, profiling_id, device_id): - """ - Search framework files from data path. - - Args: - profiling_id (str): The profiling ID. - device_id (str): The device ID. - - Raises: - ProfilerRawFileException: If the framework file type is inconsistent. - ProfilerDeviceIdMismatchException: If the device id is mismatch - with framework in the raw dir. - """ - profiling_data_path = os.path.join( - self._raw_data_dir, 'container', device_id, 'data' - ) - if not os.path.isdir(profiling_data_path): - return - - files = os.listdir(profiling_data_path) - for file in files: - pattern = re.search(self._regex_framework_in_data, file) - if not pattern or file.endswith('.done') or file.endswith('.zip'): - continue - attrs = pattern.groupdict() - - profiling_id_in_path = attrs.get('profiling_id') - if profiling_id_in_path != profiling_id: - continue - - device_id_in_path = attrs.get('device_id') - if device_id_in_path != device_id: - raise ProfilerDeviceIdMismatchException() - - data_type = attrs.get('data_type') - data_type = data_type.replace("host.", "") - if data_type.startswith('vm_'): - if self._backend_type and self._backend_type != 'vm': - raise ProfilerRawFileException('Backend type is inconsistent.') - self._backend_type = 'vm' - _, data_type = data_type.split('_', 1) - else: - if self._backend_type and self._backend_type != 'ge': - raise ProfilerRawFileException('Backend type is inconsistent.') - self._backend_type = 'ge' - if data_type.startswith('graph_desc_info'): - self._framework_path['graph'].append( - os.path.join(profiling_data_path, file) - ) - elif data_type.startswith('task_desc_info'): - self._framework_path['task'].append( - os.path.join(profiling_data_path, file) - ) - elif data_type.startswith('point'): - self._framework_path['point'].append( - os.path.join(profiling_data_path, file) - ) - - def _get_save_path(self, rank_id, output_path): - """ - Get the save path. - - Args: - rank_id (str): The rank ID. - output_path (str): The output dir. - - Returns: - str, the save path. - - Raises: - ProfilerPathErrorException: If the output path is invalid. - ProfilerDirNotFoundException: If the output dir is not found. - """ - try: - output_dir = validate_and_normalize_path(output_path) - except RuntimeError: - raise ProfilerPathErrorException('Output path is invalid.') - if not os.path.isdir(output_dir): - raise ProfilerDirNotFoundException(output_dir) - return os.path.join( - output_dir, '_'.join(['framework', 'raw', rank_id]) + '.csv' - ) - - def _parse_task_files(self): - """Parse the framework task files.""" - for path in self._framework_path['task']: - path = validate_and_normalize_path(path) + @staticmethod + def _parse_hash_dic(framework_path_dict): + """Parse the hash dic files, and return a hash value map op name dict.""" + hash_op_dict = {} + for path in framework_path_dict[FileDataType.HASH_DIC.value]: with open(path, 'r') as file: - for task_info in file: - infos = task_info.strip('\n').split(' ') - infos = infos[1:] if len(infos) == 5 else infos - # key is op name, values is task id, stream id, block_dim - self._task_cache[infos[0]] = [infos[2], infos[3], infos[1]] + for hash_str in file: + hash_value, op_name = hash_str.strip().split(':') + hash_op_dict[hash_value] = op_name + return hash_op_dict - # if the task id is less than the task id threshold, the - # stream id and task id correspond to an operator - task_id = infos[2] - if int(task_id) < self._task_id_threshold: - task_id = '_'.join([infos[3], task_id]) - self._task_id_full_op_name_dict[task_id] = infos[0] + def _parse_binary_data(self, framework_path_dict): + """Parse binary data in the FILE_DATA_STRUCT_DICT from given files, such as task data, step point data""" + all_file_data = defaultdict(list) + for file_data_type, data_struct in FILE_DATA_STRUCT_DICT.items(): + line_size = StructType.sizeof(data_struct.values()) + for path in framework_path_dict[file_data_type]: + with open(path, 'rb') as file_handler: + while True: + line = file_handler.read(line_size) + if len(line) < line_size: + break + line_data = self._transform_binary_data(data_struct, line) + all_file_data[file_data_type].append(line_data) + return all_file_data - def _parse_graph_files_and_save(self, task_cache): + def _transform_binary_data(self, data_struct, binary_data): """ - Parse the framework graph files and save the framework information. + Parse the binary data to get the actual data - Args: - task_cache (dict): The task information cache. - """ - with open(self._save_path, 'w') as save_file: - csv_writer = csv.writer(save_file) - csv_writer.writerow(self._col_names) - pre_graph_info = None - for path in self._framework_path['graph']: - first_row = True - with open(path, 'r') as graph_file: - for graph_info in graph_file: - if first_row is True: - first_row = False - # The last row of the previous file and the first row of the current file may need - # to be combined to one row - if graph_info.startswith("op_name:") is False: - pre_graph_info = pre_graph_info + graph_info - continue - if pre_graph_info is not None: - self._parse_graph_row_and_save(task_cache, csv_writer, pre_graph_info) - pre_graph_info = graph_info - - if pre_graph_info is not None: - self._parse_graph_row_and_save(task_cache, csv_writer, pre_graph_info) - - none_list = [None, None, None, None] - for key, value in task_cache.items(): - value.append(key) - value.extend(none_list) - csv_writer.writerow(value) - os.chmod(self._save_path, stat.S_IREAD | stat.S_IWRITE) - - def _parse_graph_row_and_save(self, task_cache, csv_writer, graph_info): - """ - Parse the framework graph row and save the framework information. - - Args: - task_cache (dict): The task information cache. - csv_writer (csv): Csv writer. - graph_info (str): Row info of graph. - """ - result = self._parse_one_row_graph_info(graph_info) - task_info = task_cache.get(result[0]) - if task_info: - task_info.extend(result) - csv_writer.writerow(task_info) - del task_cache[result[0]] - else: - save_info = [None, None, None] - save_info.extend(result) - csv_writer.writerow(save_info) - - def _parse_one_row_graph_info(self, row_info): - """ - Parse the graph information in one row. - - Args: - row_info (str): One row graph information. + Args： + data_struct (dict): Key is the data name, value is StructType. + binary_data (str): This value should be a binary string. Returns: - list[str], the parsed graph information. + dict, key is data name, value is a actual value. + + Example: + >>> ret = self._transform_binary_data({'op_name': StructType.UINT32}, b'1101') + >>> print(ret) + ... {'op_name': (825241905,)} + """ - full_op_name = None - op_name = None - subgraph_name = None - op_type = None - op_info = dict() - cur_op_info_key = None - - infos = row_info.strip('\n').split(' ') - for info in infos: - attr_name, attr_value = info.split(':', 1) - if attr_name == 'op_name': - full_op_name = attr_value - subgraph_name = self._get_subgraph_name(full_op_name) - op_name = self._get_op_name(full_op_name, subgraph_name) - elif attr_name == 'op_type': - op_type = attr_value - elif attr_name in ['input_id', 'output_id']: - cur_op_info_key = '{}_{}'.format( - attr_name.split('_')[0], attr_value - ) - op_info[cur_op_info_key] = dict() - elif attr_name in self._graph_attr_name: - op_attr = attr_name.split('_', 1)[1] - if op_attr == 'shape': - attr_value = attr_value.strip('"') - if self._backend_type == 'vm': - if op_attr == 'data_type': - attr_value = VmDataType.get_data_type_name( - int(attr_value) - ) + actual_data = {} + cursor = 0 + for name, data_type in data_struct.items(): + data_size = StructType.sizeof(data_type) + if isinstance(data_type, list): + if name in ('opName', 'opType'): + unpack_data = self._special_process_mixed_data(binary_data, cursor, data_size) + elif name == 'tensorData': + tensor_num = actual_data['tensorNum'] + unpack_data = self._special_process_tensor_data(cursor, data_type, binary_data, tensor_num) + elif name == 'tensorNum': + unpack_data = self._speceal_process_tensor_num(data_type, binary_data, cursor) else: - if op_attr == 'data_type': - attr_value = GeDataType.get_data_type_name( - int(attr_value) - ) - elif op_attr == 'format': - attr_value = GeFormat.get_format_name(int(attr_value)) + # skip reserve data + unpack_data = None - op_info[cur_op_info_key][op_attr] = attr_value + actual_data[name] = unpack_data + cursor += data_size + continue - # the list info are full_op_name, op_name, op_type, subgraph, op_info - return [full_op_name, op_name, op_type, subgraph_name, - json.dumps(op_info)] + unpack_data = struct.unpack(data_type.value, binary_data[cursor: cursor+data_size])[0] + cursor += data_size + actual_data[name] = unpack_data + return actual_data - def _get_subgraph_name(self, full_op_name): + def _special_process_mixed_data(self, binary_data, cursor, data_size): + """Specially processes mixed data, for example, opName and opType""" + # The first byte is type flag, 0 means data is string, 1 means data is hash value + flag = struct.unpack(StructType.UCHAR.value, binary_data[cursor:cursor + 1])[0] + + # skip rsv data, rsv has 7 bytes + skip_size = 8 + remain_size = data_size - skip_size + if flag == 0: + unpack_data = struct.unpack(StructType.CHAR.value * remain_size, + binary_data[cursor + skip_size:cursor + data_size]) + try: + unpack_data = ''.join(list(map(lambda c: c.decode(), + filter(lambda c: c != b'\x00', unpack_data)))) + except Exception as exc: + raise exc + else: + size = StructType.sizeof(StructType.UINT64) + skip_size + hash_value = struct.unpack(StructType.UINT64.value, + binary_data[cursor + skip_size:cursor + size])[0] + unpack_data = self._hash_dict[str(hash_value)] + return unpack_data + + @staticmethod + def _special_process_tensor_data(cursor, data_type, binary_data, tensor_num): + """The tensor data depends tensor num, so need to special process.""" + start = cursor + op_attr_struct = data_type[0] + op_attr_size = StructType.sizeof(op_attr_struct) + unpack_data = [] + + for _ in range(tensor_num): + buffer = binary_data[start:start + op_attr_size] + values = struct.unpack(StructType.format(op_attr_struct), buffer) + one_data = dict( + tensorType=values[0], + format=values[1], + dataType=values[2], + shape=list(filter(lambda x: x != 0, values[3:])) + ) + unpack_data.append(one_data) + start += op_attr_size + + return unpack_data + + @staticmethod + def _speceal_process_tensor_num(data_type, binary_data, cursor): + """The memory of tensorNum is aligned, so here need to special process""" + tensor_num_struct = data_type[0] + size = StructType.sizeof(tensor_num_struct) + unpack_data = struct.unpack(tensor_num_struct.value, binary_data[cursor:cursor + size])[0] + return unpack_data + + def _construct_task_id_full_op_name_dict(self, task_desc_info): + """The task desc info is a list[task_desc], task_desc is a dict, key is same as TASK_DESC_STRUCT.""" + task_id_full_op_name = {} + for task_desc in task_desc_info: + task_id = self._combine_stream_task_id(task_desc['streamId'], task_desc['taskId']) + task_id_full_op_name[task_id] = task_desc['opName'] + return task_id_full_op_name + + def _construct_point_info(self, task_id_full_op_name_dict, step_point_data): + """step_point_data is a list[step_data], step data is a dict, key is same as STEP_INFO_STRUCT.""" + point_info = {} + for step_point in step_point_data: + task_id = self._combine_stream_task_id(step_point['streamId'], step_point['taskId']) + tag = step_point['tag'] + full_op_name = task_id_full_op_name_dict[task_id] + point_info[tag] = full_op_name + return point_info + + def _construct_task_id_op_attr_dict(self, prof_tensor_data): + """prof_tensor_data is a list[tensor_data], tensor_data is a dict, key is same as TENSOR_DATA_STRUCT.""" + task_id_op_attr_dict = defaultdict(list) + for tensor_data in prof_tensor_data: + task_id = self._combine_stream_task_id(tensor_data['streamId'], tensor_data['taskId']) + for tensor_attr in tensor_data['tensorData']: + tensor_type = 'input' if tensor_attr['tensorType'] == 0 else 'output' + tensor_format = VmFormat.get_format_name(tensor_attr['format']) + op_attr = dict( + tensor_type=tensor_type, + format=tensor_format, + data_type=VmDataType.get_data_type_name(tensor_attr['dataType']), + shape=tensor_attr['shape'] + ) + task_id_op_attr_dict[task_id].append(op_attr) + + for task_id, op_attrs in task_id_op_attr_dict.items(): + input_count = 0 + output_count = 0 + new_op_attr = {} + for op_attr in op_attrs: + if op_attr['tensor_type'] == 'input': + op_attr.pop('tensor_type') + new_op_attr[f'input_{input_count}'] = op_attr + input_count += 1 + else: + op_attr.pop('tensor_type') + new_op_attr[f'output_{output_count}'] = op_attr + output_count += 1 + task_id_op_attr_dict[task_id] = new_op_attr + + return task_id_op_attr_dict + + def _construct_op_data_to_file(self, task_desc_info, task_id_op_attr_dict): + """Build data written to a file.""" + all_op_data = [] + for task_desc in task_desc_info: + task_id = task_desc['taskId'] + full_op_name = task_desc['opName'] + subgraph = self._get_subgraph_name(full_op_name) + combined_task_id = self._combine_stream_task_id(task_desc['streamId'], task_id) + op_data = OpData(task_id=task_id, + stream_id=task_desc['streamId'], + block_dim=task_desc['blockDims'], + full_op_name=full_op_name, + op_name=full_op_name.split('/')[-1], + op_type=task_desc['opType'], + subgraph=subgraph, + op_info=json.dumps(task_id_op_attr_dict[combined_task_id])) + all_op_data.append(op_data) + return all_op_data + + @staticmethod + def _write_framework_to_file(all_op_data: List[OpData], output_file): + with open(output_file, 'w') as file_handler: + csv_writer = csv.writer(file_handler) + csv_writer.writerow(COL_NAMES) + csv_writer.writerows(all_op_data) + + @staticmethod + def _get_subgraph_name(full_op_name): """ Get subgraph name. @@ -590,39 +409,16 @@ class FrameworkParser: return subgraph_name return None - def _get_op_name(self, full_op_name, subgraph_name): + @staticmethod + def _combine_stream_task_id(stream_id, task_id): """ - Get operator name. - - Args: - full_op_name (str): The full operator name. - subgraph_name (str): The subgraph name. - - Returns: - str, the operator name. + When the Task ID is less than the threshold, it will be reuse in different streams, + only the stream ID and task ID combined are unique values. """ - if subgraph_name is None: - return full_op_name - - if self._backend_type == 'vm': - return full_op_name.split('/')[-1] - - strs = full_op_name.split(subgraph_name + '/') - op_name = None - for name_str in strs: - if not name_str: - continue - if op_name is None: - op_name = name_str.split('/')[-1] - else: - op_name = '+'.join([op_name, name_str.split('/')[-1]]) - return op_name - - def _parse_point_files(self): - """Parse the framework point files.""" - for path in self._framework_path['point']: - path = validate_and_normalize_path(path) - with open(path, 'r') as file: - for point_info in file: - infos = point_info.strip('\n').split(' ') - self._point_info[int(infos[0])] = infos[1] + # if the task id is less than the task id threshold, The combination of + # task id and Stream id represents one operator, else the task id represents + # one operator + task_id_threshold = 25000 + if task_id < task_id_threshold: + return f'{stream_id}_{task_id}' + return str(task_id) diff --git a/mindspore/python/mindspore/profiler/parser/framework_struct.py b/mindspore/python/mindspore/profiler/parser/framework_struct.py new file mode 100644 index 00000000000..b2030b28f61 --- /dev/null +++ b/mindspore/python/mindspore/profiler/parser/framework_struct.py @@ -0,0 +1,61 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Thr parser for parsing framework files.""" + +from mindspore.profiler.common.struct_type import StructType + +# Note: All keys should be named with lower camel case, which are the same as those in C++. + +TASK_DESC_STRUCT = dict( + magicNumber=StructType.UINT16, + dataTag=StructType.UINT16, + taskType=StructType.UINT32, + opName=[StructType.UINT64] * 16, # opName is a mix data + opType=[StructType.UINT64] * 8, # opType is a mix data + curIterNum=StructType.UINT64, + timeStamp=StructType.UINT64, + shapeType=StructType.UINT32, + blockDims=StructType.UINT32, + modelId=StructType.UINT32, + streamId=StructType.UINT32, + taskId=StructType.UINT32, + threadId=StructType.UINT32, + reserve=[StructType.UCHAR] * 16 +) + +STEP_INFO_STRUCT = dict( + magicNumber=StructType.UINT16, + dataTag=StructType.UINT16, + modelId=StructType.UINT32, + streamId=StructType.UINT32, + taskId=StructType.UINT32, + timeStamp=StructType.UINT64, + curIterNum=StructType.UINT64, + threadId=StructType.UINT32, + tag=StructType.UCHAR, + reserve=[StructType.UCHAR] * 27 +) + +TENSOR_DATA_STRUCT = dict( + magicNumber=StructType.UINT16, + dataTag=StructType.UINT16, + modelId=StructType.UINT32, + curIterNum=StructType.UINT64, + streamId=StructType.UINT32, + taskId=StructType.UINT32, + tensorNum=[StructType.UCHAR] * 4, # Note: Here the memory is aligned. The actual memory usage is 1, 3 is padding. + tensorData=[[StructType.UINT32] * 11] * 5, + reserve=[StructType.UCHAR] * 8 # Note: Here the memory is aligned. The actual memory usage is 4, 4 is padding. +) diff --git a/mindspore/python/mindspore/profiler/parser/integrator.py b/mindspore/python/mindspore/profiler/parser/integrator.py index afe55821d55..64c197b1c62 100644 --- a/mindspore/python/mindspore/profiler/parser/integrator.py +++ b/mindspore/python/mindspore/profiler/parser/integrator.py @@ -1434,6 +1434,11 @@ class AscendTimelineGenerator(BaseTimelineGenerator): if start_time > step_end_time: per_step_time_list.append(cur_step_metric_time) step += 1 + if step >= len(step_time_list): + logger.warning("Compute profiler compute_time_inside_step time, " + "find the data length is more than step count, " + "maybe current graph has multi sub graph, skip the last data.") + break step_end_time = step_time_list[step][self._start_time_idx] + \ step_time_list[step][self._duration_idx] cur_step_metric_time = 0 diff --git a/mindspore/python/mindspore/profiler/profiling.py b/mindspore/python/mindspore/profiler/profiling.py index 045da203dec..539d438c7a7 100644 --- a/mindspore/python/mindspore/profiler/profiling.py +++ b/mindspore/python/mindspore/profiler/profiling.py @@ -25,6 +25,8 @@ import mindspore._c_expression as c_expression import mindspore._c_dataengine as cde from mindspore.profiler.common.exceptions.exceptions import ProfilerFileNotFoundException, \ ProfilerIOException, ProfilerException, ProfilerRawFileException +from mindspore.profiler.common.exceptions.exceptions import ProfilerPathErrorException +from mindspore.profiler.common.exceptions.exceptions import ProfilerDirNotFoundException from mindspore.profiler.common.util import get_file_path, fwrite_format from mindspore.profiler.common.validator.validate_path import \ validate_and_normalize_path @@ -299,6 +301,8 @@ class Profiler: job_id = self._get_profiling_job_id() logger.info("Profiling: job id is %s ", job_id) + self._check_output_path(output_path=self._output_path) + source_path = os.path.join(self._output_path, job_id) # parse hwts.log.data.45.dev file, and get task profiling data hwts_output_filename = self._hwts_output_filename_target + self._rank_id + ".txt" @@ -310,7 +314,7 @@ class Profiler: hwtslog_parser.execute() # parse Framework file, and get the relation of op and tasks - framework_parser = FrameworkParser(job_id, self._dev_id, self._rank_id, self._output_path) + framework_parser = FrameworkParser(source_path, self._rank_id, self._output_path) logger.info("Profiling: analyzing framework data.") framework_parser.parse() op_task_dict = framework_parser.to_task_id_full_op_name_dict() @@ -403,6 +407,16 @@ class Profiler: logger.info("Profiling: analyzing the operation FLOPs.") flops_parser.execute() + @staticmethod + def _check_output_path(output_path): + try: + output_path = validate_and_normalize_path(output_path) + except RuntimeError: + raise ProfilerPathErrorException('Output path is invalid.') + if not os.path.isdir(output_path): + raise ProfilerDirNotFoundException(output_path) + return output_path + def start(self): """ Used for Ascend, GPU, start profiling. Profiling can be turned on based on step and epoch. @@ -669,7 +683,6 @@ class Profiler: Returns: str, profiling job id. """ - job_id = "" job_dirs = filter(lambda item: item.startswith('JOB') or item.startswith('PROF') and \ os.path.isdir(os.path.join(self._output_path, item)), @@ -699,6 +712,9 @@ class Profiler: "profiler will ignore this job dir.", job_dir, self._dev_id, training_device_id) continue + if not os.listdir(os.path.join(job_dir, 'data')): + continue + job_start_time = self._parse_host_start_log(host_start_file_path) if not job_start_time: logger.warning("Find profiling job path %s, but fail to get job start info, " diff --git a/tests/ut/data/profiler_data/JOB1/Framework.host.vm_graph_desc_info.0.slice_0 b/tests/ut/data/profiler_data/JOB1/Framework.host.vm_graph_desc_info.0.slice_0 deleted file mode 100644 index 9b3e7b322c5..00000000000 --- a/tests/ut/data/profiler_data/JOB1/Framework.host.vm_graph_desc_info.0.slice_0 +++ /dev/null @@ -1,4 +0,0 @@ -op_name:Default/Cast-op6 op_type:Cast input_id:0 input_format:DefaultFormat input_data_type:40 input_shape:"32,3,224,224" output_id:0 output_format:DefaultFormat output_data_type:39 output_shape:"32,3,224,224" -op_name:Default/TransData-op7 op_type:TransData input_id:0 input_format:DefaultFormat input_data_type:39 input_shape:"32,3,224,224" output_id:0 output_format:NC1HWC0 output_data_type:39 output_shape:"32,1,224,224,16" -op_name:Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5 op_type:Cast input_id:0 input_format:FracZ input_data_type:40 input_shape:"49,4,16,16" output_id:0 output_format:FracZ output_data_type:39 output_shape:"49,4,16,16" -op_name:Default/network-WithLossCell/_backbone-ResNet/layer1-SequentialCell/0-ResidualBlock/conv1-Conv2d/Cast-op28 op_type:Cast input_id:0 input_format:FracZ input_data_type:40 input_shape:"4,4,16,16" output_id:0 output_format:FracZ output_data_type:39 output_shape:"4,4,16,16" diff --git a/tests/ut/data/profiler_data/JOB1/Framework.host.vm_graph_desc_info.0.slice_0.done b/tests/ut/data/profiler_data/JOB1/Framework.host.vm_graph_desc_info.0.slice_0.done deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/ut/data/profiler_data/JOB1/Framework.host.vm_task_desc_info.0.slice_0 b/tests/ut/data/profiler_data/JOB1/Framework.host.vm_task_desc_info.0.slice_0 deleted file mode 100644 index e786b35e22b..00000000000 --- a/tests/ut/data/profiler_data/JOB1/Framework.host.vm_task_desc_info.0.slice_0 +++ /dev/null @@ -1,4 +0,0 @@ -Default/Cast-op6 32 51517 0 -Default/TransData-op7 32 51518 0 -Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5 32 51519 0 -Default/network-WithLossCell/_backbone-ResNet/layer1-SequentialCell/0-ResidualBlock/conv1-Conv2d/Cast-op28 4 51522 0 diff --git a/tests/ut/data/profiler_data/JOB4/data/Framework.host.vm_graph_desc_info.0.slice_0 b/tests/ut/data/profiler_data/JOB4/data/Framework.host.vm_graph_desc_info.0.slice_0 deleted file mode 100644 index 9b3e7b322c5..00000000000 --- a/tests/ut/data/profiler_data/JOB4/data/Framework.host.vm_graph_desc_info.0.slice_0 +++ /dev/null @@ -1,4 +0,0 @@ -op_name:Default/Cast-op6 op_type:Cast input_id:0 input_format:DefaultFormat input_data_type:40 input_shape:"32,3,224,224" output_id:0 output_format:DefaultFormat output_data_type:39 output_shape:"32,3,224,224" -op_name:Default/TransData-op7 op_type:TransData input_id:0 input_format:DefaultFormat input_data_type:39 input_shape:"32,3,224,224" output_id:0 output_format:NC1HWC0 output_data_type:39 output_shape:"32,1,224,224,16" -op_name:Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5 op_type:Cast input_id:0 input_format:FracZ input_data_type:40 input_shape:"49,4,16,16" output_id:0 output_format:FracZ output_data_type:39 output_shape:"49,4,16,16" -op_name:Default/network-WithLossCell/_backbone-ResNet/layer1-SequentialCell/0-ResidualBlock/conv1-Conv2d/Cast-op28 op_type:Cast input_id:0 input_format:FracZ input_data_type:40 input_shape:"4,4,16,16" output_id:0 output_format:FracZ output_data_type:39 output_shape:"4,4,16,16" diff --git a/tests/ut/data/profiler_data/JOB4/data/Framework.host.vm_point.0.slice_0 b/tests/ut/data/profiler_data/JOB4/data/Framework.host.vm_point.0.slice_0 deleted file mode 100644 index 01bcf6f3ca6..00000000000 --- a/tests/ut/data/profiler_data/JOB4/data/Framework.host.vm_point.0.slice_0 +++ /dev/null @@ -1,2 +0,0 @@ -1 Default/Cast-op6 -2 Default/TransData-op7 diff --git a/tests/ut/data/profiler_data/JOB4/data/Framework.host.vm_task_desc_info.0.slice_0 b/tests/ut/data/profiler_data/JOB4/data/Framework.host.vm_task_desc_info.0.slice_0 deleted file mode 100644 index e49673789f9..00000000000 --- a/tests/ut/data/profiler_data/JOB4/data/Framework.host.vm_task_desc_info.0.slice_0 +++ /dev/null @@ -1,4 +0,0 @@ -Default/Cast-op6 32 1 0 -Default/TransData-op7 32 2 0 -Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5 32 3 0 -Default/network-WithLossCell/_backbone-ResNet/layer1-SequentialCell/0-ResidualBlock/conv1-Conv2d/Cast-op28 4 4 0 diff --git a/tests/ut/data/profiler_data/framework/data/Framework.hash_dic.6.slice_0 b/tests/ut/data/profiler_data/framework/data/Framework.hash_dic.6.slice_0 new file mode 100644 index 00000000000..06f030ae241 --- /dev/null +++ b/tests/ut/data/profiler_data/framework/data/Framework.hash_dic.6.slice_0 @@ -0,0 +1,6 @@ +221281922719854948:Gradients/Default/network-WithLossCell/_loss_fn-SoftmaxCrossEntropyWithLogits/gradSparseSoftmaxCrossEntropyWithLogits/OneHot-op183 +11321138686081159177:Gradients/Default/network-WithLossCell/_loss_fn-SoftmaxCrossEntropyWithLogits/gradSparseSoftmaxCrossEntropyWithLogits/SoftmaxCrossEntropyWithLogits-op184 +9987411238351573275:Gradients/Default/network-WithLossCell/_loss_fn-SoftmaxCrossEntropyWithLogits/gradSparseSoftmaxCrossEntropyWithLogits/ReduceMean-op186 +18188948999212452788:Gradients/Default/network-WithLossCell/_loss_fn-SoftmaxCrossEntropyWithLogits/gradSparseSoftmaxCrossEntropyWithLogits/Tile-op188 +4352827225462011144:Gradients/Default/network-WithLossCell/_loss_fn-SoftmaxCrossEntropyWithLogits/gradSparseSoftmaxCrossEntropyWithLogits/RealDiv-op189 +12251689346898190624:Gradients/Default/network-WithLossCell/_loss_fn-SoftmaxCrossEntropyWithLogits/gradSparseSoftmaxCrossEntropyWithLogits/Mul-op191 diff --git a/tests/ut/data/profiler_data/framework/data/Framework.step_info.6.slice_0 b/tests/ut/data/profiler_data/framework/data/Framework.step_info.6.slice_0 new file mode 100644 index 00000000000..21b7b310c9e Binary files /dev/null and b/tests/ut/data/profiler_data/framework/data/Framework.step_info.6.slice_0 differ diff --git a/tests/ut/data/profiler_data/framework/data/Framework.task_desc_info.6.slice_0 b/tests/ut/data/profiler_data/framework/data/Framework.task_desc_info.6.slice_0 new file mode 100644 index 00000000000..492270e3f67 Binary files /dev/null and b/tests/ut/data/profiler_data/framework/data/Framework.task_desc_info.6.slice_0 differ diff --git a/tests/ut/data/profiler_data/framework/data/Framework.tensor_data_info.6.slice_0 b/tests/ut/data/profiler_data/framework/data/Framework.tensor_data_info.6.slice_0 new file mode 100644 index 00000000000..0a900db50dc Binary files /dev/null and b/tests/ut/data/profiler_data/framework/data/Framework.tensor_data_info.6.slice_0 differ diff --git a/tests/ut/data/profiler_data/framework/expected/task_id_full_op_name.json b/tests/ut/data/profiler_data/framework/expected/task_id_full_op_name.json new file mode 100644 index 00000000000..5549296c5be --- /dev/null +++ b/tests/ut/data/profiler_data/framework/expected/task_id_full_op_name.json @@ -0,0 +1 @@ +{"1_4": "Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Add-op123", "1_5": "Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Sub-op110", "1_6": "Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Mul-op118", "1_7": "Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Maximum-op119", "1_8": "Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LessEqual-op111", "1_9": "Gradients/Default/network-WithLossCell/_loss_fn-SoftmaxCrossEntropyWithLogits/gradSparseSoftmaxCrossEntropyWithLogits/OneHot-op183", "1_10": "Default/NPUAllocFloatStatus-op18", "1_11": "Default/network-WithLossCell/_backbone-LeNet5/fc3-Dense/Cast-op41", "1_12": "Default/AtomicAddrClean-op472", "1_13": "Default/network-WithLossCell/_backbone-LeNet5/fc3-Dense/TransData-op314", "1_14": "Default/network-WithLossCell/_backbone-LeNet5/fc3-Dense/Cast-op34", "1_15": "Default/AtomicAddrClean-op473", "1_16": "Default/network-WithLossCell/_backbone-LeNet5/fc3-Dense/TransData-op320", "1_17": "Default/network-WithLossCell/_backbone-LeNet5/fc3-Dense/Cast-op48", "1_18": "Default/AtomicAddrClean-op474", "1_19": "Default/network-WithLossCell/_backbone-LeNet5/fc3-Dense/TransData-op273", "1_20": "Default/network-WithLossCell/_backbone-LeNet5/Cast-op20", "1_21": "Default/network-WithLossCell/_backbone-LeNet5/TransData-op337", "1_22": "Default/network-WithLossCell/_backbone-LeNet5/conv2-Conv2d/Cast-op28", "1_23": "Default/network-WithLossCell/_backbone-LeNet5/conv2-Conv2d/TransData-op344", "1_24": "Default/network-WithLossCell/_backbone-LeNet5/conv1-Conv2d/Cast-op23", "1_25": "Default/AtomicAddrClean-op475", "1_26": "Default/network-WithLossCell/_backbone-LeNet5/conv1-Conv2d/TransData-op338", "1_27": "Default/network-WithLossCell/_backbone-LeNet5/relu-ReLU/FusionOp_Conv2D_ReLUV2-op392", "1_28": "Default/AtomicAddrClean-op476", "1_29": "Default/network-WithLossCell/_backbone-LeNet5/max_pool2d-MaxPool2d/MaxPoolWithArgmax-op167", "1_30": "Default/network-WithLossCell/_backbone-LeNet5/relu-ReLU/FusionOp_Conv2D_ReLUV2-op393", "1_31": "Default/AtomicAddrClean-op477", "1_32": "Default/network-WithLossCell/_backbone-LeNet5/max_pool2d-MaxPool2d/MaxPoolWithArgmax-op169", "1_33": "Default/network-WithLossCell/_backbone-LeNet5/max_pool2d-MaxPool2d/TransData-op311", "1_34": "Default/network-WithLossCell/_backbone-LeNet5/flatten-Flatten/TransData-op319", "1_35": "Default/network-WithLossCell/_backbone-LeNet5/fc3-Dense/Cast-op37", "1_36": "Default/network-WithLossCell/_backbone-LeNet5/relu-ReLU/FusionOp_MatMul_ReLU-op383", "1_37": "Default/network-WithLossCell/_backbone-LeNet5/fc3-Dense/Cast-op44", "1_38": "Default/network-WithLossCell/_backbone-LeNet5/relu-ReLU/FusionOp_MatMul_ReLU-op382", "1_39": "Default/network-WithLossCell/_backbone-LeNet5/fc3-Dense/Cast-op51", "1_40": "Default/network-WithLossCell/_backbone-LeNet5/fc3-Dense/MatMul-op229", "1_41": "Default/network-WithLossCell/_backbone-LeNet5/fc3-Dense/TransData-op274", "1_42": "Default/network-WithLossCell/Cast-op52", "1_43": "Gradients/Default/network-WithLossCell/_loss_fn-SoftmaxCrossEntropyWithLogits/gradSparseSoftmaxCrossEntropyWithLogits/SoftmaxCrossEntropyWithLogits-op184", "1_44": "Default/AtomicAddrClean-op478", "1_45": "Gradients/Default/network-WithLossCell/_loss_fn-SoftmaxCrossEntropyWithLogits/gradSparseSoftmaxCrossEntropyWithLogits/ReduceMean-op186", "1_46": "Default/NPUClearFloatStatus-op56", "1_47": "Default/Reciprocal-op74", "1_48": "Gradients/Default/network-WithLossCell/_loss_fn-SoftmaxCrossEntropyWithLogits/gradSparseSoftmaxCrossEntropyWithLogits/Tile-op188", "1_49": "Gradients/Default/network-WithLossCell/_loss_fn-SoftmaxCrossEntropyWithLogits/gradSparseSoftmaxCrossEntropyWithLogits/RealDiv-op189", "1_50": "Gradients/Default/network-WithLossCell/_loss_fn-SoftmaxCrossEntropyWithLogits/gradSparseSoftmaxCrossEntropyWithLogits/Mul-op191", "1_51": "Gradients/Default/network-WithLossCell/gradCast/Cast-op60", "1_52": "Gradients/Default/network-WithLossCell/_backbone-LeNet5/fc3-Dense/gradBiasAdd/BiasAddGrad-op96", "1_53": "Gradients/Default/network-WithLossCell/gradCast/TransData-op303", "1_54": "Gradients/Default/network-WithLossCell/_backbone-LeNet5/gradCast/Cast-op97", "1_55": "Default/FusionOp_MatMul_Mul-op388", "1_56": "Gradients/Default/network-WithLossCell/_backbone-LeNet5/fc3-Dense/gradMatMul/MatMul-op61", "1_57": "Default/Mul-op75", "1_58": "Default/TransData-op265", "1_59": "Gradients/Default/network-WithLossCell/_backbone-LeNet5/relu-ReLU/gradReLU/ReluGrad-op62", "1_60": "Default/FusionOp_MatMul_Mul-op389", "1_61": "Gradients/Default/network-WithLossCell/_backbone-LeNet5/fc3-Dense/gradBiasAdd/BiasAddGrad-op90", "1_62": "Gradients/Default/network-WithLossCell/_backbone-LeNet5/fc3-Dense/gradMatMul/MatMul-op63", "1_63": "Default/TransData-op263", "1_64": "Gradients/Default/network-WithLossCell/_backbone-LeNet5/gradCast/Cast-op91", "1_65": "Gradients/Default/network-WithLossCell/_backbone-LeNet5/relu-ReLU/gradReLU/ReluGrad-op64", "1_66": "Default/Mul-op92", "1_67": "Default/FusionOp_MatMul_Mul-op387", "1_68": "Gradients/Default/network-WithLossCell/_backbone-LeNet5/fc3-Dense/gradBiasAdd/BiasAddGrad-op84", "1_69": "Default/TransData-op261", "1_70": "Gradients/Default/network-WithLossCell/_backbone-LeNet5/gradCast/Cast-op85", "1_71": "Default/Mul-op86", "1_72": "Gradients/Default/network-WithLossCell/_backbone-LeNet5/fc3-Dense/gradMatMul/MatMul-op65", "1_73": "Gradients/Default/network-WithLossCell/_backbone-LeNet5/fc3-Dense/gradMatMul/TransData-op342", "1_74": "Gradients/Default/network-WithLossCell/_backbone-LeNet5/flatten-Flatten/gradReshape/TransData-op329", "1_75": "Default/AtomicAddrClean-op479", "1_76": "Gradients/Default/network-WithLossCell/_backbone-LeNet5/max_pool2d-MaxPool2d/gradMaxPool/MaxPoolGradWithArgmax-op172", "1_77": "Gradients/Default/network-WithLossCell/_backbone-LeNet5/relu-ReLU/gradReLU/ReluGradV2-op235", "1_78": "Default/AtomicAddrClean-op480", "1_79": "Gradients/Default/network-WithLossCell/_backbone-LeNet5/conv2-Conv2d/gradConv2D/Conv2DBackpropFilter-op78", "1_80": "Gradients/Default/network-WithLossCell/_backbone-LeNet5/conv2-Conv2d/gradConv2D/Conv2DBackpropInput-op69", "1_81": "Default/Mul-op80", "1_82": "Default/AtomicAddrClean-op481", "1_83": "Gradients/Default/network-WithLossCell/_backbone-LeNet5/max_pool2d-MaxPool2d/gradMaxPool/MaxPoolGradWithArgmax-op174", "1_84": "Gradients/Default/network-WithLossCell/_backbone-LeNet5/relu-ReLU/gradReLU/ReluGradV2-op237", "1_85": "Default/AtomicAddrClean-op482", "1_86": "Gradients/Default/network-WithLossCell/_backbone-LeNet5/conv1-Conv2d/gradConv2D/Conv2DBackpropFilter-op72", "1_87": "Default/Mul-op76", "1_88": "Default/NPUGetFloatStatus-op99", "1_89": "Default/AtomicAddrClean-op483", "1_90": "Default/ReduceSum-op100", "1_91": "Default/LessEqual-op101", "1_92": "Default/LogicalNot-op102", "1_93": "Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op120", "1_94": "Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LogicalOr-op112", "1_95": "Default/Cast-op376", "1_96": "Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LogicalAnd-op116", "1_97": "Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Mul-op121", "1_98": "Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op113", "1_99": "Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op117", "1_100": "Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Assign-op122", "1_101": "Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Assign-op114", "1_102": "Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Assign-op125", "1_103": "Default/TransData-op257", "1_104": "Default/Assign-op207", "1_105": "Default/AtomicAddrClean-op484", "1_106": "Default/TransData-op254", "1_107": "Default/Assign-op206", "1_108": "Default/TransData-op378", "1_109": "Default/TransData-op377", "1_112": "Default/optimizer-Momentum/ApplyMomentum-op131", "1_113": "Default/optimizer-Momentum/ApplyMomentum-op133", "1_114": "Default/optimizer-Momentum/ApplyMomentum-op135", "1_115": "Default/optimizer-Momentum/ApplyMomentum-op137", "1_116": "Default/optimizer-Momentum/ApplyMomentum-op139", "1_117": "Default/optimizer-Momentum/ApplyMomentum-op141", "1_118": "Default/optimizer-Momentum/ApplyMomentum-op143", "1_119": "Default/optimizer-Momentum/ApplyMomentum-op145", "1_120": "Default/Assign-op240", "1_121": "Default/Assign-op239", "1_122": "Default/Assign-op238", "1_127": "Default/Assign-op247", "1_128": "Default/Assign-op246", "1_129": "Default/Assign-op245"} \ No newline at end of file diff --git a/tests/ut/python/profiler/__init__.py b/tests/ut/python/profiler/__init__.py index 586589132ce..0b7e93c5c4e 100644 --- a/tests/ut/python/profiler/__init__.py +++ b/tests/ut/python/profiler/__init__.py @@ -18,4 +18,5 @@ import os RAW_DATA_BASE = os.path.realpath(os.path.join(os.path.dirname(__file__), '../../data/profiler_data')) RAW_DATA = os.path.realpath(os.path.join(RAW_DATA_BASE, 'JOB1')) RAW_DATA_JOB2 = os.path.realpath(os.path.join(RAW_DATA_BASE, 'JOB2')) +FRAMEWORK_RAW_DATA = os.path.realpath(os.path.join(RAW_DATA_BASE, 'framework')) PROFILER_DIR = os.path.realpath(os.path.join(RAW_DATA_BASE, 'profiler')) diff --git a/tests/ut/python/profiler/parser/test_framework_parser.py b/tests/ut/python/profiler/parser/test_framework_parser.py index aaef79c18a8..702dacea8c1 100644 --- a/tests/ut/python/profiler/parser/test_framework_parser.py +++ b/tests/ut/python/profiler/parser/test_framework_parser.py @@ -15,16 +15,10 @@ """Test the framework parser module.""" import csv import os -import shutil +import json import tempfile -from unittest import mock - -import pytest - -from mindspore.profiler.common.exceptions.exceptions import \ - ProfilerFileNotFoundException from mindspore.profiler.parser.framework_parser import FrameworkParser -from tests.ut.python.profiler import PROFILER_DIR, RAW_DATA_BASE +from tests.ut.python.profiler import FRAMEWORK_RAW_DATA def get_framework_result(file_path): @@ -47,84 +41,36 @@ def get_framework_result(file_path): class TestFrameworkParser: """Test the class of `FrameworkParser`.""" - def setup_method(self): + def setup_class(self): """Initialization before test case execution.""" - self._output_path_1 = tempfile.NamedTemporaryFile(prefix='test_framework_parser_').name - shutil.copytree(RAW_DATA_BASE, self._output_path_1) - self._parser_1 = FrameworkParser('JOB1', '0', '0', self._output_path_1) - self._output_path_2 = tempfile.NamedTemporaryFile(prefix='test_framework_parser_').name - shutil.copytree(RAW_DATA_BASE, self._output_path_2) - self._parser_2 = FrameworkParser('JOB2', '0', '0', self._output_path_2) - self._output_path_4 = tempfile.NamedTemporaryFile(prefix='test_framework_parser_').name - shutil.copytree(RAW_DATA_BASE, self._output_path_4) - self._parser_4 = FrameworkParser('JOB4', '0', '0', self._output_path_4) + self._profiling_path = os.path.realpath(FRAMEWORK_RAW_DATA) - def teardown_method(self) -> None: - """Clear up after test case execution.""" - shutil.rmtree(self._output_path_1) - shutil.rmtree(self._output_path_2) - shutil.rmtree(self._output_path_4) - - def test_save_path(self): + def test_format_save_path(self): """Test the querying save path function.""" - expect_result = os.path.join(self._output_path_1, 'framework_raw_0.csv') - assert expect_result == self._parser_1.save_path - - expect_result = os.path.join(self._output_path_2, 'framework_raw_0.csv') - assert expect_result == self._parser_2.save_path - - def test_point_info(self): - """Test the querying point info function.""" - expect_result = { - 1: 'Default/Cast-op6', - 2: 'Default/TransData-op7' - } - assert expect_result == self._parser_4.point_info - - def test_to_task_id_full_op_name_dict(self): - """Test the querying task id and full operator name dict function.""" - expect_result = { - '51517': 'Default/Cast-op6', - '51518': 'Default/TransData-op7', - '51519': 'Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5', - '51522': 'Default/network-WithLossCell/_backbone-ResNet/' - 'layer1-SequentialCell/0-ResidualBlock/conv1-Conv2d/Cast-op28' - } - assert expect_result == self._parser_1.to_task_id_full_op_name_dict() - assert expect_result == self._parser_2.to_task_id_full_op_name_dict() - - expect_result = { - '0_1': 'Default/Cast-op6', - '0_2': 'Default/TransData-op7', - '0_3': 'Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5', - '0_4': 'Default/network-WithLossCell/_backbone-ResNet/layer1-SequentialCell/' - '0-ResidualBlock/conv1-Conv2d/Cast-op28' - } - assert expect_result == self._parser_4.to_task_id_full_op_name_dict() + output_path = os.path.realpath('./') + rank_id = '2' + parser = FrameworkParser(self._profiling_path, rank_id, output_path) + expect_result = os.path.join(output_path, FrameworkParser.output_file_format.format(rank_id=rank_id)) + assert parser.save_path == expect_result def test_parse(self): """Test the parse function.""" - expect_framework_file = os.path.join(PROFILER_DIR, 'framework_raw_0.csv') - expect_framework_file = os.path.realpath(expect_framework_file) - expect_result = get_framework_result(expect_framework_file) + with tempfile.TemporaryDirectory() as output_path: + parser = FrameworkParser(self._profiling_path, '1', output_path) + parser.parse() + assert os.path.exists(parser.save_path) - self._parser_1.parse() - framework_file = os.path.join(self._output_path_1, 'framework_raw_0.csv') - result = get_framework_result(framework_file) - assert expect_result == result + expected_point = {2: "Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Add-op123", + 3: "Default/Assign-op245", + 4: "Default/Assign-op245"} + assert parser.point_info == expected_point - self._parser_2.parse() - framework_file = os.path.join(self._output_path_2, 'framework_raw_0.csv') - result = get_framework_result(framework_file) - assert expect_result == result + task_id_full_op_name_file = os.path.join(self._profiling_path, 'expected', 'task_id_full_op_name.json') + self._compare_json(task_id_full_op_name_file, parser.to_task_id_full_op_name_dict()) - @mock.patch('os.listdir') - @mock.patch('os.path.isdir') - def test_create_framework_parser_fail_1(self, *args): - """Test the function of fail to create framework parser.""" - args[0].return_value = True - args[1].return_value = [] - with pytest.raises(ProfilerFileNotFoundException) as exc_info: - FrameworkParser('JOB1', '0', '0') - assert exc_info.value.error_code == '50546084' - assert exc_info.value.message == 'The file not found.' + @staticmethod + def _compare_json(expected_file, result_dict): + with open(expected_file, 'r') as file_handler: + expected_data = json.load(file_handler) + + assert expected_data == result_dict