From 94f28a2da34057d58e445bfa33269b52048a24b7 Mon Sep 17 00:00:00 2001 From: gzhcv Date: Thu, 24 Jun 2021 11:29:15 +0800 Subject: [PATCH] Fit Gpu LoopCount for profiler module v2 --- mindspore/ccsrc/profiler/CMakeLists.txt | 8 +++ .../profiler/device/cpu/cpu_data_saver.cc | 6 +++ .../profiler/device/cpu/cpu_data_saver.h | 7 +++ .../profiler/device/cpu/cpu_profiling.cc | 21 ++++++-- .../ccsrc/profiler/device/cpu/cpu_profiling.h | 3 ++ .../profiler/device/gpu/gpu_data_saver.cc | 51 ++++++++++++++++-- .../profiler/device/gpu/gpu_data_saver.h | 3 ++ .../profiler/device/gpu/gpu_profiling.cc | 37 ++++--------- .../ccsrc/profiler/device/gpu/gpu_profiling.h | 5 +- .../device/gpu/gpu_profiling_utils.cc | 9 +++- mindspore/ccsrc/profiler/device/profiling.cc | 52 ++++++++++++++++++- mindspore/ccsrc/profiler/device/profiling.h | 19 ++++++- .../runtime/framework/actor/recorder_actor.cc | 8 +-- .../runtime/framework/actor/recorder_actor.h | 4 +- .../runtime/framework/graph_scheduler.cc | 20 ++++++- .../ccsrc/runtime/framework/graph_scheduler.h | 3 ++ mindspore/profiler/parser/integrator.py | 20 +++---- 17 files changed, 210 insertions(+), 66 deletions(-) diff --git a/mindspore/ccsrc/profiler/CMakeLists.txt b/mindspore/ccsrc/profiler/CMakeLists.txt index f119ed62b2..bbe9a8d597 100644 --- a/mindspore/ccsrc/profiler/CMakeLists.txt +++ b/mindspore/ccsrc/profiler/CMakeLists.txt @@ -21,4 +21,12 @@ if(ENABLE_CPU AND NOT (ENABLE_D OR ENABLE_GPU)) set_property(SOURCE ${PROFILER_SRC_LIST} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_PROFILER) add_library(_mindspore_profiler_obj OBJECT ${PROFILER_SRC_LIST}) +endif() + +if(ENABLE_TESTCASES) + file(GLOB_RECURSE PROFILER_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "device/profiling.cc") + set_property(SOURCE ${PROFILER_SRC_LIST} PROPERTY COMPILE_DEFINITIONS + SUBMODULE_ID=mindspore::SubModuleId::SM_PROFILER) + add_library(_mindspore_profiler_obj OBJECT ${PROFILER_SRC_LIST}) endif() \ No newline at end of file diff --git a/mindspore/ccsrc/profiler/device/cpu/cpu_data_saver.cc b/mindspore/ccsrc/profiler/device/cpu/cpu_data_saver.cc index 3a9392918c..d3549cdc58 100644 --- a/mindspore/ccsrc/profiler/device/cpu/cpu_data_saver.cc +++ b/mindspore/ccsrc/profiler/device/cpu/cpu_data_saver.cc @@ -24,6 +24,8 @@ namespace mindspore { namespace profiler { namespace cpu { +std::shared_ptr CpuDataSaver::cpu_data_saver_inst_ = std::make_shared(); + void CpuDataSaver::WriteFile(const std::string out_path_dir) { if (op_detail_infos_.empty() || op_type_infos_.empty()) { MS_LOG(INFO) << "No cpu operation detail infos to write."; @@ -38,6 +40,10 @@ void CpuDataSaver::WriteFile(const std::string out_path_dir) { WriteOpType(out_path_dir); WriteOpTimestamp(out_path_dir); } + +OpTimestampInfo &CpuDataSaver::GetOpTimeStampInfo() { return op_timestamps_map_; } + +std::shared_ptr &CpuDataSaver::GetInstance() { return cpu_data_saver_inst_; } } // namespace cpu } // namespace profiler } // namespace mindspore diff --git a/mindspore/ccsrc/profiler/device/cpu/cpu_data_saver.h b/mindspore/ccsrc/profiler/device/cpu/cpu_data_saver.h index d9442f9158..3fdc091f0c 100644 --- a/mindspore/ccsrc/profiler/device/cpu/cpu_data_saver.h +++ b/mindspore/ccsrc/profiler/device/cpu/cpu_data_saver.h @@ -29,6 +29,8 @@ namespace profiler { namespace cpu { class CpuDataSaver : public DataSaver { public: + static std::shared_ptr &GetInstance(); + CpuDataSaver() = default; ~CpuDataSaver() = default; @@ -37,7 +39,12 @@ class CpuDataSaver : public DataSaver { CpuDataSaver &operator=(const CpuDataSaver &) = delete; + OpTimestampInfo &GetOpTimeStampInfo(); + void WriteFile(const std::string out_path); + + private: + static std::shared_ptr cpu_data_saver_inst_; }; } // namespace cpu } // namespace profiler diff --git a/mindspore/ccsrc/profiler/device/cpu/cpu_profiling.cc b/mindspore/ccsrc/profiler/device/cpu/cpu_profiling.cc index 5b268ea565..97a6f7d1f3 100644 --- a/mindspore/ccsrc/profiler/device/cpu/cpu_profiling.cc +++ b/mindspore/ccsrc/profiler/device/cpu/cpu_profiling.cc @@ -23,6 +23,7 @@ #include "pybind_api/api_register.h" #include "utils/log_adapter.h" #include "utils/utils.h" +#include "utils/ms_context.h" namespace mindspore { namespace profiler { @@ -62,13 +63,24 @@ void CPUProfiler::OpDataProducerBegin(const std::string op_name, const uint32_t op_time_start_ = GetHostMonoTimeStamp(); op_time_mono_start_ = GetHostMonoTimeStamp(); SetRunTimeData(op_name, pid); + +#if ENABLE_GPU + if (MsContext::GetInstance()->get_param(MS_CTX_ENABLE_MINDRT)) { + // For heterogeneous scene, record op name to gpu_profiler_inst. + auto gpu_profiler_inst = profiler::gpu::GPUProfiler::GetInstance(); + // For cpu network, no gpu profiler, do not to raise exception. + if (gpu_profiler_inst && gpu_profiler_inst->GetEnableFlag()) { + gpu_profiler_inst->RecordOneStepStartEndInfo(op_name); + } + } +#endif } void CPUProfiler::OpDataProducerEnd() { float op_time_elapsed = 0; op_time_stop_ = GetHostMonoTimeStamp(); op_time_elapsed = (op_time_stop_ - op_time_start_) / kNanosecondToMillisecond; - MS_LOG(DEBUG) << "Host Time Elapsed(us)," << op_name_ << "," << op_time_elapsed; + MS_LOG(DEBUG) << "Host Time Elapsed(ms)," << op_name_ << "," << op_time_elapsed; Profiler::SetRunTimeData(op_name_, op_time_elapsed); Profiler::SetRunTimeData(op_name_, op_time_mono_start_, op_time_elapsed); } @@ -83,9 +95,10 @@ void CPUProfiler::SaveProfileData() { if (profile_data_path_.empty()) { MS_LOG(WARNING) << "Profile data path is empty, skip save profile data."; } else { - CpuDataSaver dataSaver; - dataSaver.ParseOpInfo(op_info_map_); - dataSaver.WriteFile(profile_data_path_); + auto cpu_data_saver_inst = profiler::cpu::CpuDataSaver::GetInstance(); + MS_EXCEPTION_IF_NULL(cpu_data_saver_inst); + cpu_data_saver_inst->ParseOpInfo(op_info_map_); + cpu_data_saver_inst->WriteFile(profile_data_path_); } } diff --git a/mindspore/ccsrc/profiler/device/cpu/cpu_profiling.h b/mindspore/ccsrc/profiler/device/cpu/cpu_profiling.h index d647150eac..71504551c6 100644 --- a/mindspore/ccsrc/profiler/device/cpu/cpu_profiling.h +++ b/mindspore/ccsrc/profiler/device/cpu/cpu_profiling.h @@ -24,6 +24,9 @@ #include #include #include "profiler/device/profiling.h" +#if ENABLE_GPU +#include "profiler/device/gpu/gpu_profiling.h" +#endif namespace mindspore { namespace profiler { diff --git a/mindspore/ccsrc/profiler/device/gpu/gpu_data_saver.cc b/mindspore/ccsrc/profiler/device/gpu/gpu_data_saver.cc index fa9fa5229b..48619e6220 100644 --- a/mindspore/ccsrc/profiler/device/gpu/gpu_data_saver.cc +++ b/mindspore/ccsrc/profiler/device/gpu/gpu_data_saver.cc @@ -105,6 +105,22 @@ void GpuDataSaver::AddKernelEventToDevice(const Event &event, DeviceActivityInfo } } +void GpuDataSaver::CpuProfilingTimeSynchronizedToGpu(const BaseTime &start_time) { + auto cpu_data_saver_inst = profiler::cpu::CpuDataSaver::GetInstance(); + MS_EXCEPTION_IF_NULL(cpu_data_saver_inst); + auto &cpu_op_timestamps_map = cpu_data_saver_inst->GetOpTimeStampInfo(); + auto cpu_op_iter = cpu_op_timestamps_map.begin(); + while (cpu_op_iter != cpu_op_timestamps_map.end()) { + for (auto &time_iter : cpu_op_iter->second) { + time_iter.start_timestamp = + time_iter.start_timestamp - start_time.host_start_monotonic_raw_time + start_time.gpu_start_time; + // time unit from ms to us. + time_iter.duration *= kTimeUnit; + } + cpu_op_iter++; + } +} + void GpuDataSaver::WriteFile(std::string out_path_dir, const BaseTime &start_time) { if (out_path_dir.empty()) { MS_LOG(WARNING) << "Output directory. Ignore the writing data."; @@ -122,6 +138,7 @@ void GpuDataSaver::WriteFile(std::string out_path_dir, const BaseTime &start_tim WriteActivity(out_path_dir); WriteOpTimestamp(out_path_dir); WriteStartTime(out_path_dir, start_time); + CpuProfilingTimeSynchronizedToGpu(start_time); if (MsContext::GetInstance()->get_param(MS_CTX_ENABLE_MINDRT)) { WriteStepTraceAsyncLaunchKernel(out_path_dir); } else { @@ -176,6 +193,11 @@ void GpuDataSaver::WriteStepTraceAsyncLaunchKernel(const std::string &saver_base return; } + // cpu profiler information. + auto cpu_data_saver_inst = profiler::cpu::CpuDataSaver::GetInstance(); + MS_EXCEPTION_IF_NULL(cpu_data_saver_inst); + auto &cpu_op_timestamps_map = cpu_data_saver_inst->GetOpTimeStampInfo(); + // write step trace time info into file uint32_t step = 0; uint64_t duration; @@ -188,17 +210,32 @@ void GpuDataSaver::WriteStepTraceAsyncLaunchKernel(const std::string &saver_base auto bp_end_op_timestamp = op_timestamps_map_.find(step_trace_op_name_.trace_bp_end); auto iter_end_op_timestamp = op_timestamps_map_.find(iter_end_op_name); - if (iter_end_op_name == "Default/InitDataSetQueue-op0") continue; + // if iter_start/fp_start/iter_end op is executed on cpu, update it. + if (iter_start_op_timestamp == op_timestamps_map_.end()) { + iter_start_op_timestamp = cpu_op_timestamps_map.find(iter_start_op_name); + } + if (fp_op_timestamp == op_timestamps_map_.end()) { + fp_op_timestamp = cpu_op_timestamps_map.find(fp_op_name); + } + if (iter_end_op_timestamp == op_timestamps_map_.end()) { + iter_end_op_timestamp = cpu_op_timestamps_map.find(iter_end_op_name); + } + + if (iter_end_op_name == "Default/InitDataSetQueue-op0") { + continue; + } if (iter_start_op_timestamp == op_timestamps_map_.end() || fp_op_timestamp == op_timestamps_map_.end() || iter_end_op_timestamp == op_timestamps_map_.end() || bp_end_op_timestamp == op_timestamps_map_.end()) { - MS_LOG(ERROR) << "[profiling step trace] failed, do not find " << fp_op_name << " or " << iter_end_op_name << "or" - << step_trace_op_name_.trace_bp_end; + MS_LOG(ERROR) << "[profiling step trace] failed, do not find \"" << fp_op_name << "\" or \"" << iter_end_op_name + << "\" or \"" << step_trace_op_name_.trace_bp_end << "\""; + ofs.close(); return; } if (iter_start_op_timestamp->second.size() <= step || fp_op_timestamp->second.size() <= step || iter_end_op_timestamp->second.size() <= step || bp_end_op_timestamp->second.size() <= step) { MS_LOG(ERROR) << "[profiling step trace] the number of fp/bp/iter_end timestamp not enough"; + ofs.close(); return; } @@ -215,13 +252,15 @@ void GpuDataSaver::WriteStepTraceAsyncLaunchKernel(const std::string &saver_base // convert the time unit from 1ns to 10ns (keep the same with ascend) auto iter_op_timestamp = op_timestamps_map_.find(op_name); if (iter_op_timestamp == op_timestamps_map_.end()) { - MS_LOG(ERROR) << "[profiling step trace] failed, do not find " << fp_op_name << " or " << iter_end_op_name - << "or" << step_trace_op_name_.trace_bp_end; + MS_LOG(ERROR) << "[profiling step trace] failed, do not find \"" << fp_op_name << "\" or " << iter_end_op_name + << "\" or \"" << step_trace_op_name_.trace_bp_end << "\""; + ofs.close(); return; } if (iter_op_timestamp->second.size() <= step) { MS_LOG(ERROR) << "[profiling step trace] the number of communication op timestamp not enough"; + ofs.close(); return; } @@ -278,6 +317,7 @@ void GpuDataSaver::WriteStepTrace(const std::string &saver_base_dir) { ofs << std::endl; } catch (const std::exception &e) { MS_LOG(ERROR) << "Write " << file_path << "failed:" << e.what(); + ofs.close(); } } } @@ -302,6 +342,7 @@ void GpuDataSaver::WriteStartTime(const std::string &saver_base_dir, const BaseT ofs << "gpu_start_time(ns): " << start_time.gpu_start_time << std::endl; } catch (const std::exception &e) { MS_LOG(ERROR) << "Write " << file_path << "failed:" << e.what(); + ofs.close(); } ofs.close(); diff --git a/mindspore/ccsrc/profiler/device/gpu/gpu_data_saver.h b/mindspore/ccsrc/profiler/device/gpu/gpu_data_saver.h index 94b39b6401..8477e36bf8 100644 --- a/mindspore/ccsrc/profiler/device/gpu/gpu_data_saver.h +++ b/mindspore/ccsrc/profiler/device/gpu/gpu_data_saver.h @@ -23,6 +23,7 @@ #include #include #include "profiler/device/gpu/gpu_profiling.h" +#include "profiler/device/cpu/cpu_data_saver.h" #include "profiler/device/data_saver.h" namespace mindspore { namespace profiler { @@ -93,6 +94,8 @@ class GpuDataSaver : public DataSaver { void WriteStartTime(const std::string &saver_base_dir, const BaseTime &start_time); + void CpuProfilingTimeSynchronizedToGpu(const BaseTime &start_time); + AllActivityInfos activity_infos_; ProfilingTraceInfo step_trace_op_name_from_graph_; ProfilingTraceInfo step_trace_op_name_; diff --git a/mindspore/ccsrc/profiler/device/gpu/gpu_profiling.cc b/mindspore/ccsrc/profiler/device/gpu/gpu_profiling.cc index b3d01cf1d5..ffa1d513c1 100644 --- a/mindspore/ccsrc/profiler/device/gpu/gpu_profiling.cc +++ b/mindspore/ccsrc/profiler/device/gpu/gpu_profiling.cc @@ -211,7 +211,10 @@ std::string GetKernelFuncName(std::string kernel_name) { return kernel_name.substr(func_name_begin_iter); } -std::shared_ptr &GPUProfiler::GetInstance() { return profiler_inst_; } +std::shared_ptr &GPUProfiler::GetInstance() { + MS_EXCEPTION_IF_NULL(profiler_inst_); + return profiler_inst_; +} void GPUProfiler::SyncEnable(const bool enable_flag) { MS_LOG(INFO) << "GPU Profiler synchronous enable flag:" << enable_flag; @@ -434,7 +437,9 @@ void GPUProfiler::OpDataProducerBegin(const std::string op_name, void *stream) { } SetRunTimeData(op_name, stream); - if (MsContext::GetInstance()->get_param(MS_CTX_ENABLE_MINDRT)) RecordOneStepStartEndInfo(op_name); + if (MsContext::GetInstance()->get_param(MS_CTX_ENABLE_MINDRT)) { + RecordOneStepStartEndInfo(op_name); + } } void GPUProfiler::OpDataProducerEnd() { @@ -490,7 +495,7 @@ void GPUProfiler::SaveProfileData() { if (profile_data_path_.empty()) { MS_LOG(WARNING) << "Profile data path is empty, skip save profile data."; } else { - GpuDataSaver dataSaver(step_trace_op_name, all_step_start_end_info_); + GpuDataSaver dataSaver(step_trace_op_name_, all_step_start_end_info_); dataSaver.ParseOpInfo(op_info_map_); dataSaver.ParseEvent(events_); dataSaver.WriteFile(profile_data_path_, base_time_); @@ -498,30 +503,6 @@ void GPUProfiler::SaveProfileData() { } } -void GPUProfiler::RecordOneStepStartEndInfo() { - std::lock_guard locker(record_mutex_); - step_start_end_info_.iter_end_timestamp = GetCUPTITimeStamp(); - all_step_start_end_info_.push_back(step_start_end_info_); - step_start_end_info_.iter_start_op_name = ""; - step_start_end_info_.fp_start_op_name = ""; -} - -void GPUProfiler::RecordOneStepStartEndInfo(const std::string op_name) { - std::lock_guard locker(record_mutex_); - if (step_start_end_info_.iter_start_op_name.empty()) { - step_start_end_info_.iter_start_op_name = op_name; - step_start_end_info_.fp_start_op_name = op_name; - } - - std::string fp_start_op_name = step_start_end_info_.fp_start_op_name; - - auto op_type_begin_iter = fp_start_op_name.rfind('/') + 1; - auto op_type_end_iter = fp_start_op_name.rfind('-'); - auto op_type = fp_start_op_name.substr(op_type_begin_iter, op_type_end_iter - op_type_begin_iter); - if (op_type == "InitDataSetQueue" || op_type == "GetNext") step_start_end_info_.fp_start_op_name = op_name; - step_start_end_info_.iter_end_op_name = op_name; -} - void GPUProfiler::ClearInst() { op_info_map_.clear(); op_name_map_.clear(); @@ -701,7 +682,7 @@ void GPUProfiler::HandleActivityRecord(CUpti_Activity *record) { AddEvent(std::move(profilingData)); } -void GPUProfiler::SetStepTraceOpName(ProfilingTraceInfo trace_op_name) { step_trace_op_name = trace_op_name; } +void GPUProfiler::SetStepTraceOpName(ProfilingTraceInfo trace_op_name) { step_trace_op_name_ = trace_op_name; } void GPUProfiler::RegisterProfilingOp(std::shared_ptr node) { PROFILER_ERROR_IF_NULLPTR(node); diff --git a/mindspore/ccsrc/profiler/device/gpu/gpu_profiling.h b/mindspore/ccsrc/profiler/device/gpu/gpu_profiling.h index 2ae1140f33..17fdd71b93 100644 --- a/mindspore/ccsrc/profiler/device/gpu/gpu_profiling.h +++ b/mindspore/ccsrc/profiler/device/gpu/gpu_profiling.h @@ -132,7 +132,6 @@ class GPUProfiler : public Profiler { void ProcessEvents(); void RegisterProfilingOp(std::shared_ptr node); void SetStepTraceOpName(ProfilingTraceInfo trace_op_name); - void RecordOneStepStartEndInfo(); std::string ProfileDataPath() const { return profile_data_path_; } private: @@ -143,7 +142,6 @@ class GPUProfiler : public Profiler { void AddEvent(Event &&event); void SetRunTimeData(const std::string &op_name, void *stream); void FixOpNameByCorrelationId(Event *event); - void RecordOneStepStartEndInfo(std::string op_name); static std::shared_ptr profiler_inst_; bool enable_flag_ = false; @@ -175,8 +173,7 @@ class GPUProfiler : public Profiler { uint64_t op_cupti_time_start_; std::string profile_data_path_; std::map> profiling_op_; - ProfilingTraceInfo step_trace_op_name; - std::mutex record_mutex_; + ProfilingTraceInfo step_trace_op_name_; }; } // namespace gpu } // namespace profiler diff --git a/mindspore/ccsrc/profiler/device/gpu/gpu_profiling_utils.cc b/mindspore/ccsrc/profiler/device/gpu/gpu_profiling_utils.cc index a5e2e9a837..b955c53c52 100644 --- a/mindspore/ccsrc/profiler/device/gpu/gpu_profiling_utils.cc +++ b/mindspore/ccsrc/profiler/device/gpu/gpu_profiling_utils.cc @@ -41,6 +41,7 @@ ProfilingTraceInfo ProfilingUtils::GetProfilingTraceFromEnv(NotNullgraph_id()] = false; - return profiling_trace; + + // If current graph has only one node, the bp_end will be empty, so select the last graph node. + if (profiling_trace.trace_bp_end != "") { + return profiling_trace; + } else { + return last_graph_profiling_trace; + } } void ProfilingUtils::OutputStepTraceOpNameStatus() { diff --git a/mindspore/ccsrc/profiler/device/profiling.cc b/mindspore/ccsrc/profiler/device/profiling.cc index e466b102df..5acc03f089 100644 --- a/mindspore/ccsrc/profiler/device/profiling.cc +++ b/mindspore/ccsrc/profiler/device/profiling.cc @@ -19,13 +19,17 @@ #include #include #include -#include "profiler/device/cpu/cpu_data_saver.h" #include "pybind_api/api_register.h" #include "utils/log_adapter.h" #include "utils/utils.h" +#if ENABLE_GPU +#include "profiler/device/gpu/gpu_profiling.h" +#endif namespace mindspore { namespace profiler { +std::shared_ptr ProfilerManager::profiler_manager_inst_ = std::make_shared(); + uint64_t Profiler::GetHostMonoTimeStamp() const { struct timespec ts; #if defined(_WIN32) || defined(_WIN64) @@ -57,5 +61,51 @@ void Profiler::SetRunTimeData(const std::string &op_name, const uint64_t start, iter->second.start_duration.emplace_back(StartDuration({start, duration})); } } + +void Profiler::RecordOneStepStartEndInfo() { + std::lock_guard locker(record_mutex_); + all_step_start_end_info_.push_back(step_start_end_info_); + step_start_end_info_.iter_start_op_name = ""; + step_start_end_info_.fp_start_op_name = ""; +} + +void Profiler::RecordOneStepStartEndInfo(const std::string op_name) { + std::lock_guard locker(record_mutex_); + if (step_start_end_info_.iter_start_op_name.empty()) { + step_start_end_info_.iter_start_op_name = op_name; + step_start_end_info_.fp_start_op_name = op_name; + } + + std::string fp_start_op_name = step_start_end_info_.fp_start_op_name; + + auto op_type_begin_iter = fp_start_op_name.rfind('/') + 1; + auto op_type_end_iter = fp_start_op_name.rfind('-'); + auto op_type = fp_start_op_name.substr(op_type_begin_iter, op_type_end_iter - op_type_begin_iter); + if (op_type == "InitDataSetQueue" || op_type == "GetNext") { + step_start_end_info_.fp_start_op_name = op_name; + } + step_start_end_info_.iter_end_op_name = op_name; +} + +std::shared_ptr &ProfilerManager::GetInstance() { + MS_EXCEPTION_IF_NULL(profiler_manager_inst_); + return profiler_manager_inst_; +} + +bool ProfilerManager::GetEnableRecorderActorFlag() { +#if ENABLE_GPU + return profiler::gpu::GPUProfiler::GetInstance()->GetEnableFlag(); +#endif + return false; +} + +void ProfilerManager::RecordOneStepStartEndInfo() { +#if ENABLE_GPU + auto gpu_profiler_inst = profiler::gpu::GPUProfiler::GetInstance(); + if (gpu_profiler_inst->GetEnableFlag()) { + gpu_profiler_inst->RecordOneStepStartEndInfo(); + } +#endif +} } // namespace profiler } // namespace mindspore diff --git a/mindspore/ccsrc/profiler/device/profiling.h b/mindspore/ccsrc/profiler/device/profiling.h index 643ecf9396..6b3dd23676 100644 --- a/mindspore/ccsrc/profiler/device/profiling.h +++ b/mindspore/ccsrc/profiler/device/profiling.h @@ -37,8 +37,6 @@ struct OneStepStartEndInfo { std::string iter_start_op_name; std::string fp_start_op_name; std::string iter_end_op_name; - uint64_t fp_start_timestamp = 0l; - uint64_t iter_end_timestamp = 0l; }; struct OpInfo { @@ -54,6 +52,20 @@ struct OpInfo { uint32_t pid; }; +class ProfilerManager { + public: + static std::shared_ptr &GetInstance(); + ProfilerManager() = default; + ~ProfilerManager() = default; + ProfilerManager(const ProfilerManager &) = delete; + ProfilerManager &operator=(const ProfilerManager &) = delete; + bool GetEnableRecorderActorFlag(); + void RecordOneStepStartEndInfo(); + + private: + static std::shared_ptr profiler_manager_inst_; +}; + class Profiler { public: Profiler() = default; @@ -63,8 +75,10 @@ class Profiler { virtual void Stop() = 0; virtual void StepProfilingEnable(const bool enable_flag) = 0; virtual void OpDataProducerEnd() = 0; + void RecordOneStepStartEndInfo(); bool GetEnableFlag() const { return enable_flag_; } std::string ProfileDataPath() const { return profile_data_path_; } + void RecordOneStepStartEndInfo(std::string op_name); protected: void SetRunTimeData(const std::string &op_name, const float time_elapsed); @@ -77,6 +91,7 @@ class Profiler { std::unordered_map op_info_map_; OneStepStartEndInfo step_start_end_info_; std::vector all_step_start_end_info_; + std::mutex record_mutex_; }; } // namespace profiler } // namespace mindspore diff --git a/mindspore/ccsrc/runtime/framework/actor/recorder_actor.cc b/mindspore/ccsrc/runtime/framework/actor/recorder_actor.cc index fb54e94ad3..3a027bad01 100644 --- a/mindspore/ccsrc/runtime/framework/actor/recorder_actor.cc +++ b/mindspore/ccsrc/runtime/framework/actor/recorder_actor.cc @@ -65,12 +65,8 @@ void RecorderActor::RecordInfo(const std::string op_name, const KernelLaunchInfo void RecorderActor::RecordOnStepEnd(OpContext *op_context) { MS_EXCEPTION_IF_NULL(op_context); // todo clear -#if ENABLE_GPU - // Record fp_start and iter_end op name and timestamp at the step end. (GPU) - auto profiler_inst = profiler::gpu::GPUProfiler::GetInstance(); - MS_EXCEPTION_IF_NULL(profiler_inst); - if (profiler_inst->GetEnableFlag()) profiler_inst->RecordOneStepStartEndInfo(); -#endif + // Record iter_start, fp_start and iter_end op name and timestamp at the step end. (GPU) + profiler::ProfilerManager::GetInstance()->RecordOneStepStartEndInfo(); } } // namespace runtime diff --git a/mindspore/ccsrc/runtime/framework/actor/recorder_actor.h b/mindspore/ccsrc/runtime/framework/actor/recorder_actor.h index 6fef41849f..c57c5361f1 100644 --- a/mindspore/ccsrc/runtime/framework/actor/recorder_actor.h +++ b/mindspore/ccsrc/runtime/framework/actor/recorder_actor.h @@ -22,9 +22,7 @@ #include "runtime/framework/actor/actor_common.h" #include "runtime/framework/device_tensor_store.h" #include "runtime/hardware/device_context.h" -#if ENABLE_GPU -#include "profiler/device/gpu/gpu_profiling.h" -#endif +#include "profiler/device/profiling.h" namespace mindspore { namespace runtime { diff --git a/mindspore/ccsrc/runtime/framework/graph_scheduler.cc b/mindspore/ccsrc/runtime/framework/graph_scheduler.cc index 0849c4fb22..42e8d7c424 100644 --- a/mindspore/ccsrc/runtime/framework/graph_scheduler.cc +++ b/mindspore/ccsrc/runtime/framework/graph_scheduler.cc @@ -35,6 +35,7 @@ #ifdef ENABLE_DEBUGGER #include "debug/debugger/debugger.h" #endif +#include "profiler/device/profiling.h" namespace mindspore { namespace runtime { @@ -456,6 +457,13 @@ void GraphScheduler::Initialize() { << ", the computed OMP thread number : " << OMP_thread_num << ", the used OMP thread number : " << stoi(OMP_thread_num_used); + BuildAndScheduleGlobalActor(); +} + +void GraphScheduler::BuildAndScheduleGlobalActor() { + auto actorMgr = ActorMgr::GetActorMgrRef(); + MS_EXCEPTION_IF_NULL(actorMgr); + // Create and schedule memory manager actor. auto memory_manager_actor = std::make_shared(); MS_EXCEPTION_IF_NULL(memory_manager_actor); @@ -465,9 +473,17 @@ void GraphScheduler::Initialize() { // Bind single thread to response to memory alloc and free quickly. (void)actorMgr->Spawn(base_actor, false); -// Create and schedule recorder actor. + // Create and schedule recorder actor. + bool recorder_actor_need = false; + if (profiler::ProfilerManager::GetInstance()->GetEnableRecorderActorFlag()) { + recorder_actor_need = true; + } #ifdef ENABLE_DUMP_IR if (mindspore::RecorderManager::Instance().RdrEnable()) { + recorder_actor_need = true; + } +#endif + if (recorder_actor_need) { auto recorder_actor = std::make_shared(); MS_EXCEPTION_IF_NULL(recorder_actor); recorder_aid_ = &(recorder_actor->GetAID()); @@ -475,7 +491,7 @@ void GraphScheduler::Initialize() { base_recorder_actor->set_thread_pool(thread_pool_); (void)actorMgr->Spawn(base_recorder_actor, true); } -#endif + // Create and schedule debug actor. bool debugger_actor_need = DumpJsonParser::GetInstance().e2e_dump_enabled(); #ifdef ENABLE_DEBUGGER diff --git a/mindspore/ccsrc/runtime/framework/graph_scheduler.h b/mindspore/ccsrc/runtime/framework/graph_scheduler.h index f6904939b7..f2313ee0a1 100644 --- a/mindspore/ccsrc/runtime/framework/graph_scheduler.h +++ b/mindspore/ccsrc/runtime/framework/graph_scheduler.h @@ -170,6 +170,9 @@ class GraphScheduler { ~GraphScheduler() = default; DISABLE_COPY_AND_ASSIGN(GraphScheduler); + // The Global actors contain memory manager actor, recorder actor and debug actor. + void BuildAndScheduleGlobalActor(); + // Transform the nodes of graph to actors. ActorSetPtr Build(const GraphCompilerInfo &graph_compiler_info); // Link actors to DAG through the edge connection of graph and graph execution strategy. diff --git a/mindspore/profiler/parser/integrator.py b/mindspore/profiler/parser/integrator.py index ebbb67f488..472441a254 100644 --- a/mindspore/profiler/parser/integrator.py +++ b/mindspore/profiler/parser/integrator.py @@ -561,7 +561,7 @@ class BaseTimelineGenerator: os.chmod(display_file_path, stat.S_IREAD | stat.S_IWRITE) except (IOError, OSError) as err: logger.error('Error occurred when write timeline display file: %s', err) - raise ProfilerIOException + raise ProfilerIOException() def write_timeline_summary(self): """Write timeline summary to json.""" @@ -578,7 +578,7 @@ class BaseTimelineGenerator: os.chmod(timeline_summary_file_path, stat.S_IREAD | stat.S_IWRITE) except (IOError, OSError) as err: logger.error('Error occurred when write timeline summary file: %s', err) - raise ProfilerIOException + raise ProfilerIOException() @staticmethod def _update_num_of_streams(timeline, stream_count_dict): @@ -852,7 +852,7 @@ class GpuTimelineGenerator(BaseTimelineGenerator): gpu_start_time = int(lines[1].strip().split(':')[-1]) except (IOError, OSError) as err: logger.error(f'Error occurred when read {start_time_file_path}: {err}') - raise ProfilerIOException + raise ProfilerIOException() time_diff = gpu_start_time - host_monotonic_start_time for idx, time_item in enumerate(timeline_list): @@ -874,7 +874,7 @@ class GpuTimelineGenerator(BaseTimelineGenerator): op_timeline_list.append(line_list) except (IOError, OSError) as err: logger.error('Error occurred when load operator timeline data intermediate file: %s', err) - raise ProfilerIOException + raise ProfilerIOException() return op_timeline_list @@ -898,7 +898,7 @@ class GpuTimelineGenerator(BaseTimelineGenerator): activity_timeline_list.append(line_list) except (IOError, OSError) as err: logger.error('Error occurred when load activity timeline data intermediate file: %s', err) - raise ProfilerIOException + raise ProfilerIOException() return activity_timeline_list @@ -967,7 +967,7 @@ class GpuTimelineGenerator(BaseTimelineGenerator): step_num += 1 except (IOError, OSError) as err: logger.error(f'Error occurred when read {step_trace_profiling_path}: {err}') - raise ProfilerIOException + raise ProfilerIOException() return step_time_list @@ -985,7 +985,7 @@ class GpuTimelineGenerator(BaseTimelineGenerator): return bool(len(first_string.split(',')) == 2) except (IOError, OSError) as err: logger.error(f'Error occurred when read {step_trace_profiling_path}: {err}') - raise ProfilerIOException + raise ProfilerIOException() class AscendTimelineGenerator(BaseTimelineGenerator): @@ -1018,7 +1018,7 @@ class AscendTimelineGenerator(BaseTimelineGenerator): timeline_list.append(line_list) except (IOError, OSError) as err: logger.error('Error occurred when read timeline intermediate file: %s', err) - raise ProfilerIOException + raise ProfilerIOException() return timeline_list @@ -1139,7 +1139,7 @@ class AscendTimelineGenerator(BaseTimelineGenerator): host_monotonic = int(lines[2].strip().split(':')[1]) except (IOError, OSError) as err: logger.error('Error occurred when read host_start.log: %s', err) - raise ProfilerIOException + raise ProfilerIOException() try: with open(dev_start_file_path) as f_obj: lines = f_obj.readlines() @@ -1147,7 +1147,7 @@ class AscendTimelineGenerator(BaseTimelineGenerator): dev_cntvct = int(lines[2].strip().split(':')[1]) except (IOError, OSError) as err: logger.error('Error occurred when read dev_start.log: %s', err) - raise ProfilerIOException + raise ProfilerIOException() factor_ns_to_ms = 1e-6 factor_ten_ns_to_ns = 10