From 512d952865e8f1685164f9c1a8d2c3bd829c4837 Mon Sep 17 00:00:00 2001 From: liuchuting Date: Thu, 24 Mar 2022 19:32:36 +0800 Subject: [PATCH] clean code. --- .../device/profiling/profiling_reporter.cc | 4 +- .../mindspore/profiler/common/struct_type.py | 4 +- .../python/mindspore/profiler/common/util.py | 1 + .../mindspore/profiler/parser/flops_parser.py | 86 ++++++++-------- .../profiler/parser/framework_enum.py | 1 + .../profiler/parser/framework_parser.py | 9 +- .../mindspore/profiler/parser/integrator.py | 13 ++- .../profiler/parser/minddata_analyzer.py | 99 ++++++++++--------- .../parser/minddata_pipeline_parser.py | 2 +- .../profiler/parser/op_intermediate_parser.py | 2 + .../profiler/parser/optime_parser.py | 93 ++++++++--------- .../profiler/parser/step_trace_parser.py | 8 +- .../python/mindspore/profiler/profiling.py | 23 ++++- 13 files changed, 192 insertions(+), 153 deletions(-) diff --git a/mindspore/ccsrc/plugin/device/ascend/hal/device/profiling/profiling_reporter.cc b/mindspore/ccsrc/plugin/device/ascend/hal/device/profiling/profiling_reporter.cc index 880fc4e5bd..4babadb642 100644 --- a/mindspore/ccsrc/plugin/device/ascend/hal/device/profiling/profiling_reporter.cc +++ b/mindspore/ccsrc/plugin/device/ascend/hal/device/profiling/profiling_reporter.cc @@ -104,7 +104,7 @@ void ProfilingReporter::ReportStepPoint(const std::vectortag(); - (void)ReportData(device_id_, reinterpret_cast(&step_point), sizeof(step_point), "step_info"); + ReportData(device_id_, reinterpret_cast(&step_point), sizeof(step_point), "step_info"); auto cnode = GetCNode(op_name); MS_EXCEPTION_IF_NULL(cnode); @@ -165,7 +165,7 @@ void ProfilingReporter::ConstructNodeNameIndexMap() { size_t task_index = 0; for (const auto &node : cnode_list_) { MS_EXCEPTION_IF_NULL(node); - node_name_index_map_.insert(pair(node->fullname_with_scope(), task_index)); + (void)node_name_index_map_.insert(pair(node->fullname_with_scope(), task_index)); ++task_index; } } diff --git a/mindspore/python/mindspore/profiler/common/struct_type.py b/mindspore/python/mindspore/profiler/common/struct_type.py index 417e32ddb8..983ae4bd61 100644 --- a/mindspore/python/mindspore/profiler/common/struct_type.py +++ b/mindspore/python/mindspore/profiler/common/struct_type.py @@ -57,14 +57,14 @@ class StructType(Enum): if isinstance(cpp_type, dict): cpp_type = cpp_type.values() if isinstance(cpp_type, StructType): - return size_map[cpp_type.name] + return size_map.get(cpp_type.name) size = 0 for member in cpp_type: if isinstance(member, list): size += cls.sizeof(member) else: - size += size_map[member.name] + size += size_map.get(member.name) return size @classmethod diff --git a/mindspore/python/mindspore/profiler/common/util.py b/mindspore/python/mindspore/profiler/common/util.py index e7b0ca4eef..02b76aad36 100644 --- a/mindspore/python/mindspore/profiler/common/util.py +++ b/mindspore/python/mindspore/profiler/common/util.py @@ -86,6 +86,7 @@ def fwrite_format(output_data_path, data_source=None, is_print=False, is_start=F def get_log_slice_id(file_name): + """Get log slice id.""" pattern = re.compile(r'(?<=slice_)\d+') slice_list = pattern.findall(file_name) index = re.findall(r'\d+', slice_list[0]) diff --git a/mindspore/python/mindspore/profiler/parser/flops_parser.py b/mindspore/python/mindspore/profiler/parser/flops_parser.py index a25c3c70dc..36a60077b3 100644 --- a/mindspore/python/mindspore/profiler/parser/flops_parser.py +++ b/mindspore/python/mindspore/profiler/parser/flops_parser.py @@ -20,7 +20,7 @@ import stat from mindspore import log as logger from mindspore.profiler.common.exceptions.exceptions import ProfilerIOException, \ - ProfilerFileNotFoundException, ProfilerRawFileException + ProfilerFileNotFoundException, ProfilerRawFileException, ProfilerPathErrorException from mindspore.profiler.common.validator.validate_path import \ validate_and_normalize_path @@ -72,6 +72,43 @@ class FlopsParser: self._flops_sankey_diagram = {} self._max_scope_num = 0 + @staticmethod + def _read_line(start_dot, end_dot, op_avg_time_lines, op_all_step_time, op_all_step_comp): + """Read the bp and fp time from line.""" + for op_avg_idx in op_avg_time_lines: + line = op_avg_idx.split(',') + fp = float(line[start_dot]) / 100000.0 + bp = float(line[end_dot]) / 100000.0 + op_all_step_time.append([fp, bp]) + op_all_step_comp.append([0.0, bp - fp]) + return op_all_step_time, op_all_step_comp + + @staticmethod + def _add_step_flops_time(op_name, task_fops, op_idx, step_idx, op_start_time, + op_all_step_time, op_all_step_comp): + """Get the start time from the current task.""" + while((op_idx < len(op_start_time)) and (op_name != op_start_time[op_idx][0])): + op_idx += 1 + if op_idx >= len(op_start_time): + logger.info(f"Op name {op_name} does not exist in timeline dict.") + return op_idx, step_idx, op_all_step_comp + + # do not add the op FLOPS that not in fp_and_bp time. + while((step_idx < len(op_all_step_time)) and + (op_start_time[op_idx][1] >= op_all_step_time[step_idx][1])): + step_idx += 1 + if step_idx >= len(op_all_step_time): + logger.info(f"Op name {op_name} does not exist in timeline dict.") + + # add the op FLOPS that in fp_and_bp time. + if ((step_idx < len(op_all_step_time)) and + (op_start_time[op_idx][1] >= op_all_step_time[step_idx][0]) and + (op_start_time[op_idx][1] <= op_all_step_time[step_idx][1])): + op_all_step_comp[step_idx][0] += task_fops + # next op. + op_idx += 1 + return op_idx, step_idx, op_all_step_comp + def execute(self): """Get the flops of aicore operators and write to file.""" peak_flops = self._get_peak_flops() @@ -106,7 +143,7 @@ class FlopsParser: continue # Convert the unit of task_fops to MFLOPs(1e6). if op_name in op_compute_dict: - task_fops = op_compute_dict[op_name] + task_fops = op_compute_dict.get(op_name) else: task_fops = self._compute_task_flops(result) * 1e-6 op_compute_dict[op_name] = task_fops @@ -119,7 +156,7 @@ class FlopsParser: # calculate averge op FLOPS. if op_name in op_name_set: continue - op_avg_time = op_avg_time_dict[op_name] + op_avg_time = op_avg_time_dict.get(op_name) # Time unit of op_avg_time is ms. # The unit of gflop_per_second is GFLOPS(1e9). if float(op_avg_time) == 0.0: @@ -439,11 +476,11 @@ class FlopsParser: op_all_step_time, op_all_step_comp = \ self._get_bp_fp_time_by_line(lines, op_all_step_time, op_all_step_comp) except (IOError, OSError) as err: - logger.critical(f'Error occurred when read {optime_file_path} file: {err}') + logger.critical(f'Error occurred when read {_step_trace_file_path} file: {err}') raise ProfilerIOException() logger.info("the train step is %d .", len(op_all_step_time)) if not op_all_step_time: - logger.warning(f'Empty when read {optime_file_path} file, please check the valid' + logger.warning(f'Empty when read {_step_trace_file_path} file, please check the valid' 'data of this file.') return op_all_step_time, op_all_step_comp @@ -461,16 +498,6 @@ class FlopsParser: self._read_line(4, 2, op_avg_time_lines, op_all_step_time, op_all_step_comp) return op_all_step_time, op_all_step_comp - def _read_line(self, start_dot, end_dot, op_avg_time_lines, op_all_step_time, op_all_step_comp): - """Read the bp and fp time from line.""" - for op_avg_idx in op_avg_time_lines: - line = op_avg_idx.split(',') - fp = float(line[start_dot]) / 100000.0 - bp = float(line[end_dot]) / 100000.0 - op_all_step_time.append([fp, bp]) - op_all_step_comp.append([0.0, bp - fp]) - return op_all_step_time, op_all_step_comp - def _get_op_start_time(self): """Get the op average execution time.""" op_start_time = [] @@ -489,34 +516,9 @@ class FlopsParser: op_start = float(line[2]) op_start_time.append([op_name, op_start]) except (IOError, OSError) as err: - logger.critical(f'Error occurred when read {optime_file_path} file: {err}') + logger.critical(f'Error occurred when read {_timeline_file_path} file: {err}') raise ProfilerIOException() if not op_start_time: - logger.warning(f'Empty when read {optime_file_path} file, please check the valid' + logger.warning(f'Empty when read {_timeline_file_path} file, please check the valid' 'data of this file.') return op_start_time - - def _add_step_flops_time(self, op_name, task_fops, op_idx, step_idx, op_start_time, - op_all_step_time, op_all_step_comp): - """Get the start time from the current task.""" - while((op_idx < len(op_start_time)) and (op_name != op_start_time[op_idx][0])): - op_idx += 1 - if op_idx >= len(op_start_time): - logger.info(f"Op name {op_name} does not exist in timeline dict.") - return op_idx, step_idx, op_all_step_comp - - # do not add the op FLOPS that not in fp_and_bp time. - while((step_idx < len(op_all_step_time)) and - (op_start_time[op_idx][1] >= op_all_step_time[step_idx][1])): - step_idx += 1 - if step_idx >= len(op_all_step_time): - logger.info(f"Op name {op_name} does not exist in timeline dict.") - - # add the op FLOPS that in fp_and_bp time. - if ((step_idx < len(op_all_step_time)) and - (op_start_time[op_idx][1] >= op_all_step_time[step_idx][0]) and - (op_start_time[op_idx][1] <= op_all_step_time[step_idx][1])): - op_all_step_comp[step_idx][0] += task_fops - # next op. - op_idx += 1 - return op_idx, step_idx, op_all_step_comp diff --git a/mindspore/python/mindspore/profiler/parser/framework_enum.py b/mindspore/python/mindspore/profiler/parser/framework_enum.py index 8152133c31..fd2771113c 100644 --- a/mindspore/python/mindspore/profiler/parser/framework_enum.py +++ b/mindspore/python/mindspore/profiler/parser/framework_enum.py @@ -26,6 +26,7 @@ class FileDataType(Enum): @classmethod def members(cls): + """Initializes a value of an object.""" return {member.value for member in cls} diff --git a/mindspore/python/mindspore/profiler/parser/framework_parser.py b/mindspore/python/mindspore/profiler/parser/framework_parser.py index ee9cabfceb..3119fe51a9 100644 --- a/mindspore/python/mindspore/profiler/parser/framework_parser.py +++ b/mindspore/python/mindspore/profiler/parser/framework_parser.py @@ -295,7 +295,8 @@ class FrameworkParser: unpack_data = struct.unpack(tensor_num_struct.value, item_binary_data[cursor:cursor + size])[0] return unpack_data - def _construct_task_id_full_op_name_dict(self, task_desc_info): + @staticmethod + def _construct_task_id_full_op_name_dict(task_desc_info): """The task desc info is a list[task_desc], task_desc is a dict, key is same as TASK_DESC_STRUCT.""" task_id_full_op_name = {} for task_desc in task_desc_info: @@ -303,7 +304,8 @@ class FrameworkParser: task_id_full_op_name[task_id] = task_desc['opName'] return task_id_full_op_name - def _construct_point_info(self, task_id_full_op_name_dict, step_point_data): + @staticmethod + def _construct_point_info(task_id_full_op_name_dict, step_point_data): """step_point_data is a list[step_data], step data is a dict, key is same as STEP_INFO_STRUCT.""" point_info = {} for step_point in step_point_data: @@ -313,7 +315,8 @@ class FrameworkParser: point_info[tag] = full_op_name return point_info - def _construct_task_id_op_attr_dict(self, prof_tensor_data): + @staticmethod + def _construct_task_id_op_attr_dict(prof_tensor_data): """prof_tensor_data is a list[tensor_data], tensor_data is a dict, key is same as TENSOR_DATA_STRUCT.""" task_id_op_attr_dict = defaultdict(list) for tensor_data in prof_tensor_data: diff --git a/mindspore/python/mindspore/profiler/parser/integrator.py b/mindspore/python/mindspore/profiler/parser/integrator.py index 87cd45dcf5..726721e2b8 100644 --- a/mindspore/python/mindspore/profiler/parser/integrator.py +++ b/mindspore/python/mindspore/profiler/parser/integrator.py @@ -524,8 +524,6 @@ class BaseTimelineGenerator: _HOST_CPU_PID = 11000 _OP_OVERLAP_PID = 12000 - _OP_GPU_ACTIVITY_PID = 13000 - _RECEIVE_ALONE = 7997 _ALLREDUCE_ALONE = 7998 _MERGED_COMPUTATION_TID = 7999 @@ -569,6 +567,7 @@ class BaseTimelineGenerator: "communication": (self._MERGED_COMMUNICATION_TID, self._OP_OVERLAP_PID), "free_time": (self._FREE_TIME_TID, self._OP_OVERLAP_PID) } + self._step_end_op_name = "" def get_thread_label_name(self): """Get process and thread config.""" @@ -580,8 +579,6 @@ class BaseTimelineGenerator: {"name": "process_labels", "ph": "M", "pid": self._HOST_CPU_PID, "args": {"labels": "Host CPU Op"}}, {"name": "process_labels", "ph": "M", "pid": self._OP_OVERLAP_PID, "args": {"labels": "Op Overlap Analyse"}}, - {"name": "process_labels", "ph": "M", "pid": self._OP_GPU_ACTIVITY_PID, - "args": {"labels": "Activity Op"}}, {"name": "process_sort_index", "ph": "M", "pid": self._device_id, "args": {"sort_index": 0}}, {"name": "process_sort_index", "ph": "M", "pid": self._AI_CPU_PID, "args": {"sort_index": 10}}, @@ -877,6 +874,8 @@ class BaseTimelineGenerator: "is not supported in offline parse mode.") parallel_mode = "data_parallel" stage_num = 1 + finally: + pass if stage_num > 1: parallel_mode = "pipeline-parallel" elif parallel_mode != "data_parallel": @@ -1310,6 +1309,8 @@ class GpuTimelineGenerator(BaseTimelineGenerator): computation_time.append(step_info[step][self._duration_idx] - comm_alone_time[step]) except IndexError as e: logger.error(e) + finally: + pass metrices_per_step_list = [computation_time, comm_alone_time, stage_time, recieve_alone_time, collective_comm_alone_time] @@ -1705,6 +1706,8 @@ class AscendTimelineGenerator(BaseTimelineGenerator): computation_time.append(step_info[step][self._duration_idx] - comm_alone_time[step]) except IndexError as err: logger.error(err) + finally: + pass metrices_per_step_list = [computation_time, comm_alone_time, stage_time, recieve_alone_time, collective_comm_alone_time] if step_num > 1: @@ -1837,6 +1840,8 @@ class AscendTimelineGenerator(BaseTimelineGenerator): except (IOError, OSError) as err: logger.critical(f'Error occurred when read {start_time_file_path}: {err}') raise ProfilerIOException() + finally: + pass time_diff = gpu_start_time * 1000 - host_monotonic_start_time for idx, time_item in enumerate(timeline_list): timeline_list[idx][self._start_time_idx] = int(time_item[self._start_time_idx]) + time_diff diff --git a/mindspore/python/mindspore/profiler/parser/minddata_analyzer.py b/mindspore/python/mindspore/profiler/parser/minddata_analyzer.py index 6a022478c1..55b1b33ba5 100644 --- a/mindspore/python/mindspore/profiler/parser/minddata_analyzer.py +++ b/mindspore/python/mindspore/profiler/parser/minddata_analyzer.py @@ -437,6 +437,56 @@ class MinddataProfilingAnalyzer: return_dict['avg_cpu_pct'] = oplist_avg_cpu_pct return return_dict + @staticmethod + def _compute_composite_info(summary_dict): + """ + Compute composite analysis information from the current summary pipeline data. + + Args: + summary_dict (dict): Input summary pipeline information. + + Returns: + Dictionary with composite analysis output information + Dictionary consists of: + avg_cpu_pct_per_worker: Average CPU utilization percentage per worker + """ + return_dict = {} + + # Build list: average CPU utilization percentage per worker - for each op + avg_cpu_pct_per_worker = [] + for c, n in zip(summary_dict.get('avg_cpu_pct'), summary_dict.get('num_workers')): + avg_cpu_pct_per_worker.append(round(c / n if (n != 0 and c >= 0) else -1, 2)) + return_dict['avg_cpu_pct_per_worker'] = avg_cpu_pct_per_worker + + return return_dict + + @staticmethod + def _analyze_for_bottleneck_op(summary_dict): + """ + Analyze the MindData summary information and identify any potential bottleneck operator + in the MindData pipeline. + + Args: + summary_dict (dict): Input summary pipeline information. + + Returns: + Dictionary with the following information, if applicable: + - CPU utilization analysis + - queue utilization analysis + - bottleneck warning: Information on the bottleneck op + (This is returned only if a potential bottleneck is identified.) + - bottleneck suggestion: Reason why the subject op is it is identified as + a potential bottleneck, plus suggestion on how to resolve the bottleneck. + (This is returned only if a potential bottleneck is identified.) + """ + try: + bottleneck_analyzer = BottleneckAnalyzer(summary_dict) + return_dict = bottleneck_analyzer.analyze() + except IndexError: + return_dict = {} + + return return_dict + def _parse_device_trace_info(self, device_trace_info): """ Parse and process the device trace profiling information. @@ -494,55 +544,6 @@ class MinddataProfilingAnalyzer: return return_dict - def _compute_composite_info(self, summary_dict): - """ - Compute composite analysis information from the current summary pipeline data. - - Args: - summary_dict (dict): Input summary pipeline information. - - Returns: - Dictionary with composite analysis output information - Dictionary consists of: - avg_cpu_pct_per_worker: Average CPU utilization percentage per worker - """ - return_dict = {} - - # Build list: average CPU utilization percentage per worker - for each op - avg_cpu_pct_per_worker = [] - for c, n in zip(summary_dict.get('avg_cpu_pct'), summary_dict.get('num_workers')): - avg_cpu_pct_per_worker.append(round(c / n if (n != 0 and c >= 0) else -1, 2)) - return_dict['avg_cpu_pct_per_worker'] = avg_cpu_pct_per_worker - - return return_dict - - @staticmethod - def _analyze_for_bottleneck_op(summary_dict): - """ - Analyze the MindData summary information and identify any potential bottleneck operator - in the MindData pipeline. - - Args: - summary_dict (dict): Input summary pipeline information. - - Returns: - Dictionary with the following information, if applicable: - - CPU utilization analysis - - queue utilization analysis - - bottleneck warning: Information on the bottleneck op - (This is returned only if a potential bottleneck is identified.) - - bottleneck suggestion: Reason why the subject op is it is identified as - a potential bottleneck, plus suggestion on how to resolve the bottleneck. - (This is returned only if a potential bottleneck is identified.) - """ - try: - bottleneck_analyzer = BottleneckAnalyzer(summary_dict) - return_dict = bottleneck_analyzer.analyze() - except IndexError: - return_dict = {} - - return return_dict - def _save_as_csv_file(self, data_dict): """ Save data dictionary information to CSV file. diff --git a/mindspore/python/mindspore/profiler/parser/minddata_pipeline_parser.py b/mindspore/python/mindspore/profiler/parser/minddata_pipeline_parser.py index 4aee2eaa1d..adf8502dd8 100644 --- a/mindspore/python/mindspore/profiler/parser/minddata_pipeline_parser.py +++ b/mindspore/python/mindspore/profiler/parser/minddata_pipeline_parser.py @@ -129,7 +129,7 @@ class MinddataPipelineParser: """ try: output_dir = validate_and_normalize_path(output_path) - except ValidationError: + except RuntimeError: logger.warning('Output path is invalid.') raise ProfilerPathErrorException('Output path is invalid.') if not os.path.isdir(output_dir): diff --git a/mindspore/python/mindspore/profiler/parser/op_intermediate_parser.py b/mindspore/python/mindspore/profiler/parser/op_intermediate_parser.py index a069ceb053..ec0c049c58 100644 --- a/mindspore/python/mindspore/profiler/parser/op_intermediate_parser.py +++ b/mindspore/python/mindspore/profiler/parser/op_intermediate_parser.py @@ -75,6 +75,8 @@ class OPIntermediateParser: except (IOError, OSError) as err: logger.critical('Error occurred when read timeline intermediate file: %s', err) raise ProfilerIOException() + finally: + pass return timeline_list diff --git a/mindspore/python/mindspore/profiler/parser/optime_parser.py b/mindspore/python/mindspore/profiler/parser/optime_parser.py index 214d1102bf..20e196e073 100644 --- a/mindspore/python/mindspore/profiler/parser/optime_parser.py +++ b/mindspore/python/mindspore/profiler/parser/optime_parser.py @@ -49,6 +49,50 @@ class OPComputeTimeParser: self._device_id = device_id self._min_cycle_counter = float("inf") + @property + def min_cycle_counter(self): + """Get minimum cycle counter.""" + return self._min_cycle_counter + + @staticmethod + def _convert_op_time_unit(op_data_list, op_name_time_dict, op_name_stream_dict, + op_name_count_dict, op_name_task_dict, op_name_start_time): + """ + Calculate the execution time of operator and convert it into millisecond. + + Args: + op_data_list (list): The list of operator metadata. + op_name_time_dict (dict): The mapping relation of operator name and its execution time. + op_name_stream_dict (dict): The mapping relation of operator name and its stream id. + op_name_count_dict (dict): The mapping relation of operator name and its count. + op_name_task_dict (dict): The mapping relation of operator name and its task id. + op_name_start_time (dict): The mapping relation of operator name and its start time. + """ + factor = 1e5 + for item in op_data_list: + op_name = item.op_name + # Unit conversion: converting the cycle counter into ms. + op_start_time_str = str(item.cycle_counter / factor) + op_duration = item.duration / factor + op_duration_str = str(item.duration / factor) + if op_name in op_name_time_dict.keys(): + op_name_time_dict[op_name] += op_duration + if item.task_id == op_name_task_dict[op_name]: + op_name_count_dict[op_name] += 1 + op_name_start_time[op_name].append( + (op_start_time_str, op_duration_str) + ) + + else: + op_name_time_dict[op_name] = op_duration + op_name_stream_dict[op_name] = item.stream_id + op_name_task_dict[op_name] = item.task_id + op_name_count_dict[op_name] = 1 + op_name_start_time[op_name] = [] + op_name_start_time[op_name].append( + (op_start_time_str, op_duration_str) + ) + def _get_op_task_id_map(self): """ Read hwts data file, get the task time info. @@ -102,10 +146,10 @@ class OPComputeTimeParser: total_time = 0 for op_name, time in op_name_time_dict.items(): if op_name in op_name_stream_dict.keys(): - stream_id = op_name_stream_dict[op_name] - if op_name_count_dict[op_name] == 0: + stream_id = op_name_stream_dict.get(op_name) + if op_name_count_dict.get(op_name) == 0: raise ValueError("The number of operations can not be 0.") - avg_time = time / op_name_count_dict[op_name] + avg_time = time / op_name_count_dict.get(op_name) total_time += avg_time result_data += ("%s %s %s\n" % (op_name, str(avg_time), stream_id)) result_data += ("total op %s 0" % (str(total_time))) @@ -205,46 +249,3 @@ class OPComputeTimeParser: self._min_cycle_counter = min_cycle_counter / 1e5 # Convert the time unit from 10ns to 1ms return tmp_result_data - - def _convert_op_time_unit(self, op_data_list, op_name_time_dict, op_name_stream_dict, - op_name_count_dict, op_name_task_dict, op_name_start_time): - """ - Calculate the execution time of operator and convert it into millisecond. - - Args: - op_data_list (list): The list of operator metadata. - op_name_time_dict (dict): The mapping relation of operator name and its execution time. - op_name_stream_dict (dict): The mapping relation of operator name and its stream id. - op_name_count_dict (dict): The mapping relation of operator name and its count. - op_name_task_dict (dict): The mapping relation of operator name and its task id. - op_name_start_time (dict): The mapping relation of operator name and its start time. - """ - factor = 1e5 - for item in op_data_list: - op_name = item.op_name - # Unit conversion: converting the cycle counter into ms. - op_start_time_str = str(item.cycle_counter / factor) - op_duration = item.duration / factor - op_duration_str = str(item.duration / factor) - if op_name in op_name_time_dict.keys(): - op_name_time_dict[op_name] += op_duration - if item.task_id == op_name_task_dict[op_name]: - op_name_count_dict[op_name] += 1 - op_name_start_time[op_name].append( - (op_start_time_str, op_duration_str) - ) - - else: - op_name_time_dict[op_name] = op_duration - op_name_stream_dict[op_name] = item.stream_id - op_name_task_dict[op_name] = item.task_id - op_name_count_dict[op_name] = 1 - op_name_start_time[op_name] = [] - op_name_start_time[op_name].append( - (op_start_time_str, op_duration_str) - ) - - @property - def min_cycle_counter(self): - """Get minimum cycle counter.""" - return self._min_cycle_counter diff --git a/mindspore/python/mindspore/profiler/parser/step_trace_parser.py b/mindspore/python/mindspore/profiler/parser/step_trace_parser.py index b5466225c6..61738b8021 100644 --- a/mindspore/python/mindspore/profiler/parser/step_trace_parser.py +++ b/mindspore/python/mindspore/profiler/parser/step_trace_parser.py @@ -31,6 +31,7 @@ from mindspore.profiler.common.util import combine_stream_task_id class PointTag(Enum): + """Initializing indexes.""" MODEL_START = 0 MODEL_END = 1 FP_START = 2 @@ -352,6 +353,8 @@ class GpuStepTraceParser(BaseStepTraceParser): except (IOError, OSError) as err: log.warning(f'Failed to read {source_file}', err) raise ProfilerIOException + finally: + pass for step_num in range(num_of_step): step_trace = { @@ -458,6 +461,7 @@ class AscendStepTraceParser(BaseStepTraceParser): self._task_id_op_name_dict = {} def set_task_id_op_name_dict(self, task_id_op_name_dict): + """The operator task id matches the operator name.""" self._task_id_op_name_dict = task_id_op_name_dict def record_point_info(self, output_path): @@ -567,7 +571,7 @@ class AscendStepTraceParser(BaseStepTraceParser): """Save step trace data to result.""" step_trace = {'reduce': defaultdict(list), 'start': '-'} for ts_track in ts_tracks: - if ts_track['rptType'] != STEP_TRACE_RPT_TYPE: + if ts_track.get('rptType') != STEP_TRACE_RPT_TYPE: continue self._construct_step_trace(ts_track, step_trace) @@ -599,6 +603,8 @@ class AscendStepTraceParser(BaseStepTraceParser): except (IOError, OSError) as err: log.critical("Can not parse profiler file, open file %s failed, detail: %s.", path, str(err)) raise ProfilerIOException() + finally: + pass log.info("Profiler found %d ts track step trace data.", len(ts_tracks)) return ts_tracks diff --git a/mindspore/python/mindspore/profiler/profiling.py b/mindspore/python/mindspore/profiler/profiling.py index 051432ecd8..910d87a973 100644 --- a/mindspore/python/mindspore/profiler/profiling.py +++ b/mindspore/python/mindspore/profiler/profiling.py @@ -51,6 +51,7 @@ INIT_OP_NAME = 'Default/InitDataSetQueue' def deprecated(name, version): + """Warning notices.""" msg = f"The {name} is deprecated from MindSpore {version} and will be removed in a future version." logger.warning(msg) @@ -418,6 +419,8 @@ class Profiler: pipeline_parser.parse() except ProfilerException as err: logger.warning(err.message) + finally: + pass # Analyze minddata information try: @@ -426,6 +429,8 @@ class Profiler: md_analyzer.analyze() except ProfilerException as err: logger.warning(err.message) + finally: + pass def _ascend_graph_analyse(self): """Ascend graph mode analyse.""" @@ -450,6 +455,8 @@ class Profiler: self._analyser_op_info() except ProfilerException as err: logger.warning(err.message) + finally: + pass # analyse step trace info points = None @@ -460,6 +467,8 @@ class Profiler: points, is_training_mode_flag = self._analyse_step_trace(source_path, framework_parser) except ProfilerException as err: logger.warning(err.message) + finally: + pass # analyse timeline info try: @@ -467,6 +476,8 @@ class Profiler: self._analyse_timeline(aicpu_data_parser, optime_parser, source_path) except (ProfilerIOException, ProfilerFileNotFoundException, RuntimeError) as err: logger.warning('Fail to write timeline data: %s', err) + finally: + pass # analyse memory usage info if self._profile_memory: @@ -475,6 +486,8 @@ class Profiler: self._analyse_memory_usage(points) except (ProfilerIOException, ProfilerFileNotFoundException, ProfilerRawFileException) as err: logger.warning(err.message) + finally: + pass # analyse hccl profiler info if self._profile_communication: @@ -483,6 +496,8 @@ class Profiler: self._analyse_hccl_info() except (ProfilerIOException, ProfilerFileNotFoundException, ProfilerRawFileException) as err: logger.warning(err.message) + finally: + pass # get op FLOPs from aicore.data.x.slice.0 file, and compute FLOPS, write output_op_flops_x.txt flops_parser = FlopsParser(source_path, self._output_path, op_task_dict, @@ -492,10 +507,13 @@ class Profiler: @staticmethod def _check_output_path(output_path): + """Checking path validity.""" try: output_path = validate_and_normalize_path(output_path) except RuntimeError: raise ProfilerPathErrorException(f'profiling data output path {output_path} is invalid.') + finally: + pass if not os.path.isdir(output_path): raise ProfilerDirNotFoundException(output_path) return output_path @@ -583,9 +601,6 @@ class Profiler: if not os.path.exists(data_path): os.makedirs(data_path, exist_ok=True) - # add job id env through user input later - self._job_id_env = 0 - self._ascend_profiler.start() def stop(self): @@ -683,6 +698,8 @@ class Profiler: ) except ProfilerException as err: logger.warning(err.message) + finally: + pass logger.warning( '\nThe training and inference process does not support profiler currently, '