timekiller
/
mindspore2022
forked from huawei/mindspore2022
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!31873 MS clean code. 

Merge pull request !31873 from liuchuting/clean_code
This commit is contained in:
i-robot 2022-03-25 08:55:46 +00:00 committed by Gitee
parent b4b2d9291f 512d952865
commit 540dc43cb0
No known key found for this signature in database
GPG Key ID: 173E9B9CA92EEF8F
13 changed files with 192 additions and 153 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
mindspore/ccsrc/plugin/device/ascend/hal/device/profiling/profiling_reporter.cc
View File

@ -104,7 +104,7 @@ void ProfilingReporter::ReportStepPoint(const std::vector<std::shared_ptr<StepPo
step_point.curIterNum = 0;
step_point.threadId = 0;
step_point.tag = point->tag();
(void)ReportData(device_id_, reinterpret_cast<unsigned char *>(&step_point), sizeof(step_point), "step_info");
ReportData(device_id_, reinterpret_cast<unsigned char *>(&step_point), sizeof(step_point), "step_info");
auto cnode = GetCNode(op_name);
MS_EXCEPTION_IF_NULL(cnode);
@ -165,7 +165,7 @@ void ProfilingReporter::ConstructNodeNameIndexMap() {
size_t task_index = 0;
for (const auto &node : cnode_list_) {
MS_EXCEPTION_IF_NULL(node);
node_name_index_map_.insert(pair<string, int>(node->fullname_with_scope(), task_index));
(void)node_name_index_map_.insert(pair<string, size_t>(node->fullname_with_scope(), task_index));
++task_index;
}
}

4
mindspore/python/mindspore/profiler/common/struct_type.py
View File

@ -57,14 +57,14 @@ class StructType(Enum):
if isinstance(cpp_type, dict):
cpp_type = cpp_type.values()
if isinstance(cpp_type, StructType):
return size_map[cpp_type.name]
return size_map.get(cpp_type.name)
size = 0
for member in cpp_type:
if isinstance(member, list):
size += cls.sizeof(member)
else:
size += size_map[member.name]
size += size_map.get(member.name)
return size
@classmethod

1
mindspore/python/mindspore/profiler/common/util.py
View File

@ -86,6 +86,7 @@ def fwrite_format(output_data_path, data_source=None, is_print=False, is_start=F
def get_log_slice_id(file_name):
"""Get log slice id."""
pattern = re.compile(r'(?<=slice_)\d+')
slice_list = pattern.findall(file_name)
index = re.findall(r'\d+', slice_list[0])

86
mindspore/python/mindspore/profiler/parser/flops_parser.py
View File

@ -20,7 +20,7 @@ import stat
from mindspore import log as logger
from mindspore.profiler.common.exceptions.exceptions import ProfilerIOException, \
ProfilerFileNotFoundException, ProfilerRawFileException
ProfilerFileNotFoundException, ProfilerRawFileException, ProfilerPathErrorException
from mindspore.profiler.common.validator.validate_path import \
validate_and_normalize_path
@ -72,6 +72,43 @@ class FlopsParser:
self._flops_sankey_diagram = {}
self._max_scope_num = 0
@staticmethod
def _read_line(start_dot, end_dot, op_avg_time_lines, op_all_step_time, op_all_step_comp):
"""Read the bp and fp time from line."""
for op_avg_idx in op_avg_time_lines:
line = op_avg_idx.split(',')
fp = float(line[start_dot]) / 100000.0
bp = float(line[end_dot]) / 100000.0
op_all_step_time.append([fp, bp])
op_all_step_comp.append([0.0, bp - fp])
return op_all_step_time, op_all_step_comp
@staticmethod
def _add_step_flops_time(op_name, task_fops, op_idx, step_idx, op_start_time,
op_all_step_time, op_all_step_comp):
"""Get the start time from the current task."""
while((op_idx < len(op_start_time)) and (op_name != op_start_time[op_idx][0])):
op_idx += 1
if op_idx >= len(op_start_time):
logger.info(f"Op name {op_name} does not exist in timeline dict.")
return op_idx, step_idx, op_all_step_comp
# do not add the op FLOPS that not in fp_and_bp time.
while((step_idx < len(op_all_step_time)) and
(op_start_time[op_idx][1] >= op_all_step_time[step_idx][1])):
step_idx += 1
if step_idx >= len(op_all_step_time):
logger.info(f"Op name {op_name} does not exist in timeline dict.")
# add the op FLOPS that in fp_and_bp time.
if ((step_idx < len(op_all_step_time)) and
(op_start_time[op_idx][1] >= op_all_step_time[step_idx][0]) and
(op_start_time[op_idx][1] <= op_all_step_time[step_idx][1])):
op_all_step_comp[step_idx][0] += task_fops
# next op.
op_idx += 1
return op_idx, step_idx, op_all_step_comp
def execute(self):
"""Get the flops of aicore operators and write to file."""
peak_flops = self._get_peak_flops()
@ -106,7 +143,7 @@ class FlopsParser:
continue
# Convert the unit of task_fops to MFLOPs(1e6).
if op_name in op_compute_dict:
task_fops = op_compute_dict[op_name]
task_fops = op_compute_dict.get(op_name)
else:
task_fops = self._compute_task_flops(result) * 1e-6
op_compute_dict[op_name] = task_fops
@ -119,7 +156,7 @@ class FlopsParser:
# calculate averge op FLOPS.
if op_name in op_name_set:
continue
op_avg_time = op_avg_time_dict[op_name]
op_avg_time = op_avg_time_dict.get(op_name)
# Time unit of op_avg_time is ms.
# The unit of gflop_per_second is GFLOPS(1e9).
if float(op_avg_time) == 0.0:
@ -439,11 +476,11 @@ class FlopsParser:
op_all_step_time, op_all_step_comp = \
self._get_bp_fp_time_by_line(lines, op_all_step_time, op_all_step_comp)
except (IOError, OSError) as err:
logger.critical(f'Error occurred when read {optime_file_path} file: {err}')
logger.critical(f'Error occurred when read {_step_trace_file_path} file: {err}')
raise ProfilerIOException()
logger.info("the train step is %d .", len(op_all_step_time))
if not op_all_step_time:
logger.warning(f'Empty when read {optime_file_path} file, please check the valid'
logger.warning(f'Empty when read {_step_trace_file_path} file, please check the valid'
'data of this file.')
return op_all_step_time, op_all_step_comp
@ -461,16 +498,6 @@ class FlopsParser:
self._read_line(4, 2, op_avg_time_lines, op_all_step_time, op_all_step_comp)
return op_all_step_time, op_all_step_comp
def _read_line(self, start_dot, end_dot, op_avg_time_lines, op_all_step_time, op_all_step_comp):
"""Read the bp and fp time from line."""
for op_avg_idx in op_avg_time_lines:
line = op_avg_idx.split(',')
fp = float(line[start_dot]) / 100000.0
bp = float(line[end_dot]) / 100000.0
op_all_step_time.append([fp, bp])
op_all_step_comp.append([0.0, bp - fp])
return op_all_step_time, op_all_step_comp
def _get_op_start_time(self):
"""Get the op average execution time."""
op_start_time = []
@ -489,34 +516,9 @@ class FlopsParser:
op_start = float(line[2])
op_start_time.append([op_name, op_start])
except (IOError, OSError) as err:
logger.critical(f'Error occurred when read {optime_file_path} file: {err}')
logger.critical(f'Error occurred when read {_timeline_file_path} file: {err}')
raise ProfilerIOException()
if not op_start_time:
logger.warning(f'Empty when read {optime_file_path} file, please check the valid'
logger.warning(f'Empty when read {_timeline_file_path} file, please check the valid'
'data of this file.')
return op_start_time
def _add_step_flops_time(self, op_name, task_fops, op_idx, step_idx, op_start_time,
op_all_step_time, op_all_step_comp):
"""Get the start time from the current task."""
while((op_idx < len(op_start_time)) and (op_name != op_start_time[op_idx][0])):
op_idx += 1
if op_idx >= len(op_start_time):
logger.info(f"Op name {op_name} does not exist in timeline dict.")
return op_idx, step_idx, op_all_step_comp
# do not add the op FLOPS that not in fp_and_bp time.
while((step_idx < len(op_all_step_time)) and
(op_start_time[op_idx][1] >= op_all_step_time[step_idx][1])):
step_idx += 1
if step_idx >= len(op_all_step_time):
logger.info(f"Op name {op_name} does not exist in timeline dict.")
# add the op FLOPS that in fp_and_bp time.
if ((step_idx < len(op_all_step_time)) and
(op_start_time[op_idx][1] >= op_all_step_time[step_idx][0]) and
(op_start_time[op_idx][1] <= op_all_step_time[step_idx][1])):
op_all_step_comp[step_idx][0] += task_fops
# next op.
op_idx += 1
return op_idx, step_idx, op_all_step_comp

1
mindspore/python/mindspore/profiler/parser/framework_enum.py
View File

@ -26,6 +26,7 @@ class FileDataType(Enum):
@classmethod
def members(cls):
"""Initializes a value of an object."""
return {member.value for member in cls}

9
mindspore/python/mindspore/profiler/parser/framework_parser.py
View File

@ -295,7 +295,8 @@ class FrameworkParser:
unpack_data = struct.unpack(tensor_num_struct.value, item_binary_data[cursor:cursor + size])[0]
return unpack_data
def _construct_task_id_full_op_name_dict(self, task_desc_info):
@staticmethod
def _construct_task_id_full_op_name_dict(task_desc_info):
"""The task desc info is a list[task_desc], task_desc is a dict, key is same as TASK_DESC_STRUCT."""
task_id_full_op_name = {}
for task_desc in task_desc_info:
@ -303,7 +304,8 @@ class FrameworkParser:
task_id_full_op_name[task_id] = task_desc['opName']
return task_id_full_op_name
def _construct_point_info(self, task_id_full_op_name_dict, step_point_data):
@staticmethod
def _construct_point_info(task_id_full_op_name_dict, step_point_data):
"""step_point_data is a list[step_data], step data is a dict, key is same as STEP_INFO_STRUCT."""
point_info = {}
for step_point in step_point_data:
@ -313,7 +315,8 @@ class FrameworkParser:
point_info[tag] = full_op_name
return point_info
def _construct_task_id_op_attr_dict(self, prof_tensor_data):
@staticmethod
def _construct_task_id_op_attr_dict(prof_tensor_data):
"""prof_tensor_data is a list[tensor_data], tensor_data is a dict, key is same as TENSOR_DATA_STRUCT."""
task_id_op_attr_dict = defaultdict(list)
for tensor_data in prof_tensor_data:

13
mindspore/python/mindspore/profiler/parser/integrator.py
View File

@ -524,8 +524,6 @@ class BaseTimelineGenerator:
_HOST_CPU_PID = 11000
_OP_OVERLAP_PID = 12000
_OP_GPU_ACTIVITY_PID = 13000
_RECEIVE_ALONE = 7997
_ALLREDUCE_ALONE = 7998
_MERGED_COMPUTATION_TID = 7999
@ -569,6 +567,7 @@ class BaseTimelineGenerator:
"communication": (self._MERGED_COMMUNICATION_TID, self._OP_OVERLAP_PID),
"free_time": (self._FREE_TIME_TID, self._OP_OVERLAP_PID)
}
self._step_end_op_name = ""
def get_thread_label_name(self):
"""Get process and thread config."""
@ -580,8 +579,6 @@ class BaseTimelineGenerator:
{"name": "process_labels", "ph": "M", "pid": self._HOST_CPU_PID, "args": {"labels": "Host CPU Op"}},
{"name": "process_labels", "ph": "M", "pid": self._OP_OVERLAP_PID,
"args": {"labels": "Op Overlap Analyse"}},
{"name": "process_labels", "ph": "M", "pid": self._OP_GPU_ACTIVITY_PID,
"args": {"labels": "Activity Op"}},
{"name": "process_sort_index", "ph": "M", "pid": self._device_id, "args": {"sort_index": 0}},
{"name": "process_sort_index", "ph": "M", "pid": self._AI_CPU_PID, "args": {"sort_index": 10}},
@ -877,6 +874,8 @@ class BaseTimelineGenerator:
"is not supported in offline parse mode.")
parallel_mode = "data_parallel"
stage_num = 1
finally:
pass
if stage_num > 1:
parallel_mode = "pipeline-parallel"
elif parallel_mode != "data_parallel":
@ -1310,6 +1309,8 @@ class GpuTimelineGenerator(BaseTimelineGenerator):
computation_time.append(step_info[step][self._duration_idx] - comm_alone_time[step])
except IndexError as e:
logger.error(e)
finally:
pass
metrices_per_step_list = [computation_time, comm_alone_time, stage_time,
recieve_alone_time, collective_comm_alone_time]
@ -1705,6 +1706,8 @@ class AscendTimelineGenerator(BaseTimelineGenerator):
computation_time.append(step_info[step][self._duration_idx] - comm_alone_time[step])
except IndexError as err:
logger.error(err)
finally:
pass
metrices_per_step_list = [computation_time, comm_alone_time, stage_time,
recieve_alone_time, collective_comm_alone_time]
if step_num > 1:
@ -1837,6 +1840,8 @@ class AscendTimelineGenerator(BaseTimelineGenerator):
except (IOError, OSError) as err:
logger.critical(f'Error occurred when read {start_time_file_path}: {err}')
raise ProfilerIOException()
finally:
pass
time_diff = gpu_start_time * 1000 - host_monotonic_start_time
for idx, time_item in enumerate(timeline_list):
timeline_list[idx][self._start_time_idx] = int(time_item[self._start_time_idx]) + time_diff

99
mindspore/python/mindspore/profiler/parser/minddata_analyzer.py
View File

@ -437,6 +437,56 @@ class MinddataProfilingAnalyzer:
return_dict['avg_cpu_pct'] = oplist_avg_cpu_pct
return return_dict
@staticmethod
def _compute_composite_info(summary_dict):
"""
Compute composite analysis information from the current summary pipeline data.
Args:
summary_dict (dict): Input summary pipeline information.
Returns:
Dictionary with composite analysis output information
Dictionary consists of:
avg_cpu_pct_per_worker: Average CPU utilization percentage per worker
"""
return_dict = {}
# Build list: average CPU utilization percentage per worker - for each op
avg_cpu_pct_per_worker = []
for c, n in zip(summary_dict.get('avg_cpu_pct'), summary_dict.get('num_workers')):
avg_cpu_pct_per_worker.append(round(c / n if (n != 0 and c >= 0) else -1, 2))
return_dict['avg_cpu_pct_per_worker'] = avg_cpu_pct_per_worker
return return_dict
@staticmethod
def _analyze_for_bottleneck_op(summary_dict):
"""
Analyze the MindData summary information and identify any potential bottleneck operator
in the MindData pipeline.
Args:
summary_dict (dict): Input summary pipeline information.
Returns:
Dictionary with the following information, if applicable:
- CPU utilization analysis
- queue utilization analysis
- bottleneck warning: Information on the bottleneck op
(This is returned only if a potential bottleneck is identified.)
- bottleneck suggestion: Reason why the subject op is it is identified as
a potential bottleneck, plus suggestion on how to resolve the bottleneck.
(This is returned only if a potential bottleneck is identified.)
"""
try:
bottleneck_analyzer = BottleneckAnalyzer(summary_dict)
return_dict = bottleneck_analyzer.analyze()
except IndexError:
return_dict = {}
return return_dict
def _parse_device_trace_info(self, device_trace_info):
"""
Parse and process the device trace profiling information.
@ -494,55 +544,6 @@ class MinddataProfilingAnalyzer:
return return_dict
def _compute_composite_info(self, summary_dict):
"""
Compute composite analysis information from the current summary pipeline data.
Args:
summary_dict (dict): Input summary pipeline information.
Returns:
Dictionary with composite analysis output information
Dictionary consists of:
avg_cpu_pct_per_worker: Average CPU utilization percentage per worker
"""
return_dict = {}
# Build list: average CPU utilization percentage per worker - for each op
avg_cpu_pct_per_worker = []
for c, n in zip(summary_dict.get('avg_cpu_pct'), summary_dict.get('num_workers')):
avg_cpu_pct_per_worker.append(round(c / n if (n != 0 and c >= 0) else -1, 2))
return_dict['avg_cpu_pct_per_worker'] = avg_cpu_pct_per_worker
return return_dict
@staticmethod
def _analyze_for_bottleneck_op(summary_dict):
"""
Analyze the MindData summary information and identify any potential bottleneck operator
in the MindData pipeline.
Args:
summary_dict (dict): Input summary pipeline information.
Returns:
Dictionary with the following information, if applicable:
- CPU utilization analysis
- queue utilization analysis
- bottleneck warning: Information on the bottleneck op
(This is returned only if a potential bottleneck is identified.)
- bottleneck suggestion: Reason why the subject op is it is identified as
a potential bottleneck, plus suggestion on how to resolve the bottleneck.
(This is returned only if a potential bottleneck is identified.)
"""
try:
bottleneck_analyzer = BottleneckAnalyzer(summary_dict)
return_dict = bottleneck_analyzer.analyze()
except IndexError:
return_dict = {}
return return_dict
def _save_as_csv_file(self, data_dict):
"""
Save data dictionary information to CSV file.

2
mindspore/python/mindspore/profiler/parser/minddata_pipeline_parser.py
View File

@ -129,7 +129,7 @@ class MinddataPipelineParser:
"""
try:
output_dir = validate_and_normalize_path(output_path)
except ValidationError:
except RuntimeError:
logger.warning('Output path is invalid.')
raise ProfilerPathErrorException('Output path is invalid.')
if not os.path.isdir(output_dir):

2
mindspore/python/mindspore/profiler/parser/op_intermediate_parser.py
View File

@ -75,6 +75,8 @@ class OPIntermediateParser:
except (IOError, OSError) as err:
logger.critical('Error occurred when read timeline intermediate file: %s', err)
raise ProfilerIOException()
finally:
pass
return timeline_list

93
mindspore/python/mindspore/profiler/parser/optime_parser.py
View File

@ -49,6 +49,50 @@ class OPComputeTimeParser:
self._device_id = device_id
self._min_cycle_counter = float("inf")
@property
def min_cycle_counter(self):
"""Get minimum cycle counter."""
return self._min_cycle_counter
@staticmethod
def _convert_op_time_unit(op_data_list, op_name_time_dict, op_name_stream_dict,
op_name_count_dict, op_name_task_dict, op_name_start_time):
"""
Calculate the execution time of operator and convert it into millisecond.
Args:
op_data_list (list): The list of operator metadata.
op_name_time_dict (dict): The mapping relation of operator name and its execution time.
op_name_stream_dict (dict): The mapping relation of operator name and its stream id.
op_name_count_dict (dict): The mapping relation of operator name and its count.
op_name_task_dict (dict): The mapping relation of operator name and its task id.
op_name_start_time (dict): The mapping relation of operator name and its start time.
"""
factor = 1e5
for item in op_data_list:
op_name = item.op_name
# Unit conversion: converting the cycle counter into ms.
op_start_time_str = str(item.cycle_counter / factor)
op_duration = item.duration / factor
op_duration_str = str(item.duration / factor)
if op_name in op_name_time_dict.keys():
op_name_time_dict[op_name] += op_duration
if item.task_id == op_name_task_dict[op_name]:
op_name_count_dict[op_name] += 1
op_name_start_time[op_name].append(
(op_start_time_str, op_duration_str)
)
else:
op_name_time_dict[op_name] = op_duration
op_name_stream_dict[op_name] = item.stream_id
op_name_task_dict[op_name] = item.task_id
op_name_count_dict[op_name] = 1
op_name_start_time[op_name] = []
op_name_start_time[op_name].append(
(op_start_time_str, op_duration_str)
)
def _get_op_task_id_map(self):
"""
Read hwts data file, get the task time info.
@ -102,10 +146,10 @@ class OPComputeTimeParser:
total_time = 0
for op_name, time in op_name_time_dict.items():
if op_name in op_name_stream_dict.keys():
stream_id = op_name_stream_dict[op_name]
if op_name_count_dict[op_name] == 0:
stream_id = op_name_stream_dict.get(op_name)
if op_name_count_dict.get(op_name) == 0:
raise ValueError("The number of operations can not be 0.")
avg_time = time / op_name_count_dict[op_name]
avg_time = time / op_name_count_dict.get(op_name)
total_time += avg_time
result_data += ("%s %s %s\n" % (op_name, str(avg_time), stream_id))
result_data += ("total op %s 0" % (str(total_time)))
@ -205,46 +249,3 @@ class OPComputeTimeParser:
self._min_cycle_counter = min_cycle_counter / 1e5 # Convert the time unit from 10ns to 1ms
return tmp_result_data
def _convert_op_time_unit(self, op_data_list, op_name_time_dict, op_name_stream_dict,
op_name_count_dict, op_name_task_dict, op_name_start_time):
"""
Calculate the execution time of operator and convert it into millisecond.
Args:
op_data_list (list): The list of operator metadata.
op_name_time_dict (dict): The mapping relation of operator name and its execution time.
op_name_stream_dict (dict): The mapping relation of operator name and its stream id.
op_name_count_dict (dict): The mapping relation of operator name and its count.
op_name_task_dict (dict): The mapping relation of operator name and its task id.
op_name_start_time (dict): The mapping relation of operator name and its start time.
"""
factor = 1e5
for item in op_data_list:
op_name = item.op_name
# Unit conversion: converting the cycle counter into ms.
op_start_time_str = str(item.cycle_counter / factor)
op_duration = item.duration / factor
op_duration_str = str(item.duration / factor)
if op_name in op_name_time_dict.keys():
op_name_time_dict[op_name] += op_duration
if item.task_id == op_name_task_dict[op_name]:
op_name_count_dict[op_name] += 1
op_name_start_time[op_name].append(
(op_start_time_str, op_duration_str)
)
else:
op_name_time_dict[op_name] = op_duration
op_name_stream_dict[op_name] = item.stream_id
op_name_task_dict[op_name] = item.task_id
op_name_count_dict[op_name] = 1
op_name_start_time[op_name] = []
op_name_start_time[op_name].append(
(op_start_time_str, op_duration_str)
)
@property
def min_cycle_counter(self):
"""Get minimum cycle counter."""
return self._min_cycle_counter

8
mindspore/python/mindspore/profiler/parser/step_trace_parser.py
View File

@ -31,6 +31,7 @@ from mindspore.profiler.common.util import combine_stream_task_id
class PointTag(Enum):
"""Initializing indexes."""
MODEL_START = 0
MODEL_END = 1
FP_START = 2
@ -352,6 +353,8 @@ class GpuStepTraceParser(BaseStepTraceParser):
except (IOError, OSError) as err:
log.warning(f'Failed to read {source_file}', err)
raise ProfilerIOException
finally:
pass
for step_num in range(num_of_step):
step_trace = {
@ -458,6 +461,7 @@ class AscendStepTraceParser(BaseStepTraceParser):
self._task_id_op_name_dict = {}
def set_task_id_op_name_dict(self, task_id_op_name_dict):
"""The operator task id matches the operator name."""
self._task_id_op_name_dict = task_id_op_name_dict
def record_point_info(self, output_path):
@ -567,7 +571,7 @@ class AscendStepTraceParser(BaseStepTraceParser):
"""Save step trace data to result."""
step_trace = {'reduce': defaultdict(list), 'start': '-'}
for ts_track in ts_tracks:
if ts_track['rptType'] != STEP_TRACE_RPT_TYPE:
if ts_track.get('rptType') != STEP_TRACE_RPT_TYPE:
continue
self._construct_step_trace(ts_track, step_trace)
@ -599,6 +603,8 @@ class AscendStepTraceParser(BaseStepTraceParser):
except (IOError, OSError) as err:
log.critical("Can not parse profiler file, open file %s failed, detail: %s.", path, str(err))
raise ProfilerIOException()
finally:
pass
log.info("Profiler found %d ts track step trace data.", len(ts_tracks))
return ts_tracks

23
mindspore/python/mindspore/profiler/profiling.py
View File

@ -51,6 +51,7 @@ INIT_OP_NAME = 'Default/InitDataSetQueue'
def deprecated(name, version):
"""Warning notices."""
msg = f"The {name} is deprecated from MindSpore {version} and will be removed in a future version."
logger.warning(msg)
@ -418,6 +419,8 @@ class Profiler:
pipeline_parser.parse()
except ProfilerException as err:
logger.warning(err.message)
finally:
pass
# Analyze minddata information
try:
@ -426,6 +429,8 @@ class Profiler:
md_analyzer.analyze()
except ProfilerException as err:
logger.warning(err.message)
finally:
pass
def _ascend_graph_analyse(self):
"""Ascend graph mode analyse."""
@ -450,6 +455,8 @@ class Profiler:
self._analyser_op_info()
except ProfilerException as err:
logger.warning(err.message)
finally:
pass
# analyse step trace info
points = None
@ -460,6 +467,8 @@ class Profiler:
points, is_training_mode_flag = self._analyse_step_trace(source_path, framework_parser)
except ProfilerException as err:
logger.warning(err.message)
finally:
pass
# analyse timeline info
try:
@ -467,6 +476,8 @@ class Profiler:
self._analyse_timeline(aicpu_data_parser, optime_parser, source_path)
except (ProfilerIOException, ProfilerFileNotFoundException, RuntimeError) as err:
logger.warning('Fail to write timeline data: %s', err)
finally:
pass
# analyse memory usage info
if self._profile_memory:
@ -475,6 +486,8 @@ class Profiler:
self._analyse_memory_usage(points)
except (ProfilerIOException, ProfilerFileNotFoundException, ProfilerRawFileException) as err:
logger.warning(err.message)
finally:
pass
# analyse hccl profiler info
if self._profile_communication:
@ -483,6 +496,8 @@ class Profiler:
self._analyse_hccl_info()
except (ProfilerIOException, ProfilerFileNotFoundException, ProfilerRawFileException) as err:
logger.warning(err.message)
finally:
pass
# get op FLOPs from aicore.data.x.slice.0 file, and compute FLOPS, write output_op_flops_x.txt
flops_parser = FlopsParser(source_path, self._output_path, op_task_dict,
@ -492,10 +507,13 @@ class Profiler:
@staticmethod
def _check_output_path(output_path):
"""Checking path validity."""
try:
output_path = validate_and_normalize_path(output_path)
except RuntimeError:
raise ProfilerPathErrorException(f'profiling data output path {output_path} is invalid.')
finally:
pass
if not os.path.isdir(output_path):
raise ProfilerDirNotFoundException(output_path)
return output_path
@ -583,9 +601,6 @@ class Profiler:
if not os.path.exists(data_path):
os.makedirs(data_path, exist_ok=True)
# add job id env through user input later
self._job_id_env = 0
self._ascend_profiler.start()
def stop(self):
@ -683,6 +698,8 @@ class Profiler:
)
except ProfilerException as err:
logger.warning(err.message)
finally:
pass
logger.warning(
'\nThe training and inference process does not support profiler currently, '