vilc
/
mindspore2022
forked from huawei/mindspore2022
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

!31791 [CodeClean]Code clean for file of offline_debug 

Merge pull request !31791 from maoyaomin/mym_fix
This commit is contained in:
i-robot 2022-03-23 15:19:01 +00:00 committed by Gitee
parent aeb3aa915a 151aba6cf4
commit 2934e55379
No known key found for this signature in database
GPG Key ID: 173E9B9CA92EEF8F
3 changed files with 81 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

83
mindspore/python/mindspore/offline_debug/convert_async.py
View File

@ -146,6 +146,49 @@ class AsyncDumpConverter:
self.output_path, 'convert_failed_file_list.txt')
self.clear_failed_list_file()
@staticmethod
def _get_file_list(files, convert_obj):
"""
Process to get file lists in multi_process.
"""
multi_process_file_list = []
big_file_list = []
max_file_size = 0
if hasattr(convert_obj, 'multi_process'):
max_file_size = getattr(convert_obj.multi_process, 'get_max_file_size')()
else:
max_file_size = getattr(convert_obj, '_get_max_file_size')()
for cur_file in files:
cur_path = cur_file
if os.path.isfile(cur_path):
if os.path.getsize(cur_path) > max_file_size:
big_file_list.append(cur_path)
else:
multi_process_file_list.append(cur_path)
return multi_process_file_list, big_file_list
@staticmethod
def _process_func(convert_obj):
"""
get function to process format transformation.
"""
if hasattr(convert_obj, '_convert_format_for_one_file'):
func = getattr(convert_obj, '_convert_format_for_one_file')
else:
func = getattr(convert_obj, 'convert_format_for_one_file')
return func
@staticmethod
def _result_callback_func(convert_obj):
"""
get result callback function.
"""
if hasattr(convert_obj, 'multi_process'):
func = getattr(convert_obj.multi_process, '_handle_result_callback')
else:
func = getattr(convert_obj, '_handle_result_callback')
return func
def clear_failed_list_file(self):
"""
Remove existing failed txt file.
@ -201,26 +244,6 @@ class AsyncDumpConverter:
+ self.failed_file_path + '".')
return return_code
def _get_file_list(self, files, convert_obj):
"""
Process to get file lists in multi_process.
"""
multi_process_file_list = []
big_file_list = []
max_file_size = 0
if hasattr(convert_obj, 'multi_process'):
max_file_size = getattr(convert_obj.multi_process, 'get_max_file_size')()
else:
max_file_size = getattr(convert_obj, '_get_max_file_size')()
for cur_file in files:
cur_path = cur_file
if os.path.isfile(cur_path):
if os.path.getsize(cur_path) > max_file_size:
big_file_list.append(cur_path)
else:
multi_process_file_list.append(cur_path)
return multi_process_file_list, big_file_list
def _process_in_single_process(self, big_file_list, convert_obj):
"""
Process big file in single process.
@ -255,23 +278,3 @@ class AsyncDumpConverter:
if cur_ret != self.convert_tool.compare_none_error:
return cur_ret
return self.convert_tool.compare_none_error
def _process_func(self, convert_obj):
"""
get function to process format transformation.
"""
if hasattr(convert_obj, '_convert_format_for_one_file'):
func = getattr(convert_obj, '_convert_format_for_one_file')
else:
func = getattr(convert_obj, 'convert_format_for_one_file')
return func
def _result_callback_func(self, convert_obj):
"""
get result callback function.
"""
if hasattr(convert_obj, 'multi_process'):
func = getattr(convert_obj.multi_process, '_handle_result_callback')
else:
func = getattr(convert_obj, '_handle_result_callback')
return func

57
mindspore/python/mindspore/offline_debug/dbg_services.py
View File

@ -67,31 +67,8 @@ class DbgServices:
self.version = self.dbg_instance.GetVersion()
self.initialized = False
@check_initialize
def initialize(self, net_name, is_sync_mode=True, max_mem_usage=0):
"""
Initialize Debug Service.
Args:
net_name (str): Network name.
is_sync_mode (bool): Whether to process synchronous or asynchronous dump files mode
(default: True (synchronous)).
max_mem_usage (int): Maximum memory size of the debugger internal tensor cache in Megabytes(MB),
(default: 0 (disable memory restriction feature)).
Returns:
Initialized Debug Service instance.
Examples:
>>> from mindspore.ccsrc.debug.debugger.offline_debug import dbg_services
>>> d = dbg_services.DbgServices(dump_file_path="dump_file_path")
>>> d_init = d.initialize(net_name="network name", is_sync_mode=True, max_mem_usage=4096)
"""
logger.info("in Python Initialize dump_file_path %s", self.dump_file_path)
self.initialized = True
return self.dbg_instance.Initialize(net_name, self.dump_file_path, is_sync_mode, max_mem_usage)
def transform_check_node_list(self, info_name, info_param, node_name, check_node_list):
@staticmethod
def transform_check_node_list(info_name, info_param, node_name, check_node_list):
"""
Transforming check_node_list based on info_name and info_param.
@ -124,6 +101,30 @@ class DbgServices:
check_node_list[node_name][info_name] = list(map(str, info_param))
return check_node_list
@check_initialize
def initialize(self, net_name, is_sync_mode=True, max_mem_usage=0):
"""
Initialize Debug Service.
Args:
net_name (str): Network name.
is_sync_mode (bool): Whether to process synchronous or asynchronous dump files mode
(default: True (synchronous)).
max_mem_usage (int): Maximum memory size of the debugger internal tensor cache in Megabytes(MB),
(default: 0 (disable memory restriction feature)).
Returns:
Initialized Debug Service instance.
Examples:
>>> from mindspore.ccsrc.debug.debugger.offline_debug import dbg_services
>>> d = dbg_services.DbgServices(dump_file_path="dump_file_path")
>>> d_init = d.initialize(net_name="network name", is_sync_mode=True, max_mem_usage=4096)
"""
logger.info("in Python Initialize dump_file_path %s", self.dump_file_path)
self.initialized = True
return self.dbg_instance.Initialize(net_name, self.dump_file_path, is_sync_mode, max_mem_usage)
@check_initialize_done
@check_add_watchpoint
def add_watchpoint(self, watchpoint_id, watch_condition, check_node_list, parameter_list):
@ -251,6 +252,12 @@ class DbgServices:
@check_initialize_done
def check_watchpoint_progress(self):
"""
Returning the progress percentage of checking watchpoint.
Returns:
float, progress percentage.
"""
progress_percentage = self.dbg_instance.CheckWatchpointProgress()
return progress_percentage

6
mindspore/python/mindspore/offline_debug/mi_validator_helpers.py
View File

@ -25,12 +25,14 @@ UINT64_MIN = 0
def pad_arg_name(arg_name):
"""Add a space for arg_name."""
if arg_name != "":
arg_name = arg_name + " "
return arg_name
def check_value(arg, valid_range, arg_name=""):
"""Check the value of arg is in a valid range."""
arg_name = pad_arg_name(arg_name)
if arg < valid_range[0] or arg > valid_range[1]:
raise ValueError(
@ -39,21 +41,25 @@ def check_value(arg, valid_range, arg_name=""):
def check_uint32(arg, arg_name=""):
"""Check arg type is uint32."""
type_check(arg, (int,), arg_name)
check_value(arg, [UINT32_MIN, UINT32_MAX])
def check_uint64(arg, arg_name=""):
"""Check arg type is uint64."""
type_check(arg, (int,), arg_name)
check_value(arg, [UINT64_MIN, UINT64_MAX])
def check_iteration(arg, arg_name=""):
"""Check arg is in a valid range."""
type_check(arg, (int,), arg_name)
check_value(arg, [-1, UINT64_MAX])
def check_dir(dataset_dir):
"""Check the dataset_dir is a valid dir."""
if not os.path.isdir(dataset_dir) or not os.access(dataset_dir, os.R_OK):
raise ValueError("The folder {} does not exist or permission denied!".format(dataset_dir))