diff --git a/mindspore/python/mindspore/offline_debug/convert_async.py b/mindspore/python/mindspore/offline_debug/convert_async.py index 5efd6940672..fb4ba752f8d 100644 --- a/mindspore/python/mindspore/offline_debug/convert_async.py +++ b/mindspore/python/mindspore/offline_debug/convert_async.py @@ -146,6 +146,49 @@ class AsyncDumpConverter: self.output_path, 'convert_failed_file_list.txt') self.clear_failed_list_file() + @staticmethod + def _get_file_list(files, convert_obj): + """ + Process to get file lists in multi_process. + """ + multi_process_file_list = [] + big_file_list = [] + max_file_size = 0 + if hasattr(convert_obj, 'multi_process'): + max_file_size = getattr(convert_obj.multi_process, 'get_max_file_size')() + else: + max_file_size = getattr(convert_obj, '_get_max_file_size')() + for cur_file in files: + cur_path = cur_file + if os.path.isfile(cur_path): + if os.path.getsize(cur_path) > max_file_size: + big_file_list.append(cur_path) + else: + multi_process_file_list.append(cur_path) + return multi_process_file_list, big_file_list + + @staticmethod + def _process_func(convert_obj): + """ + get function to process format transformation. + """ + if hasattr(convert_obj, '_convert_format_for_one_file'): + func = getattr(convert_obj, '_convert_format_for_one_file') + else: + func = getattr(convert_obj, 'convert_format_for_one_file') + return func + + @staticmethod + def _result_callback_func(convert_obj): + """ + get result callback function. + """ + if hasattr(convert_obj, 'multi_process'): + func = getattr(convert_obj.multi_process, '_handle_result_callback') + else: + func = getattr(convert_obj, '_handle_result_callback') + return func + def clear_failed_list_file(self): """ Remove existing failed txt file. @@ -201,26 +244,6 @@ class AsyncDumpConverter: + self.failed_file_path + '".') return return_code - def _get_file_list(self, files, convert_obj): - """ - Process to get file lists in multi_process. - """ - multi_process_file_list = [] - big_file_list = [] - max_file_size = 0 - if hasattr(convert_obj, 'multi_process'): - max_file_size = getattr(convert_obj.multi_process, 'get_max_file_size')() - else: - max_file_size = getattr(convert_obj, '_get_max_file_size')() - for cur_file in files: - cur_path = cur_file - if os.path.isfile(cur_path): - if os.path.getsize(cur_path) > max_file_size: - big_file_list.append(cur_path) - else: - multi_process_file_list.append(cur_path) - return multi_process_file_list, big_file_list - def _process_in_single_process(self, big_file_list, convert_obj): """ Process big file in single process. @@ -255,23 +278,3 @@ class AsyncDumpConverter: if cur_ret != self.convert_tool.compare_none_error: return cur_ret return self.convert_tool.compare_none_error - - def _process_func(self, convert_obj): - """ - get function to process format transformation. - """ - if hasattr(convert_obj, '_convert_format_for_one_file'): - func = getattr(convert_obj, '_convert_format_for_one_file') - else: - func = getattr(convert_obj, 'convert_format_for_one_file') - return func - - def _result_callback_func(self, convert_obj): - """ - get result callback function. - """ - if hasattr(convert_obj, 'multi_process'): - func = getattr(convert_obj.multi_process, '_handle_result_callback') - else: - func = getattr(convert_obj, '_handle_result_callback') - return func diff --git a/mindspore/python/mindspore/offline_debug/dbg_services.py b/mindspore/python/mindspore/offline_debug/dbg_services.py index 24546c63eb0..8039b145967 100644 --- a/mindspore/python/mindspore/offline_debug/dbg_services.py +++ b/mindspore/python/mindspore/offline_debug/dbg_services.py @@ -67,31 +67,8 @@ class DbgServices: self.version = self.dbg_instance.GetVersion() self.initialized = False - @check_initialize - def initialize(self, net_name, is_sync_mode=True, max_mem_usage=0): - """ - Initialize Debug Service. - - Args: - net_name (str): Network name. - is_sync_mode (bool): Whether to process synchronous or asynchronous dump files mode - (default: True (synchronous)). - max_mem_usage (int): Maximum memory size of the debugger internal tensor cache in Megabytes(MB), - (default: 0 (disable memory restriction feature)). - - Returns: - Initialized Debug Service instance. - - Examples: - >>> from mindspore.ccsrc.debug.debugger.offline_debug import dbg_services - >>> d = dbg_services.DbgServices(dump_file_path="dump_file_path") - >>> d_init = d.initialize(net_name="network name", is_sync_mode=True, max_mem_usage=4096) - """ - logger.info("in Python Initialize dump_file_path %s", self.dump_file_path) - self.initialized = True - return self.dbg_instance.Initialize(net_name, self.dump_file_path, is_sync_mode, max_mem_usage) - - def transform_check_node_list(self, info_name, info_param, node_name, check_node_list): + @staticmethod + def transform_check_node_list(info_name, info_param, node_name, check_node_list): """ Transforming check_node_list based on info_name and info_param. @@ -124,6 +101,30 @@ class DbgServices: check_node_list[node_name][info_name] = list(map(str, info_param)) return check_node_list + @check_initialize + def initialize(self, net_name, is_sync_mode=True, max_mem_usage=0): + """ + Initialize Debug Service. + + Args: + net_name (str): Network name. + is_sync_mode (bool): Whether to process synchronous or asynchronous dump files mode + (default: True (synchronous)). + max_mem_usage (int): Maximum memory size of the debugger internal tensor cache in Megabytes(MB), + (default: 0 (disable memory restriction feature)). + + Returns: + Initialized Debug Service instance. + + Examples: + >>> from mindspore.ccsrc.debug.debugger.offline_debug import dbg_services + >>> d = dbg_services.DbgServices(dump_file_path="dump_file_path") + >>> d_init = d.initialize(net_name="network name", is_sync_mode=True, max_mem_usage=4096) + """ + logger.info("in Python Initialize dump_file_path %s", self.dump_file_path) + self.initialized = True + return self.dbg_instance.Initialize(net_name, self.dump_file_path, is_sync_mode, max_mem_usage) + @check_initialize_done @check_add_watchpoint def add_watchpoint(self, watchpoint_id, watch_condition, check_node_list, parameter_list): @@ -251,6 +252,12 @@ class DbgServices: @check_initialize_done def check_watchpoint_progress(self): + """ + Returning the progress percentage of checking watchpoint. + + Returns: + float, progress percentage. + """ progress_percentage = self.dbg_instance.CheckWatchpointProgress() return progress_percentage diff --git a/mindspore/python/mindspore/offline_debug/mi_validator_helpers.py b/mindspore/python/mindspore/offline_debug/mi_validator_helpers.py index c346fe4d214..690ed9d981d 100644 --- a/mindspore/python/mindspore/offline_debug/mi_validator_helpers.py +++ b/mindspore/python/mindspore/offline_debug/mi_validator_helpers.py @@ -25,12 +25,14 @@ UINT64_MIN = 0 def pad_arg_name(arg_name): + """Add a space for arg_name.""" if arg_name != "": arg_name = arg_name + " " return arg_name def check_value(arg, valid_range, arg_name=""): + """Check the value of arg is in a valid range.""" arg_name = pad_arg_name(arg_name) if arg < valid_range[0] or arg > valid_range[1]: raise ValueError( @@ -39,21 +41,25 @@ def check_value(arg, valid_range, arg_name=""): def check_uint32(arg, arg_name=""): + """Check arg type is uint32.""" type_check(arg, (int,), arg_name) check_value(arg, [UINT32_MIN, UINT32_MAX]) def check_uint64(arg, arg_name=""): + """Check arg type is uint64.""" type_check(arg, (int,), arg_name) check_value(arg, [UINT64_MIN, UINT64_MAX]) def check_iteration(arg, arg_name=""): + """Check arg is in a valid range.""" type_check(arg, (int,), arg_name) check_value(arg, [-1, UINT64_MAX]) def check_dir(dataset_dir): + """Check the dataset_dir is a valid dir.""" if not os.path.isdir(dataset_dir) or not os.access(dataset_dir, os.R_OK): raise ValueError("The folder {} does not exist or permission denied!".format(dataset_dir))