From 315df426ac5a30a962e23f000dce7e1e273b7305 Mon Sep 17 00:00:00 2001 From: zengzitao Date: Wed, 23 Mar 2022 17:29:12 +0800 Subject: [PATCH] fix pylint warnings for graph kernel --- .../graph_kernel/parallel_cost_model.cc | 9 +++--- .../graph_kernel/expanders/squeeze.py | 4 +-- .../_extends/graph_kernel/expanders/tile.py | 6 ++-- .../graph_kernel/model/graph_parallel.py | 30 +++++++++---------- .../graph_kernel/parallel_estimate.py | 2 +- .../_extends/graph_kernel/splitter.py | 6 ++-- .../akg_compiler/akg_process.py | 4 +-- 7 files changed, 30 insertions(+), 31 deletions(-) diff --git a/mindspore/ccsrc/common/graph_kernel/parallel_cost_model.cc b/mindspore/ccsrc/common/graph_kernel/parallel_cost_model.cc index fef258376b1..5694686bf79 100644 --- a/mindspore/ccsrc/common/graph_kernel/parallel_cost_model.cc +++ b/mindspore/ccsrc/common/graph_kernel/parallel_cost_model.cc @@ -38,11 +38,12 @@ int ParallelCostModel::GetNodeCalAmount(const AnfNodePtr &node) const { auto json_desc_str = json_desc.dump(); auto ret = python_adapter::CallPyFn(kGraphKernelModule, kGraphKernelGetNodeCalAmount, json_desc_str); - if (py::isinstance(ret)) { - MS_LOG(EXCEPTION) << "CallPyFn: [" << kGraphKernelSplitFunc << "] return invalid result. input json:\n" + auto bottleneck = py::cast(ret); + if (bottleneck == -1) { + MS_LOG(EXCEPTION) << "CallPyFn: [" << kGraphKernelGetNodeCalAmount << "] return invalid result. input json:\n" << json_desc_str; } - return py::cast(ret); + return bottleneck; } std::tuple, int, FusionInfoPtr> ParallelCostModel::CalFuseInfo( @@ -59,7 +60,7 @@ std::tuple, int, FusionInfoPtr> ParallelCostModel::CalFu auto json_desc_str = json_desc.dump(); auto ret = python_adapter::CallPyFn(kGraphKernelModule, kGraphKernelEstimateOps, json_desc_str); if (py::isinstance(ret)) { - MS_LOG(EXCEPTION) << "CallPyFn: [" << kGraphKernelSplitFunc << "] return invalid result. input json:\n" + MS_LOG(EXCEPTION) << "CallPyFn: [" << kGraphKernelEstimateOps << "] return invalid result. input json:\n" << json_desc_str; } diff --git a/mindspore/python/mindspore/_extends/graph_kernel/expanders/squeeze.py b/mindspore/python/mindspore/_extends/graph_kernel/expanders/squeeze.py index aa7bb50b88d..a5e631b5101 100644 --- a/mindspore/python/mindspore/_extends/graph_kernel/expanders/squeeze.py +++ b/mindspore/python/mindspore/_extends/graph_kernel/expanders/squeeze.py @@ -32,10 +32,10 @@ class Squeeze(Expander): """infer shape for squeeze""" def squeeze_axis(shape, axis): if not axis: - out_shape = [d for d in shape if d != 1] + out_shape = list(d for d in shape if d != 1) else: ndim = len(shape) - out_shape = [shape[i] for i in range(ndim) if not (i in axis or (i - ndim) in axis)] + out_shape = list(shape[i] for i in range(ndim) if not (i in axis or (i - ndim) in axis)) if not out_shape: out_shape = [1] return out_shape diff --git a/mindspore/python/mindspore/_extends/graph_kernel/expanders/tile.py b/mindspore/python/mindspore/_extends/graph_kernel/expanders/tile.py index 27009b1d372..918b7486324 100644 --- a/mindspore/python/mindspore/_extends/graph_kernel/expanders/tile.py +++ b/mindspore/python/mindspore/_extends/graph_kernel/expanders/tile.py @@ -25,11 +25,9 @@ class Tile(Expander): def _get_output_shape(self): """Get output shape""" - shape = self.inputs[0].shape - multiples = self.attrs["multiples"] + shape = list(self.inputs[0].shape) + multiples = list(self.attrs["multiples"]) - shape = list(shape) - multiples = list(multiples) diff_len = len(multiples) - len(shape) if diff_len < 0: raise GKException("For 'Tile', dimensions of attr 'multiples' should be greater than or equal to " diff --git a/mindspore/python/mindspore/_extends/graph_kernel/model/graph_parallel.py b/mindspore/python/mindspore/_extends/graph_kernel/model/graph_parallel.py index 8090b427937..be948869724 100644 --- a/mindspore/python/mindspore/_extends/graph_kernel/model/graph_parallel.py +++ b/mindspore/python/mindspore/_extends/graph_kernel/model/graph_parallel.py @@ -41,7 +41,7 @@ class ScheduleAnalyzer: self.block_weight = 0 _, outputs = graph.deduce_parameters() self.ops = graph.ops - self.dom_op = [out.op for out in outputs] + self.dom_op = list(out.op for out in outputs) @staticmethod def prod(shape): @@ -60,7 +60,7 @@ class ScheduleAnalyzer: def injective_analyze(self): """analyze injective case""" - const_size = max([self.prod(op.output.shape) for op in self.dom_op]) + const_size = max((self.prod(op.output.shape) for op in self.dom_op)) const_size = (const_size + self.MAX_NUM_THREADS - 1) // self.MAX_NUM_THREADS * self.MAX_NUM_THREADS @@ -114,7 +114,7 @@ class ScheduleAnalyzer: if size > space: space = size return space - space = max([_cal_default_space(op) for op in self.dom_op]) + space = max((_cal_default_space(op) for op in self.dom_op)) # each sm least 4 wrap block = (space + (self.WRAP_SIZE * 4) - 1) // (self.WRAP_SIZE * 4) @@ -125,7 +125,7 @@ class ScheduleAnalyzer: """analyze ops""" def _ops_type(ops, dom_op): have_reduce = any( - [PrimLib.iter_type(op) == PrimLib.REDUCE for op in ops]) + (PrimLib.iter_type(op) == PrimLib.REDUCE for op in ops)) if have_reduce: return True return PrimLib.iter_type(dom_op[0]) @@ -167,8 +167,8 @@ class ScheduleAnalyzer: classes (list[list[int]]): The list of clusters. Each cluster is a list of indices. """ def _cal_mean(classes): - class_datas = [[data[cid] for cid in cls] for cls in classes] - return [sum(cls) / len(cls) if cls else float('inf') for cls in class_datas] + class_datas = list(list(data[cid] for cid in cls) for cls in classes) + return list(sum(cls) / len(cls) if cls else float('inf') for cls in class_datas) def _cal_distance(a, b): return abs(a - b) @@ -180,7 +180,7 @@ class ScheduleAnalyzer: return False if len(data) < class_n: - return None + return [] classes = [] for i, _ in enumerate(data): if i in exclude_id: @@ -190,7 +190,7 @@ class ScheduleAnalyzer: classes.append([i]) changed = True while changed: - new_classes = [[] for cls in classes] + new_classes = list([] for cls in classes) means = _cal_mean(classes) for idx, d in enumerate(data): if idx in exclude_id: @@ -216,18 +216,18 @@ class ScheduleAnalyzer: def _take_second(elem): return elem[1] - simple_indicators = [_simple_factor(b, s) - for b, s in zip(blocks, op_sizes)] + simple_indicators = list(_simple_factor(b, s) + for b, s in zip(blocks, op_sizes)) # 2 classes, one heavy, the other light classes = ScheduleAnalyzer.k_mean(simple_indicators, 2, exclude_id) if not classes: return [] - means = [sum([simple_indicators[idx] for idx in cls]) / - len(cls) if cls else float('inf') for cls in classes] + means = list(sum([simple_indicators[idx] for idx in cls]) / + len(cls) if cls else float('inf') for cls in classes) # The target two clusters should be a heavy one and a light one. # The light one maybe suitable to run with pipeline optimized. - classes_infos = [[cls, m] for cls, m in zip(classes, means)] + classes_infos = list([cls, m] for cls, m in zip(classes, means)) classes_infos.sort(key=_take_second) pipeline_target = None for ci in classes_infos: @@ -284,7 +284,7 @@ def block_parallel_estimate(graphs): if not s.suitable_to_pipeline(): exclude_gid.append(gid) if sum_block > ScheduleAnalyzer.MAX_SM * 32: - return ParalGain("none", sum_weight, 0, [0 for _ in graphs], None) + return ParalGain("none", sum_weight, 0, list(0 for _ in graphs), None) fusion_type, type_info = ScheduleAnalyzer.fusion_consult(blocks, op_sizes, tuple(exclude_gid)) return ParalGain(fusion_type, max_weight, sum_weight - max_weight, blocks, type_info) @@ -296,6 +296,6 @@ def parallel_estimate(graphs, target): fusion_type = "block_fusion" type_info = None fake_estimate = 1000 - fake_blocks = [1 for g in graphs] + fake_blocks = list(1 for g in graphs) return ParalGain(fusion_type, fake_estimate, fake_estimate, fake_blocks, type_info) return block_parallel_estimate(graphs) diff --git a/mindspore/python/mindspore/_extends/graph_kernel/parallel_estimate.py b/mindspore/python/mindspore/_extends/graph_kernel/parallel_estimate.py index 48b5b6261af..c3b307cf109 100644 --- a/mindspore/python/mindspore/_extends/graph_kernel/parallel_estimate.py +++ b/mindspore/python/mindspore/_extends/graph_kernel/parallel_estimate.py @@ -55,6 +55,6 @@ def estimate_calculation_amount(json_str): return estimation.bottleneck except jd.JSONDecodeError: logger.error(traceback.format_exc()) - return None + return -1 finally: pass diff --git a/mindspore/python/mindspore/_extends/graph_kernel/splitter.py b/mindspore/python/mindspore/_extends/graph_kernel/splitter.py index 027a588c22b..e3da3ec71d4 100644 --- a/mindspore/python/mindspore/_extends/graph_kernel/splitter.py +++ b/mindspore/python/mindspore/_extends/graph_kernel/splitter.py @@ -1,4 +1,4 @@ -# Copyright 2020-2021 Huawei Technologies Co., Ltd +# Copyright 2020-2022 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -41,13 +41,13 @@ def split_with_json(json_str, flags_str): return json.dumps(result) except jd.JSONDecodeError: logger.error(traceback.format_exc()) - return None + return "" def _reset_graphmode_for_inplaceassign(graph_list, graph_mode): """Operator with InplaceAssign should always be composite op""" for i, g in enumerate(graph_list): - if any([op['name'] == 'InplaceAssign' for op in g['op_desc']]): + if any((op['name'] == 'InplaceAssign' for op in g['op_desc'])): graph_mode[i] = 'composite' diff --git a/mindspore/python/mindspore/_extends/parallel_compile/akg_compiler/akg_process.py b/mindspore/python/mindspore/_extends/parallel_compile/akg_compiler/akg_process.py index ebb15d731a6..bc505dbb859 100644 --- a/mindspore/python/mindspore/_extends/parallel_compile/akg_compiler/akg_process.py +++ b/mindspore/python/mindspore/_extends/parallel_compile/akg_compiler/akg_process.py @@ -98,7 +98,7 @@ class AkgProcess: process_num = 1 max_proc_num = 16 self.process_num = min([cpu_count(), max_proc_num, process_num]) - self.args = [[] for _ in range(self.process_num)] + self.args = list([] for _ in range(self.process_num)) self.wait_time = wait_time self.platform = platform self.argc = 0 @@ -111,7 +111,7 @@ class AkgProcess: """ if self.argc == 0: raise ValueError("json must be not null") - args = [(arg, attrs) for arg in self.args] + args = list((arg, attrs) for arg in self.args) if self.platform == "ASCEND": with Pool(processes=self.process_num) as pool: res = pool.starmap_async(_compile_akg_task_ascend, args)