vilc
/
mindspore2022
forked from huawei/mindspore2022
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix pylint warnings for graph kernel

This commit is contained in:
zengzitao 2022-03-23 17:29:12 +08:00
parent 2934e55379
commit 315df426ac
7 changed files with 30 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
mindspore/ccsrc/common/graph_kernel/parallel_cost_model.cc
View File

@ -38,11 +38,12 @@ int ParallelCostModel::GetNodeCalAmount(const AnfNodePtr &node) const {
auto json_desc_str = json_desc.dump();
auto ret = python_adapter::CallPyFn(kGraphKernelModule, kGraphKernelGetNodeCalAmount, json_desc_str);
if (py::isinstance<py::none>(ret)) {
MS_LOG(EXCEPTION) << "CallPyFn: [" << kGraphKernelSplitFunc << "] return invalid result. input json:\n"
auto bottleneck = py::cast<int>(ret);
if (bottleneck == -1) {
MS_LOG(EXCEPTION) << "CallPyFn: [" << kGraphKernelGetNodeCalAmount << "] return invalid result. input json:\n"
<< json_desc_str;
}
return py::cast<int>(ret);
return bottleneck;
}
std::tuple<std::vector<DimInfoPtr>, int, FusionInfoPtr> ParallelCostModel::CalFuseInfo(
@ -59,7 +60,7 @@ std::tuple<std::vector<DimInfoPtr>, int, FusionInfoPtr> ParallelCostModel::CalFu
auto json_desc_str = json_desc.dump();
auto ret = python_adapter::CallPyFn(kGraphKernelModule, kGraphKernelEstimateOps, json_desc_str);
if (py::isinstance<py::none>(ret)) {
MS_LOG(EXCEPTION) << "CallPyFn: [" << kGraphKernelSplitFunc << "] return invalid result. input json:\n"
MS_LOG(EXCEPTION) << "CallPyFn: [" << kGraphKernelEstimateOps << "] return invalid result. input json:\n"
<< json_desc_str;
}

4
mindspore/python/mindspore/_extends/graph_kernel/expanders/squeeze.py
View File

@ -32,10 +32,10 @@ class Squeeze(Expander):
"""infer shape for squeeze"""
def squeeze_axis(shape, axis):
if not axis:
out_shape = [d for d in shape if d != 1]
out_shape = list(d for d in shape if d != 1)
else:
ndim = len(shape)
out_shape = [shape[i] for i in range(ndim) if not (i in axis or (i - ndim) in axis)]
out_shape = list(shape[i] for i in range(ndim) if not (i in axis or (i - ndim) in axis))
if not out_shape:
out_shape = [1]
return out_shape

6
mindspore/python/mindspore/_extends/graph_kernel/expanders/tile.py
View File

@ -25,11 +25,9 @@ class Tile(Expander):
def _get_output_shape(self):
"""Get output shape"""
shape = self.inputs[0].shape
multiples = self.attrs["multiples"]
shape = list(self.inputs[0].shape)
multiples = list(self.attrs["multiples"])
shape = list(shape)
multiples = list(multiples)
diff_len = len(multiples) - len(shape)
if diff_len < 0:
raise GKException("For 'Tile', dimensions of attr 'multiples' should be greater than or equal to "

30
mindspore/python/mindspore/_extends/graph_kernel/model/graph_parallel.py
View File

@ -41,7 +41,7 @@ class ScheduleAnalyzer:
self.block_weight = 0
_, outputs = graph.deduce_parameters()
self.ops = graph.ops
self.dom_op = [out.op for out in outputs]
self.dom_op = list(out.op for out in outputs)
@staticmethod
def prod(shape):
@ -60,7 +60,7 @@ class ScheduleAnalyzer:
def injective_analyze(self):
"""analyze injective case"""
const_size = max([self.prod(op.output.shape) for op in self.dom_op])
const_size = max((self.prod(op.output.shape) for op in self.dom_op))
const_size = (const_size + self.MAX_NUM_THREADS -
1) // self.MAX_NUM_THREADS * self.MAX_NUM_THREADS
@ -114,7 +114,7 @@ class ScheduleAnalyzer:
if size > space:
space = size
return space
space = max([_cal_default_space(op) for op in self.dom_op])
space = max((_cal_default_space(op) for op in self.dom_op))
# each sm least 4 wrap
block = (space + (self.WRAP_SIZE * 4) - 1) // (self.WRAP_SIZE * 4)
@ -125,7 +125,7 @@ class ScheduleAnalyzer:
"""analyze ops"""
def _ops_type(ops, dom_op):
have_reduce = any(
[PrimLib.iter_type(op) == PrimLib.REDUCE for op in ops])
(PrimLib.iter_type(op) == PrimLib.REDUCE for op in ops))
if have_reduce:
return True
return PrimLib.iter_type(dom_op[0])
@ -167,8 +167,8 @@ class ScheduleAnalyzer:
classes (list[list[int]]): The list of clusters. Each cluster is a list of indices.
"""
def _cal_mean(classes):
class_datas = [[data[cid] for cid in cls] for cls in classes]
return [sum(cls) / len(cls) if cls else float('inf') for cls in class_datas]
class_datas = list(list(data[cid] for cid in cls) for cls in classes)
return list(sum(cls) / len(cls) if cls else float('inf') for cls in class_datas)
def _cal_distance(a, b):
return abs(a - b)
@ -180,7 +180,7 @@ class ScheduleAnalyzer:
return False
if len(data) < class_n:
return None
return []
classes = []
for i, _ in enumerate(data):
if i in exclude_id:
@ -190,7 +190,7 @@ class ScheduleAnalyzer:
classes.append([i])
changed = True
while changed:
new_classes = [[] for cls in classes]
new_classes = list([] for cls in classes)
means = _cal_mean(classes)
for idx, d in enumerate(data):
if idx in exclude_id:
@ -216,18 +216,18 @@ class ScheduleAnalyzer:
def _take_second(elem):
return elem[1]
simple_indicators = [_simple_factor(b, s)
for b, s in zip(blocks, op_sizes)]
simple_indicators = list(_simple_factor(b, s)
for b, s in zip(blocks, op_sizes))
# 2 classes, one heavy, the other light
classes = ScheduleAnalyzer.k_mean(simple_indicators, 2, exclude_id)
if not classes:
return []
means = [sum([simple_indicators[idx] for idx in cls]) /
len(cls) if cls else float('inf') for cls in classes]
means = list(sum([simple_indicators[idx] for idx in cls]) /
len(cls) if cls else float('inf') for cls in classes)
# The target two clusters should be a heavy one and a light one.
# The light one maybe suitable to run with pipeline optimized.
classes_infos = [[cls, m] for cls, m in zip(classes, means)]
classes_infos = list([cls, m] for cls, m in zip(classes, means))
classes_infos.sort(key=_take_second)
pipeline_target = None
for ci in classes_infos:
@ -284,7 +284,7 @@ def block_parallel_estimate(graphs):
if not s.suitable_to_pipeline():
exclude_gid.append(gid)
if sum_block > ScheduleAnalyzer.MAX_SM * 32:
return ParalGain("none", sum_weight, 0, [0 for _ in graphs], None)
return ParalGain("none", sum_weight, 0, list(0 for _ in graphs), None)
fusion_type, type_info = ScheduleAnalyzer.fusion_consult(blocks, op_sizes, tuple(exclude_gid))
return ParalGain(fusion_type, max_weight, sum_weight - max_weight, blocks, type_info)
@ -296,6 +296,6 @@ def parallel_estimate(graphs, target):
fusion_type = "block_fusion"
type_info = None
fake_estimate = 1000
fake_blocks = [1 for g in graphs]
fake_blocks = list(1 for g in graphs)
return ParalGain(fusion_type, fake_estimate, fake_estimate, fake_blocks, type_info)
return block_parallel_estimate(graphs)

2
mindspore/python/mindspore/_extends/graph_kernel/parallel_estimate.py
View File

@ -55,6 +55,6 @@ def estimate_calculation_amount(json_str):
return estimation.bottleneck
except jd.JSONDecodeError:
logger.error(traceback.format_exc())
return None
return -1
finally:
pass

6
mindspore/python/mindspore/_extends/graph_kernel/splitter.py
View File

@ -1,4 +1,4 @@
# Copyright 2020-2021 Huawei Technologies Co., Ltd
# Copyright 2020-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -41,13 +41,13 @@ def split_with_json(json_str, flags_str):
return json.dumps(result)
except jd.JSONDecodeError:
logger.error(traceback.format_exc())
return None
return ""
def _reset_graphmode_for_inplaceassign(graph_list, graph_mode):
"""Operator with InplaceAssign should always be composite op"""
for i, g in enumerate(graph_list):
if any([op['name'] == 'InplaceAssign' for op in g['op_desc']]):
if any((op['name'] == 'InplaceAssign' for op in g['op_desc'])):
graph_mode[i] = 'composite'

4
mindspore/python/mindspore/_extends/parallel_compile/akg_compiler/akg_process.py
View File

@ -98,7 +98,7 @@ class AkgProcess:
process_num = 1
max_proc_num = 16
self.process_num = min([cpu_count(), max_proc_num, process_num])
self.args = [[] for _ in range(self.process_num)]
self.args = list([] for _ in range(self.process_num))
self.wait_time = wait_time
self.platform = platform
self.argc = 0
@ -111,7 +111,7 @@ class AkgProcess:
"""
if self.argc == 0:
raise ValueError("json must be not null")
args = [(arg, attrs) for arg in self.args]
args = list((arg, attrs) for arg in self.args)
if self.platform == "ASCEND":
with Pool(processes=self.process_num) as pool:
res = pool.starmap_async(_compile_akg_task_ascend, args)