From d2dc22ff71b9d74e2a6fe8c0487c688c6ed325b7 Mon Sep 17 00:00:00 2001 From: yao_yf Date: Mon, 21 Jun 2021 16:58:56 +0800 Subject: [PATCH] parallel_weight_init_adapt_pipeline_increment_predict --- mindspore/ccsrc/frontend/parallel/context.cc | 7 ++++++- mindspore/ccsrc/frontend/parallel/context.h | 1 + mindspore/common/api.py | 6 +++--- mindspore/nn/cell.py | 14 ++++++++++++-- 4 files changed, 22 insertions(+), 6 deletions(-) diff --git a/mindspore/ccsrc/frontend/parallel/context.cc b/mindspore/ccsrc/frontend/parallel/context.cc index a31481b6571..59453fe72c7 100644 --- a/mindspore/ccsrc/frontend/parallel/context.cc +++ b/mindspore/ccsrc/frontend/parallel/context.cc @@ -197,7 +197,12 @@ void ParallelContext::ParallelParameterContextInitShape(const FuncGraphPtr &func if (!func_graph->has_flag(AUTO_PARALLEL)) { return; } - + if (func_graph->has_flag(IS_FIRST_ITERATION)) { + param_shapes.clear(); + init_param_shape_ = true; + MS_LOG(INFO) << "Init the parameter shape dict in increment predict with two graph"; + return; + } if (!func_graph->has_flag(TRAINING)) { init_param_shape_ = false; MS_LOG(INFO) << "In parallel evaluation or prediction, may be need to restore the parameter shape"; diff --git a/mindspore/ccsrc/frontend/parallel/context.h b/mindspore/ccsrc/frontend/parallel/context.h index bb05e00f098..78b9136b26b 100644 --- a/mindspore/ccsrc/frontend/parallel/context.h +++ b/mindspore/ccsrc/frontend/parallel/context.h @@ -50,6 +50,7 @@ constexpr char ALL_GROUP_PARALLEL[] = "all_group_parallel"; constexpr char SAME_SERVER_GROUP_PARALLEL[] = "same_server_group_parallel"; constexpr char NO_GROUP_PARALLEL[] = "no_group_parallel"; +constexpr char IS_FIRST_ITERATION[] = "is_first_iteration"; class ParallelContext { public: ~ParallelContext() = default; diff --git a/mindspore/common/api.py b/mindspore/common/api.py index 7be9313cf65..e8e187d9607 100644 --- a/mindspore/common/api.py +++ b/mindspore/common/api.py @@ -564,10 +564,10 @@ class _Executor: return obj.parameter_layout_dict = self._executor.get_parameter_layout(phase) - if _get_pipeline_stages() > 1: - obj.parallel_parameter_name_list = self._executor.get_parallel_parameter_name_list(phase) - obj.remove_redundant_parameters() + obj.parallel_parameter_name_list = self._executor.get_parallel_parameter_name_list(phase) replace = obj.init_parameters_data(auto_parallel_mode=True) + if _get_pipeline_stages() > 1 and (not hasattr(obj, "is_first_iteration") or not obj.is_first_iteration): + obj.remove_redundant_parameters() if not context.get_context("enable_debug_runtime") or context.get_context("enable_ge"): obj.load_parameter_slice(None) diff --git a/mindspore/nn/cell.py b/mindspore/nn/cell.py index 9e2da362984..c2ccd9f2936 100755 --- a/mindspore/nn/cell.py +++ b/mindspore/nn/cell.py @@ -782,14 +782,24 @@ class Cell(Cell_): for _, cell in cells: params = cell._params.items() for param_name, param in params: - cell._params[param_name] = _updata(param) + if not auto_parallel_mode: + cell._params[param_name] = _updata(param) + continue + if param.name in self.parallel_parameter_name_list: + cell._params[param_name] = _updata(param) cell_dict = cell.__dict__ for key in cell_dict: if isinstance(cell_dict[key], ParameterTuple): param_tuple = cell_dict[key] new_param_tuple = [] for param in param_tuple: - new_param_tuple.append(_updata(param)) + if not auto_parallel_mode: + new_param_tuple.append(_updata(param)) + continue + if param.name in self.parallel_parameter_name_list: + new_param_tuple.append(_updata(param)) + else: + new_param_tuple.append(param) cell.__dict__[key] = ParameterTuple(new_param_tuple) return replace