timekiller
/
mindspore2022
forked from huawei/mindspore2022
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!19597 [AutoParallel]fix pipeline split bug 

Merge pull request !19597 from lichen/fix_pipeline_split_bug
This commit is contained in:
i-robot 2021-07-08 01:41:41 +00:00 committed by Gitee
parent ebb7684b55 51238243b4
commit 418f9eb903
11 changed files with 40 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
mindspore/ccsrc/frontend/parallel/auto_parallel/rec_core/rec_generate_strategy.cc
View File

@ -49,7 +49,7 @@ void GenerateStrategy(const std::shared_ptr<Graph> &graph, const std::vector<std
// Set user-defined strategy
auto attrs = op->attrs();
if (StrategyFound(attrs)) {
StrategyPtr user_defined_stra = parallel::ExtractStrategy(attrs);
StrategyPtr user_defined_stra = parallel::ExtractStrategy(attrs[STRATEGY]);
op->SetSelectedStrategyAndCost(user_defined_stra, op->selected_cost());
}
// Set back to raw strategy for special node in predict/eval

12
mindspore/ccsrc/frontend/parallel/graph_util/pipeline_split_utils.cc
View File

@ -60,10 +60,7 @@ void SetStridedSliceStrategy(const AnfNodePtr &node) {
}
auto cnode = node->cast<CNodePtr>();
MS_EXCEPTION_IF_NULL(cnode);
PrimitivePtr prim = GetValueNode<PrimitivePtr>(cnode->input(0));
MS_EXCEPTION_IF_NULL(prim);
int64_t dev_num = 1;
auto attrs_temp = prim->attrs();
std::vector<Shapes> shape_list = ExtractShape(cnode);
if (shape_list.empty()) {
MS_LOG(EXCEPTION) << "Failure:node " << cnode->ToString() << " failed to extract shape";
@ -80,8 +77,7 @@ void SetStridedSliceStrategy(const AnfNodePtr &node) {
elements.push_back(MakeValue(input_strategy));
}
ValueTuplePtr strategy = std::make_shared<ValueTuple>(elements);
attrs_temp[STRATEGY] = strategy;
(void)prim->SetAttrs(attrs_temp);
cnode->AddPrimalAttr(STRATEGY, strategy);
}
void InsertVirtualAssignAdd(const std::pair<AnfNodePtr, int> &node_user, const FuncGraphManagerPtr &manager,
@ -467,12 +463,6 @@ void ParameterStartNode(const std::vector<AnfNodePtr> &all_nodes, const FuncGrap
auto prim = GetCNodePrimitive(node);
if (prim && prim->HasAttr(PARAMETER_START)) {
auto micro = Micro(cnode, &node_users_map);
OperatorAttrs attrs_;
auto op = CreatOpInstance(attrs_, prim->name(), "");
auto value_node = NewValueNode(op);
auto new_prim = GetValueNode(value_node)->cast<PrimitivePtr>();
new_prim->SetAttrs(prim->attrs());
manager->SetEdge(cnode, 0, value_node);
cnode->AddPrimalAttr(MICRO, micro);
cnode->AddPrimalAttr(PARAMETER_START, micro);
}

2
mindspore/ccsrc/frontend/parallel/pipeline_transformer/pipeline_transformer.cc
View File

@ -295,7 +295,7 @@ OperatorInfoPtr PipelineTransformer::CreateOpInfo(const CNodePtr &cnode, int tup
if (!StrategyFound(attrs)) {
strategy = GenerateBatchParallelStrategy(op_info, prim);
} else {
strategy = ExtractStrategy(attrs);
strategy = ExtractStrategy(attrs[STRATEGY]);
}
MS_EXCEPTION_IF_NULL(strategy);
if (op_info->Init(strategy) == FAILED) {

6
mindspore/ccsrc/frontend/parallel/step_auto_parallel.cc
View File

@ -242,12 +242,12 @@ void InitCostGraph() {
}
void SetStrategyToOperator(const OperatorInfoPtr &operator_info, const PrimitivePtr &prim,
const std::unordered_map<std::string, ValuePtr> &attrs, bool is_last_nodes,
StrategyMap *stra_map, const std::string &strategy_key_name) {
std::unordered_map<std::string, ValuePtr> attrs, bool is_last_nodes, StrategyMap *stra_map,
const std::string &strategy_key_name) {
// In this case, the configured strategy should be extracted to help setting cost
StrategyPtr strategyPtr;
if (StrategyFound(attrs)) {
strategyPtr = parallel::ExtractStrategy(attrs);
strategyPtr = parallel::ExtractStrategy(attrs[STRATEGY]);
} else {
strategyPtr = (*stra_map)[strategy_key_name];
}

12
mindspore/ccsrc/frontend/parallel/step_parallel.cc
View File

@ -1435,12 +1435,12 @@ OperatorInfoPtr NewOperatorInstance(const PrimitivePtr &prim, const PrimitiveAtt
return operator_;
}
StrategyPtr ExtractStrategy(std::unordered_map<std::string, ValuePtr> attrs) {
ValueTuplePtr var = attrs[STRATEGY]->cast<ValueTuplePtr>();
StrategyPtr ExtractStrategy(const ValuePtr &stra) {
ValueTuplePtr var = stra->cast<ValueTuplePtr>();
StrategyPtr strategyPtr;
int64_t stage_id = g_device_manager->stage_id();
MS_LOG(INFO) << "Extract information: strategy " << attrs[STRATEGY]->ToString();
MS_LOG(INFO) << "Extract information: strategy " << stra->ToString();
if (var == nullptr) {
MS_LOG(EXCEPTION) << "Strategy value is nullptr";
}
@ -2180,12 +2180,14 @@ void ExtractInformation(const std::vector<AnfNodePtr> &all_nodes, bool is_traini
}
bool load_strategy_from_ckpt =
StrategyCheckpoint::GetInstance().LoadCheckPointOn() && stra_map.find(strategy_key_name) != stra_map.end();
if ((!StrategyFound(attrs) && !load_strategy_from_ckpt)) {
if ((!StrategyFound(attrs) && !load_strategy_from_ckpt) && !cnode->HasPrimalAttr(STRATEGY)) {
MS_LOG(INFO) << "ExtractInformation: the strategy of node " << node->ToString() << " prim " << prim->name()
<< " is empty, using batch parallel";
strategyPtr = GenerateBatchParallelStrategy(operator_, prim);
} else if (cnode->HasPrimalAttr(STRATEGY)) {
strategyPtr = ExtractStrategy(cnode->GetPrimalAttr(STRATEGY));
} else if (StrategyFound(attrs)) {
strategyPtr = ExtractStrategy(attrs);
strategyPtr = ExtractStrategy(attrs[STRATEGY]);
} else {
strategyPtr = stra_map[strategy_key_name];
}

2
mindspore/ccsrc/frontend/parallel/step_parallel.h
View File

@ -99,7 +99,7 @@ OperatorInfoPtr NewOperatorInstance(const PrimitivePtr &prim, const PrimitiveAtt
std::vector<Shapes> shape_list);
// Extract strategy from attr
StrategyPtr ExtractStrategy(std::unordered_map<std::string, ValuePtr> attrs);
StrategyPtr ExtractStrategy(const ValuePtr &strategy);
Shapes GetNodeShape(const AnfNodePtr &node);

22
mindspore/ccsrc/pipeline/jit/pipeline_split.cc
View File

@ -25,7 +25,7 @@
namespace mindspore {
namespace pipeline {
static int64_t GetRank() {
std::string GetWorldGroup() {
auto ms_context = MsContext::GetInstance();
MS_EXCEPTION_IF_NULL(ms_context);
std::string world_group;
@ -37,6 +37,13 @@ static int64_t GetRank() {
} else {
MS_LOG(EXCEPTION) << "Invalid backend: " << backend;
}
return world_group;
}
static int64_t GetRank() {
auto ms_context = MsContext::GetInstance();
MS_EXCEPTION_IF_NULL(ms_context);
auto world_group = GetWorldGroup();
int64_t global_rank = parallel::ParallelContext::GetInstance()->global_rank();
uint32_t rank_id = 0;
if (!parallel::ParallelContext::GetInstance()->global_rank_is_set()) {
@ -75,7 +82,18 @@ bool PipelineSplit(const ResourcePtr &res) {
auto manager = res->manager();
auto root = res->func_graph();
auto global_rank = GetRank();
auto device_num = parallel::ParallelContext::GetInstance()->device_num();
auto world_group = GetWorldGroup();
uint32_t world_rank_size = 0;
int64_t device_num = 0;
if (!parallel::ParallelContext::GetInstance()->device_num_is_set()) {
if (!CommManager::GetInstance().GetRankSize(world_group, &world_rank_size)) {
MS_LOG(EXCEPTION) << "Get rank size failed";
}
device_num = UintToInt(world_rank_size);
MS_LOG(INFO) << "Get device num from communication model, the device num is " << device_num;
} else {
device_num = parallel::ParallelContext::GetInstance()->device_num();
}
if (device_num < 1) {
MS_LOG(EXCEPTION) << "Invalid device num: " << device_num;
}

2
mindspore/ccsrc/pipeline/jit/pipeline_split.h
View File

@ -17,11 +17,13 @@
#ifndef MINDSPORE_CCSRC_PIPELINE_JIT_PIPELINE_SPLIT_H_
#define MINDSPORE_CCSRC_PIPELINE_JIT_PIPELINE_SPLIT_H_
#include <string>
#include "pipeline/jit/resource.h"
namespace mindspore {
namespace pipeline {
bool PipelineSplit(const ResourcePtr &res);
std::string GetWorldGroup();
} // namespace pipeline
} // namespace mindspore

2
mindspore/nn/wrap/cell_wrapper.py
View File

@ -487,7 +487,7 @@ class _MicroBatch(Cell):
input_shape = self.shape(each_input)
micro_batch_begin = i * input_shape[0] // self.micro_size
micro_batch_end = (i + 1) * input_shape[0] // self.micro_size
micro_input = each_input[micro_batch_begin:micro_batch_end, :]
micro_input = each_input[micro_batch_begin:micro_batch_end]
micro_inputs += (micro_input,)
return micro_inputs

4
model_zoo/official/nlp/pangu_alpha/src/pangu_alpha_wrapcell.py
View File

@ -76,14 +76,14 @@ def tensor_grad_scale_pipeline(scale, grad, accu_grad):
new_grad = accu_grad * reciprocal(scale)
accu_grad = F.depend(accu_grad, new_grad)
zeros = F.tensor_mul(accu_grad, 0.0)
_ = F.assign(accu_grad, zeros)
new_grad = F.depend(new_grad, F.assign(accu_grad, zeros))
return new_grad
@shard_grad_scale.register("Tensor", "Tensor", "Tensor")
def tensor_shard_grad_scale_pipeline(scale, grad, accu_grad):
new_grad = grad * reciprocal(scale)
accu_grad = F.depend(accu_grad, new_grad)
_ = F.assign(accu_grad, F.zeros_like(accu_grad))
new_grad = F.depend(new_grad, F.assign(accu_grad, F.zeros_like(accu_grad)))
return new_grad
class PanguAlphaTrainOneStepWithLossScaleCell(TrainOneStepWithLossScaleCell):

2
tests/ut/cpp/parallel/step_parallel_test.cc
View File

@ -228,7 +228,7 @@ TEST_F(TestStepParallel, ExtractStrategy) {
ValueTuplePtr strategy_tuple = std::make_shared<ValueTuple>(elements);
attrs["strategy"] = strategy_tuple;
Strategys strategy_expect = {v1, v2};
StrategyPtr strategy = ExtractStrategy(attrs);
StrategyPtr strategy = ExtractStrategy(attrs["strategy"]);
Strategys strategy_test = strategy->GetInputDim();
ASSERT_EQ(strategy_expect, strategy_test);