From 03a56f2bb0c7d9f54027ab9b5904773ce3ebfa24 Mon Sep 17 00:00:00 2001 From: zhoufeng Date: Tue, 10 Aug 2021 20:17:48 +0800 Subject: [PATCH] alltoall exception handle Signed-off-by: zhoufeng --- .../kernel_compiler/hccl/hccl_kernel.cc | 3 +- .../ascend/mindir/all_to_all_unify_mindir.cc | 6 + .../device/ascend/ascend_kernel_runtime.cc | 2 +- .../runtime/hccl_adapter/hccl_adapter.cc | 46 ++- .../ccsrc/runtime/hccl_adapter/hccl_adapter.h | 3 +- mindspore/core/ops/neighborexchange.cc | 121 +++++- mindspore/core/ops/neighborexchange.h | 2 - .../ops/_grad_experimental/grad_comm_ops.py | 4 +- mindspore/ops/operations/_inner_ops.py | 15 +- mindspore/ops/operations/comm_ops.py | 19 +- tests/ut/cpp/device/hccl_adapter_test.cc | 63 +++ .../mindir/all_to_all_unify_mindir_test.cc | 2 +- .../all_to_all_unify_mindir_test.py | 10 +- tests/ut/cpp/runtest.sh | 2 +- tests/ut/cpp/stub/ge/ge_task_launch_stub.cc | 2 +- tests/ut/python/parallel/test_alltoall.py | 207 ++++++++++ .../python/parallel/test_neighborexchange.py | 386 ++++++++++++++++-- 17 files changed, 818 insertions(+), 75 deletions(-) diff --git a/mindspore/ccsrc/backend/kernel_compiler/hccl/hccl_kernel.cc b/mindspore/ccsrc/backend/kernel_compiler/hccl/hccl_kernel.cc index 77edb57eab..e71eab88cd 100644 --- a/mindspore/ccsrc/backend/kernel_compiler/hccl/hccl_kernel.cc +++ b/mindspore/ccsrc/backend/kernel_compiler/hccl/hccl_kernel.cc @@ -197,7 +197,8 @@ const std::vector &HcclKernel::GetWorkspaceSizeList() const { MS_EXCEPTION_IF_NULL(context_ptr); bool is_task_sink = context_ptr->get_param(MS_CTX_ENABLE_TASK_SINK); auto mode = context_ptr->get_param(MS_CTX_EXECUTION_MODE); - if (!workspace_size_list_.empty() || hccl_data_type_list_.empty() || (!is_task_sink && mode == kGraphMode)) { + if (!workspace_size_list_.empty() || hccl_data_type_list_.empty() || (!is_task_sink && mode == kGraphMode) || + mode == kPynativeMode) { return workspace_size_list_; } workspace_size_list_.emplace_back( diff --git a/mindspore/ccsrc/backend/optimizer/ascend/mindir/all_to_all_unify_mindir.cc b/mindspore/ccsrc/backend/optimizer/ascend/mindir/all_to_all_unify_mindir.cc index 6b2d57b9a1..884f762f3f 100644 --- a/mindspore/ccsrc/backend/optimizer/ascend/mindir/all_to_all_unify_mindir.cc +++ b/mindspore/ccsrc/backend/optimizer/ascend/mindir/all_to_all_unify_mindir.cc @@ -27,6 +27,10 @@ namespace { constexpr size_t kCNodePrimitiveIdx = 0; constexpr size_t kAllToAllInputIdx = 1; +inline int64_t NormalizeDim(const std::vector &shape, int64_t dim) { + return dim < 0 ? SizeToLong(shape.size()) + dim : dim; +} + void ChangePrimitiveToAllToAllV(const AnfNodePtr &node) { MS_EXCEPTION_IF_NULL(node); auto neighbor_exchange = node->cast(); @@ -66,6 +70,7 @@ CNodePtr CreateSplitNode(const FuncGraphPtr &graph, const CNodePtr &all_to_all) MS_EXCEPTION_IF_NULL(split_v); auto dtype = AnfAlgo::GetOutputInferDataType(all_to_all_input, 0); auto shape = AnfAlgo::GetOutputInferShape(all_to_all_input, 0); + split_dim = NormalizeDim(shape, split_dim); if (SizeToLong(shape.size()) <= split_dim) { MS_LOG(EXCEPTION) << "Invalid split dim " << split_dim << " is over the shape size " << shape.size(); } @@ -133,6 +138,7 @@ CNodePtr CreateConcatNode(const FuncGraphPtr &graph, const CNodePtr &all_to_all, auto concat = graph->NewCNode(concat_input); MS_EXCEPTION_IF_NULL(concat); auto single_shape = AnfAlgo::GetOutputInferShape(all_to_all_v_outputs[0], 0); + concat_dim = NormalizeDim(single_shape, concat_dim); if (LongToSize(concat_dim) >= single_shape.size()) { MS_LOG(EXCEPTION) << "Invalid concat dim " << concat_dim << " is greater than shape size " << single_shape.size(); } diff --git a/mindspore/ccsrc/runtime/device/ascend/ascend_kernel_runtime.cc b/mindspore/ccsrc/runtime/device/ascend/ascend_kernel_runtime.cc index 6dad937581..71245edd76 100644 --- a/mindspore/ccsrc/runtime/device/ascend/ascend_kernel_runtime.cc +++ b/mindspore/ccsrc/runtime/device/ascend/ascend_kernel_runtime.cc @@ -876,7 +876,7 @@ bool AscendKernelRuntime::HcclInit() { } MS_LOG(INFO) << "MINDSPORE_HCCL_CONFIG_PATH : " << full_path << ", RANK_ID: " << rank_id_str; bool ret = hccl::HcclAdapter::GetInstance().InitHccl(context_ptr->get_param(MS_CTX_DEVICE_ID), rank_id_str, - full_path); + full_path, mode == kGraphMode); free(full_path); if (!ret) { MS_LOG(ERROR) << "Hcom init failed."; diff --git a/mindspore/ccsrc/runtime/hccl_adapter/hccl_adapter.cc b/mindspore/ccsrc/runtime/hccl_adapter/hccl_adapter.cc index 3fef5113bd..75cd4f0d17 100644 --- a/mindspore/ccsrc/runtime/hccl_adapter/hccl_adapter.cc +++ b/mindspore/ccsrc/runtime/hccl_adapter/hccl_adapter.cc @@ -137,43 +137,53 @@ bool HcclAdapter::InitHccl() { return true; } -bool HcclAdapter::InitHccl(uint32_t device_id, std::string_view rank_id, std::string_view rank_file) { - MS_LOG(INFO) << "Start init hccl adapter."; +bool HcclAdapter::InitHccl(uint32_t device_id, std::string_view rank_id, std::string_view rank_file, + bool is_graph_mode) { + MS_LOG(INFO) << "Start init hccl adapter for " << (is_graph_mode ? "graph mode." : "pynative mode."); std::lock_guard lock(init_mutex_); if (init_flag_) { MS_LOG(INFO) << "Hccl has been inited, skip."; return true; } - + is_graph_mode_ = is_graph_mode; InitPlugin(); - bool ret = InitKernelInfoStore(device_id, rank_id, rank_file); - if (!ret) { - return false; - } - ret = InitHcclComm(rank_id, rank_file); - if (!ret) { - return false; - } - ret = InitHcclExec(); - if (!ret) { - return false; + if (is_graph_mode_) { + bool ret = InitKernelInfoStore(device_id, rank_id, rank_file); + if (!ret) { + return false; + } + + ret = InitHcclExec(); + if (!ret) { + return false; + } + } else { + bool ret = InitHcclComm(rank_id, rank_file); + if (!ret) { + return false; + } } + init_flag_ = true; MS_LOG(INFO) << "Init hccl adapter success."; return true; } bool HcclAdapter::FinalizeHccl() { - MS_LOG(INFO) << "Start destroy hccl adapter."; std::lock_guard lock(init_mutex_); + MS_LOG(INFO) << "Start destroy hccl adapter for " << (is_graph_mode_ ? "graph mode." : "pynative mode."); if (!init_flag_) { MS_LOG(INFO) << "Hccl has never been inited, skip."; return true; } - (void)FinalizeHcclExec(); - (void)FinalizeHcclComm(); - (void)FinalizeKernelInfoStore(); + if (is_graph_mode_) { + (void)FinalizeHcclExec(); + (void)FinalizeKernelInfoStore(); + } else { + (void)FinalizeHcclComm(); + } + FinalizePlugin(); init_flag_ = false; MS_LOG(INFO) << "Destroy hccl adapter success."; diff --git a/mindspore/ccsrc/runtime/hccl_adapter/hccl_adapter.h b/mindspore/ccsrc/runtime/hccl_adapter/hccl_adapter.h index 2ed8685d9f..ad6f8c93dc 100644 --- a/mindspore/ccsrc/runtime/hccl_adapter/hccl_adapter.h +++ b/mindspore/ccsrc/runtime/hccl_adapter/hccl_adapter.h @@ -42,7 +42,7 @@ class HcclAdapter { static HcclAdapter &GetInstance(); // common - bool InitHccl(uint32_t device_id, std::string_view rank_id, std::string_view rank_file); + bool InitHccl(uint32_t device_id, std::string_view rank_id, std::string_view rank_file, bool is_graph_mode); bool InitHccl(); bool FinalizeHccl(); @@ -121,6 +121,7 @@ class HcclAdapter { std::shared_ptr<::ge::OpsKernelBuilder> ops_kernel_builder_ = nullptr; bool init_flag_ = false; + bool is_graph_mode_ = false; std::mutex init_mutex_; }; } // namespace mindspore::hccl diff --git a/mindspore/core/ops/neighborexchange.cc b/mindspore/core/ops/neighborexchange.cc index b4c47454bc..23e17ab382 100644 --- a/mindspore/core/ops/neighborexchange.cc +++ b/mindspore/core/ops/neighborexchange.cc @@ -15,18 +15,125 @@ */ #include "ops/neighborexchange.h" -#include "ops/op_utils.h" +#include #include "utils/check_convert_utils.h" #include "abstract/primitive_infer_map.h" namespace mindspore { namespace ops { -abstract::TupleShapePtr InferShape(const PrimitivePtr &primitive, const std::vector &input_args) { +namespace { +constexpr auto kRecvShapes = "recv_shapes"; +constexpr auto kRecvRankIds = "recv_rank_ids"; +constexpr auto kRecvType = "recv_type"; +constexpr auto kSendShapes = "send_shapes"; +constexpr auto kSendRankIds = "send_rank_ids"; +constexpr auto kGroup = "group"; + +inline std::string GetShapeStr(const std::vector &shape) { + std::string shape_str = "["; + for (size_t i = 0; i < shape.size(); ++i) { + if (i == 0) { + shape_str += std::to_string(shape[i]); + } else { + shape_str += "," + std::to_string(shape[i]); + } + } + return shape_str + "]"; +} + +void CheckAttr(const PrimitivePtr &primitive, const std::string &shape_attr_name, + const std::string &rank_ids_attr_name) { + MS_EXCEPTION_IF_NULL(primitive); + // size of send/recv_rank_ids equal to size of send/recv_shapes + ValuePtrList attr_shapes; + try { + auto attr = primitive->GetAttr(shape_attr_name); + attr_shapes = GetValue(attr); + } catch (const std::exception &) { + MS_EXCEPTION(TypeError) << "Attr " << shape_attr_name << " should be a tuple(list, list, ...)."; + } + if (!attr_shapes.empty()) { + auto ele = attr_shapes[0]->cast(); + if (ele == nullptr) { + MS_EXCEPTION(TypeError) << "Attr " << shape_attr_name << " must be a tuple."; + } + } + std::vector attr_rank_ids; + try { + auto attr = primitive->GetAttr(rank_ids_attr_name); + attr_rank_ids = GetValue>(attr); + } catch (const std::exception &) { + MS_EXCEPTION(TypeError) << "Attr " << rank_ids_attr_name << " should be a list[int, int, ...]."; + } + if (attr_shapes.size() != attr_rank_ids.size()) { + MS_EXCEPTION(ValueError) << "Invalid " << primitive->name() << " attr " << shape_attr_name << " size " + << attr_shapes.size() << " must be equal to attr " << rank_ids_attr_name << " size " + << attr_rank_ids.size(); + } +} + +void Check(const PrimitivePtr &primitive, const std::vector &input_args) { MS_EXCEPTION_IF_NULL(primitive); auto prim_name = primitive->name(); + CheckAttr(primitive, kRecvShapes, kRecvRankIds); + CheckAttr(primitive, kSendShapes, kSendRankIds); + // check recv type + auto recv_type_attr = primitive->GetAttr(kRecvType); + MS_EXCEPTION_IF_NULL(recv_type_attr); + if (!recv_type_attr->isa()) { + MS_EXCEPTION(TypeError) << "Attr " << kRecvType << " should be a mindspore data type."; + } + // check group + auto group_attr = primitive->GetAttr(kGroup); + try { + MS_EXCEPTION_IF_NULL(group_attr); + (void)GetValue(group_attr); + } catch (const std::exception &) { + MS_EXCEPTION(TypeError) << "Attr " << kGroup << " should be a str."; + } + // check empty input + auto send_rank_ids = GetValue>(primitive->GetAttr(kSendRankIds)); + if (send_rank_ids.empty()) { + (void)CheckAndConvertUtils::CheckInteger("input_numbers", input_args.size(), kEqual, 0, prim_name); + return; + } + // check input shape & attr send shape (void)CheckAndConvertUtils::CheckInteger("input_numbers", input_args.size(), kEqual, 1, prim_name); CheckAndConvertUtils::CheckArgs(prim_name, input_args, 0); - auto recv_shapes = primitive->GetAttr(RecvShapes); + auto abstract_tuple = input_args[0]->cast(); + MS_EXCEPTION_IF_NULL(abstract_tuple); + auto abstract_element = abstract_tuple->elements(); + auto send_shapes = GetValue(primitive->GetAttr(kSendShapes)); + if (abstract_element.size() != send_shapes.size()) { + MS_EXCEPTION(ArgumentError) << "Input tuple size " << abstract_element.size() << " must be equal to attr " + << kSendShapes << " size " << send_shapes.size(); + } + for (size_t i = 0; i < abstract_element.size(); ++i) { + // get attr shape + MS_EXCEPTION_IF_NULL(send_shapes[i]); + auto send_shape_value = send_shapes[i]->cast(); + MS_EXCEPTION_IF_NULL(send_shape_value); + std::vector send_shape = GetValue>(send_shape_value); + // get input tensor shape + MS_EXCEPTION_IF_NULL(abstract_element[i]); + auto arg_base_shape = abstract_element[i]->BuildShape(); + MS_EXCEPTION_IF_NULL(arg_base_shape); + auto shape = arg_base_shape->cast(); + if (shape == nullptr) { + MS_EXCEPTION(ArgumentError) << "Input " << i << " should be a tensor."; + } + // comp two shape + auto shape_vec = shape->shape(); + if (shape_vec != send_shape) { + MS_EXCEPTION(ArgumentError) << "Input " << i << " shape: " << GetShapeStr(shape_vec) + << " but attr shape : " << GetShapeStr(send_shape); + } + } +} + +abstract::TupleShapePtr InferShape(const PrimitivePtr &primitive, const std::vector &input_args) { + MS_EXCEPTION_IF_NULL(primitive); + auto recv_shapes = primitive->GetAttr(kRecvShapes); MS_EXCEPTION_IF_NULL(recv_shapes); auto shapes_seq = recv_shapes->cast(); MS_EXCEPTION_IF_NULL(shapes_seq); @@ -49,25 +156,25 @@ TypePtr InferType(const PrimitivePtr &primitive, const std::vectorGetAttr(RecvShapes); + auto recv_shapes = primitive->GetAttr(kRecvShapes); MS_EXCEPTION_IF_NULL(recv_shapes); auto shapes_seq = recv_shapes->cast(); MS_EXCEPTION_IF_NULL(shapes_seq); auto shapes_value = shapes_seq->value(); auto out_num = shapes_value.size(); - auto recv_type = primitive->GetAttr(RecvType)->cast(); + auto recv_type = primitive->GetAttr(kRecvType)->cast(); MS_EXCEPTION_IF_NULL(recv_type); std::vector type_vec(out_num, recv_type); return std::make_shared(type_vec); } - +} // namespace AbstractBasePtr NeighborExchangeInfer(const abstract::AnalysisEnginePtr &, const PrimitivePtr &primitive, const std::vector &input_args) { + Check(primitive, input_args); auto type = InferType(primitive, input_args); auto shape = InferShape(primitive, input_args); return abstract::MakeAbstract(shape, type); } - REGISTER_PRIMITIVE_EVAL_IMPL(NeighborExchange, prim::kPrimNeighborExchange, NeighborExchangeInfer, nullptr, true); } // namespace ops } // namespace mindspore diff --git a/mindspore/core/ops/neighborexchange.h b/mindspore/core/ops/neighborexchange.h index 40900667ba..d3ff559f2c 100644 --- a/mindspore/core/ops/neighborexchange.h +++ b/mindspore/core/ops/neighborexchange.h @@ -25,8 +25,6 @@ namespace mindspore { namespace ops { constexpr auto kNameNeighborExchange = "NeighborExchange"; -constexpr auto RecvShapes = "recv_shapes"; -constexpr auto RecvType = "recv_type"; class MS_CORE_API NeighborExchange : public PrimitiveC { public: NeighborExchange() : PrimitiveC(kNameNeighborExchange) {} diff --git a/mindspore/ops/_grad_experimental/grad_comm_ops.py b/mindspore/ops/_grad_experimental/grad_comm_ops.py index 878a2d094a..d0cff6bc7c 100644 --- a/mindspore/ops/_grad_experimental/grad_comm_ops.py +++ b/mindspore/ops/_grad_experimental/grad_comm_ops.py @@ -25,9 +25,11 @@ def get_bprop_neighborexchange(self): send_rank_ids = self.recv_rank_ids recv_rank_ids = self.send_rank_ids recv_shapes = self.send_shapes + send_shapes = self.recv_shapes recv_type = self.recv_type - neighborexchange_grad = NeighborExchange(send_rank_ids, recv_rank_ids, recv_shapes, recv_shapes, recv_type, group) + neighborexchange_grad = NeighborExchange(send_rank_ids, recv_rank_ids, recv_shapes, send_shapes, recv_type, group) def bprop(x, out, dout): return (neighborexchange_grad(dout),) + return bprop diff --git a/mindspore/ops/operations/_inner_ops.py b/mindspore/ops/operations/_inner_ops.py index 07acdef27f..25e1c8cc0b 100755 --- a/mindspore/ops/operations/_inner_ops.py +++ b/mindspore/ops/operations/_inner_ops.py @@ -500,10 +500,10 @@ class NeighborExchange(Primitive): as while receive data from recv_rank_ids. Args: - send_rank_ids (list): Ranks which the data is sent to. - recv_rank_ids (list): Ranks which the data is received from. - recv_shapes (list): Data shape which received from recv_rank_ids. - send_shapes (list): Data shape which send to the send_rank_ids. + send_rank_ids (list(int)): Ranks which the data is sent to. + recv_rank_ids (list(int)): Ranks which the data is received from. + recv_shapes (tuple(list(int))): Data shape which received from recv_rank_ids. + send_shapes (tuple(list(int))): Data shape which send to the send_rank_ids. recv_type (type): Data type which received from recv_rank_ids group (str): """ @@ -518,6 +518,9 @@ class NeighborExchange(Primitive): self.send_shapes = send_shapes self.recv_type = recv_type + def __call__(self, tensor): + raise NotImplementedError + class MatrixSetDiag(PrimitiveWithInfer): r""" @@ -954,6 +957,7 @@ class StackInit(PrimitiveWithInfer): [[1 3] [2 0]] """ + @prim_attr_register def __init__(self, index=1): """StackInit""" @@ -979,6 +983,7 @@ class StackPush(PrimitiveWithInfer): Examples: Please refer to the usage of `StackInit`. """ + @prim_attr_register def __init__(self, index=1): """StackPush""" @@ -1007,6 +1012,7 @@ class StackPop(PrimitiveWithInfer): Examples: Please refer to the usage of `StackInit`. """ + @prim_attr_register def __init__(self, index=1, shape=(1,), dtype=mstype.float32): """StackPop""" @@ -1046,6 +1052,7 @@ class StackDestroy(PrimitiveWithInfer): Examples: Please refer to the usage of `StackInit`. """ + @prim_attr_register def __init__(self, index=1): """StackDestroy""" diff --git a/mindspore/ops/operations/comm_ops.py b/mindspore/ops/operations/comm_ops.py index 19826bd226..bc1d4f298c 100644 --- a/mindspore/ops/operations/comm_ops.py +++ b/mindspore/ops/operations/comm_ops.py @@ -218,6 +218,7 @@ class _MiniStepAllGather(PrimitiveWithInfer): group (str): The communication group to work on. Default: None. grad_accumulation_step (int): The grad accumulation step. Default: None. """ + @prim_attr_register def __init__(self, group=GlobalComm.WORLD_COMM_GROUP, grad_accumulation_step=None, mean_flag=None): """Initialize _MiniStepAllGather.""" @@ -250,6 +251,7 @@ class _MicroStepAllGather(PrimitiveWithInfer): Args: group (str): The communication group to work on. Default: None. """ + @prim_attr_register def __init__(self, group=GlobalComm.WORLD_COMM_GROUP, mean_flag=None): validator.check_value_type('group', _get_group(group), (str,), self.name) @@ -421,6 +423,7 @@ class _HostReduceScatter(PrimitiveWithInfer): ValueError: If the first dimension of input can not be divided by group size, or group is not set, or rank_id not in [0, 7]. """ + @prim_attr_register def __init__(self, op=ReduceOp.SUM, group=None): """Initialize _HostReduceScatter.""" @@ -603,12 +606,21 @@ class _AlltoAll(PrimitiveWithInfer): def __init__(self, split_count, split_dim, concat_dim, group=GlobalComm.WORLD_COMM_GROUP): """Initialize AlltoAll""" validator.check_value_type('group', _get_group(group), (str,), self.name) + validator.check_is_int(split_count, int) + validator.check_is_int(split_dim, int) + validator.check_is_int(concat_dim, int) self.split_count = split_count self.split_dim = split_dim self.concat_dim = concat_dim self.add_prim_attr('group', _get_group(group)) def infer_shape(self, x_shape): + rank_size = get_group_size(_get_group(self.group)) + if self.split_count != rank_size: + raise ValueError(f"split count '{self.split_count}' must be equal to rank size '{rank_size}'.") + if x_shape[self.split_dim] % self.split_count != 0: + raise ValueError( + f"split count '{self.split_count}' must be divisible by rank size '{x_shape[self.split_dim]}'.") x_shape[self.concat_dim] = x_shape[self.concat_dim] * self.split_count x_shape[self.split_dim] = int(x_shape[self.split_dim] / self.split_count) return x_shape @@ -618,7 +630,7 @@ class _AlltoAll(PrimitiveWithInfer): return x_dtype def __call__(self, tensor): - return + raise NotImplementedError class _MirrorOperator(PrimitiveWithInfer): @@ -687,6 +699,7 @@ class _VirtualDiv(PrimitiveWithInfer): Args: divisor: float32 """ + @prim_attr_register def __init__(self, divisor=None): """Initialize _VirtualDiv.""" @@ -704,6 +717,7 @@ virtual_div = _VirtualDiv() class _VirtualAdd(PrimitiveWithInfer): """Auto parallel virtual operator. Do nothing in forward, do Add in backward.""" + @prim_attr_register def __init__(self): """Initialize _VirtualAdd.""" @@ -742,6 +756,7 @@ class _VirtualAssignAdd(PrimitiveWithInfer): internal use of parallel modules and cannot be called by users. """ + @prim_attr_register def __init__(self): """Initialize _VirtualAssignAdd.""" @@ -761,6 +776,7 @@ class _VirtualAccuGrad(PrimitiveWithInfer): Auto parallel virtual operator. Do nothing in forward, return y in backward. It is only for internal use of parallel modules and cannot be called by users. """ + @prim_attr_register def __init__(self): """Initialize _VirtualAccuGrad.""" @@ -817,6 +833,7 @@ class _VirtualOutput(PrimitiveWithInfer): def infer_dtype(self, x_dtype): return x_dtype + class _GetTensorSlice(PrimitiveWithInfer): """ Gets tensor slice by device matrix and tensor map. diff --git a/tests/ut/cpp/device/hccl_adapter_test.cc b/tests/ut/cpp/device/hccl_adapter_test.cc index 6c3b6fdeb5..7095f148df 100644 --- a/tests/ut/cpp/device/hccl_adapter_test.cc +++ b/tests/ut/cpp/device/hccl_adapter_test.cc @@ -83,6 +83,12 @@ class TestHcclAdapter : public UT::Common { std::string format_ = "NCHW"; }; +/// Feature: AllToAllvCalcParam +/// Description: on 2p, send to rank 1, and recv nothing +/// Expectation: send count 0 1 +/// send offset 0 0 +/// recv count 0 0 +/// recv offset 0 0 TEST_F(TestHcclAdapter, test_all_to_all_v_calc_param_2p_only_send) { auto graph = std::make_shared(); ASSERT_TRUE(graph != nullptr); @@ -100,6 +106,12 @@ TEST_F(TestHcclAdapter, test_all_to_all_v_calc_param_2p_only_send) { EXPECT_EQ(calc.GetRecvDispls(), std::vector({0, 0})); } +/// Feature: AllToAllvCalcParam +/// Description: on 2p, send nothing, and recv from rank 0 and rank 1 +/// Expectation: send count 0 0 +/// send offset 0 0 +/// recv count 1 1 +/// recv offset 0 128 TEST_F(TestHcclAdapter, test_all_to_all_v_calc_param_2p_only_recv) { auto graph = std::make_shared(); ASSERT_TRUE(graph != nullptr); @@ -117,6 +129,12 @@ TEST_F(TestHcclAdapter, test_all_to_all_v_calc_param_2p_only_recv) { EXPECT_EQ(calc.GetRecvDispls(), std::vector({0, 128})); } +/// Feature: AllToAllvCalcParam +/// Description: on 4p, send to rank1,2,3, and recv nothing +/// Expectation: send count 0 1 1 1 +/// send offset 0 0 128 256 +/// recv count 0 0 0 0 +/// recv offset 0 0 0 0 TEST_F(TestHcclAdapter, test_all_to_all_v_calc_param_4p_only_send) { auto graph = std::make_shared(); ASSERT_TRUE(graph != nullptr); @@ -135,6 +153,12 @@ TEST_F(TestHcclAdapter, test_all_to_all_v_calc_param_4p_only_send) { EXPECT_EQ(calc.GetRecvDispls(), std::vector({0, 0, 0, 0})); } +/// Feature: AllToAllvCalcParam +/// Description: on 4p, send to rank1,3, and recv nothing +/// Expectation: send count 0 1 0 1 +/// send offset 0 0 128 128 +/// recv count 0 0 0 0 +/// recv offset 0 0 0 0 TEST_F(TestHcclAdapter, test_all_to_all_v_calc_param_4p_only_send_2) { auto graph = std::make_shared(); ASSERT_TRUE(graph != nullptr); @@ -153,6 +177,12 @@ TEST_F(TestHcclAdapter, test_all_to_all_v_calc_param_4p_only_send_2) { EXPECT_EQ(calc.GetRecvDispls(), std::vector({0, 0, 0, 0})); } +/// Feature: AllToAllvCalcParam +/// Description: on 2p, send to rank1, and recv from rank1 +/// Expectation: send count 0 1 +/// send offset 0 0 +/// recv count 0 1 +/// recv offset 0 0 TEST_F(TestHcclAdapter, test_all_to_all_v_calc_param_2p_exchange) { auto graph = std::make_shared(); ASSERT_TRUE(graph != nullptr); @@ -170,6 +200,12 @@ TEST_F(TestHcclAdapter, test_all_to_all_v_calc_param_2p_exchange) { EXPECT_EQ(calc.GetRecvDispls(), std::vector({0, 0})); } +/// Feature: AllToAllvCalcParam +/// Description: on 2p, send to rank0, and recv from rank0 +/// Expectation: send count 1 0 +/// send offset 0 128 +/// recv count 1 0 +/// recv offset 0 128 TEST_F(TestHcclAdapter, test_all_to_all_v_calc_param_2p_send_to_self) { auto graph = std::make_shared(); ASSERT_TRUE(graph != nullptr); @@ -187,6 +223,12 @@ TEST_F(TestHcclAdapter, test_all_to_all_v_calc_param_2p_send_to_self) { EXPECT_EQ(calc.GetRecvDispls(), std::vector({0, 128})); } +/// Feature: AllToAllvCalcParam +/// Description: on 4p, send to rank0123, and recv from rank0123 +/// Expectation: send count 1 1 1 1 +/// send offset 0 128 256 384 +/// recv count 1 1 1 1 +/// recv offset 0 128 256 384 TEST_F(TestHcclAdapter, test_all_to_all_v_calc_param_4p_all_to_all) { auto graph = std::make_shared(); ASSERT_TRUE(graph != nullptr); @@ -205,6 +247,12 @@ TEST_F(TestHcclAdapter, test_all_to_all_v_calc_param_4p_all_to_all) { EXPECT_EQ(calc.GetRecvDispls(), std::vector({0, 128, 256, 384})); } +/// Feature: AllToAllvCalcParam +/// Description: on 4p, send to rank0123, and recv from rank0123, but recv order is wrong +/// Expectation: send count 1 1 1 1 +/// send offset 0 128 256 384 +/// recv count 1 1 1 1 +/// recv offset 256 128 384 0 TEST_F(TestHcclAdapter, test_all_to_all_v_calc_param_4p_all_in_all_in_wrong_order) { auto graph = std::make_shared(); ASSERT_TRUE(graph != nullptr); @@ -223,6 +271,12 @@ TEST_F(TestHcclAdapter, test_all_to_all_v_calc_param_4p_all_in_all_in_wrong_orde EXPECT_EQ(calc.GetRecvDispls(), std::vector({256, 128, 384, 0})); } +/// Feature: AllToAllvCalcParam +/// Description: on 4p, send to rank123, and recv from nothing, but send order is wrong +/// Expectation: send count 0 1 1 1 +/// send offset 0 128 256 0 +/// recv count 0 0 0 0 +/// recv offset 0 0 0 0 TEST_F(TestHcclAdapter, test_all_to_all_v_calc_param_4p_only_send_in_wrong_order) { auto graph = std::make_shared(); ASSERT_TRUE(graph != nullptr); @@ -241,6 +295,9 @@ TEST_F(TestHcclAdapter, test_all_to_all_v_calc_param_4p_only_send_in_wrong_order EXPECT_EQ(calc.GetRecvDispls(), std::vector({0, 0, 0, 0})); } +/// Feature: AllToAllvCalcParam +/// Description: on 2p, rank id over valid range +/// Expectation: throw exception TEST_F(TestHcclAdapter, test_all_to_all_v_calc_param_2p_invalid_rank_id) { auto graph = std::make_shared(); ASSERT_TRUE(graph != nullptr); @@ -254,6 +311,9 @@ TEST_F(TestHcclAdapter, test_all_to_all_v_calc_param_2p_invalid_rank_id) { ASSERT_ANY_THROW(calc.CalcOpParam()); } +/// Feature: AllToAllvCalcParam +/// Description: on 2p, has 2 outputs but only 1 recv_rank_ids is set +/// Expectation: throw exception TEST_F(TestHcclAdapter, test_all_to_all_v_calc_param_2p_invalid_rank_id_2) { auto graph = std::make_shared(); ASSERT_TRUE(graph != nullptr); @@ -267,6 +327,9 @@ TEST_F(TestHcclAdapter, test_all_to_all_v_calc_param_2p_invalid_rank_id_2) { ASSERT_ANY_THROW(calc.CalcOpParam()); } +/// Feature: AllToAllvCalcParam +/// Description: on 2p, rank id over valid range +/// Expectation: throw exception TEST_F(TestHcclAdapter, test_all_to_all_v_calc_param_2p_wrong_order_and_invalid_rank_id) { auto graph = std::make_shared(); ASSERT_TRUE(graph != nullptr); diff --git a/tests/ut/cpp/pre_activate/ascend/mindir/all_to_all_unify_mindir_test.cc b/tests/ut/cpp/pre_activate/ascend/mindir/all_to_all_unify_mindir_test.cc index 9338e293e0..b5460d1eb3 100644 --- a/tests/ut/cpp/pre_activate/ascend/mindir/all_to_all_unify_mindir_test.cc +++ b/tests/ut/cpp/pre_activate/ascend/mindir/all_to_all_unify_mindir_test.cc @@ -38,7 +38,7 @@ class TestAllToAllUnifyMindIr : public BackendCommon { TEST_F(TestAllToAllUnifyMindIr, test_neighbor_exchange) { FuncGraphPtr g = getPyFun_.CallAndParseRet("test_neighbor_exchange", "before"); ASSERT_TRUE(g != nullptr); - std::vector shp_x{2, 3}; + std::vector shp_x{2, 2}; auto x_abstract = std::make_shared( AbstractBasePtrList{std::make_shared(kFloat32, shp_x)}); AbstractBasePtrList args_spec_list{x_abstract}; diff --git a/tests/ut/cpp/python_input/gtest_input/pre_activate/all_to_all_unify_mindir_test.py b/tests/ut/cpp/python_input/gtest_input/pre_activate/all_to_all_unify_mindir_test.py index 08d49cee30..f65fbf6003 100644 --- a/tests/ut/cpp/python_input/gtest_input/pre_activate/all_to_all_unify_mindir_test.py +++ b/tests/ut/cpp/python_input/gtest_input/pre_activate/all_to_all_unify_mindir_test.py @@ -13,8 +13,15 @@ # limitations under the License. # ============================================================================ import mindspore as ms +from mindspore import context from mindspore.ops.operations._inner_ops import NeighborExchange from mindspore.ops.operations.comm_ops import _AlltoAll +from mindspore.communication.management import GlobalComm, init + +context.set_context(device_target="Ascend") +GlobalComm.CHECK_ENVS = False +init("hccl") +GlobalComm.CHECK_ENVS = True class FnDict: def __init__(self): @@ -28,7 +35,7 @@ class FnDict: def test_neighbor_exchange(tag): fns = FnDict() - neighbor = NeighborExchange(send_rank_ids=[0], recv_rank_ids=[1], recv_shapes=([2, 3],), send_shapes=([2, 2],), + neighbor = NeighborExchange(send_rank_ids=[0], recv_rank_ids=[1], recv_shapes=([2, 2],), send_shapes=([2, 2],), recv_type=ms.float32) @fns def before(x): @@ -37,6 +44,7 @@ def test_neighbor_exchange(tag): return fns[tag] def test_all_to_all(tag): + context.set_auto_parallel_context(device_num=8, global_rank=0) fns = FnDict() altoall = _AlltoAll(split_count=8, split_dim=2, concat_dim=3) @fns diff --git a/tests/ut/cpp/runtest.sh b/tests/ut/cpp/runtest.sh index df1f81e9bd..29a9b2a7a9 100755 --- a/tests/ut/cpp/runtest.sh +++ b/tests/ut/cpp/runtest.sh @@ -29,7 +29,7 @@ cd ${BUILD_PATH}/mindspore/tests/ut/cpp export LD_LIBRARY_PATH=${BUILD_PATH}/mindspore/googletest/googlemock/gtest:${PROJECT_PATH}/mindspore:\ ${PROJECT_PATH}/mindspore/lib:${PROJECT_PATH}/graphengine/third_party/prebuild/x86_64:\ ${PROJECT_PATH}/graphengine/third_party/prebuild/aarch64:${LD_LIBRARY_PATH} -export PYTHONPATH=${PROJECT_PATH}/tests/ut/cpp/python_input:$PYTHONPATH:${PROJECT_PATH} +export PYTHONPATH=${PROJECT_PATH}/tests/ut/cpp/python_input:$PYTHONPATH:${PROJECT_PATH}:${PROJECT_PATH}/tests/ut/python export GLOG_v=2 export GC_COLLECT_IN_CELL=1 ## set op info config path diff --git a/tests/ut/cpp/stub/ge/ge_task_launch_stub.cc b/tests/ut/cpp/stub/ge/ge_task_launch_stub.cc index 7be74ba8d7..e9df5c180d 100644 --- a/tests/ut/cpp/stub/ge/ge_task_launch_stub.cc +++ b/tests/ut/cpp/stub/ge/ge_task_launch_stub.cc @@ -23,7 +23,7 @@ HcclAdapter &HcclAdapter::GetInstance() { return instance; } bool HcclAdapter::InitHccl() { return true; } -bool HcclAdapter::InitHccl(uint32_t, std::string_view, std::string_view) { return true; } +bool HcclAdapter::InitHccl(uint32_t, std::string_view, std::string_view, bool) { return true; } bool HcclAdapter::FinalizeHccl() { return true; } HcclResult HcclAdapter::HcclCreateGroup(const std::string &, uint32_t, uint32_t *) const { return HCCL_SUCCESS; } HcclResult HcclAdapter::HcclDestroyGroup(const std::string &) const { return HCCL_SUCCESS; } diff --git a/tests/ut/python/parallel/test_alltoall.py b/tests/ut/python/parallel/test_alltoall.py index 8cc29cfa0a..df7537342d 100644 --- a/tests/ut/python/parallel/test_alltoall.py +++ b/tests/ut/python/parallel/test_alltoall.py @@ -13,6 +13,7 @@ # limitations under the License. import re +import pytest import numpy as np import mindspore as ms @@ -24,11 +25,20 @@ from mindspore.common.parameter import Parameter from mindspore.nn.loss import SoftmaxCrossEntropyWithLogits from mindspore.nn.optim.momentum import Momentum from mindspore.ops import operations as P +from mindspore.ops.operations.comm_ops import _AlltoAll from mindspore.parallel._utils import _reset_op_id from mindspore.train import Model from mindspore.context import ParallelMode +from mindspore.communication.management import GlobalComm, init from tests.dataset_mock import MindData +context.set_context(device_target="Ascend") +GlobalComm.CHECK_ENVS = False +init("hccl") +GlobalComm.CHECK_ENVS = True + +_x1 = Tensor(np.ones([64, 3, 224, 224]), dtype=ms.float32) + class Dataset(MindData): def __init__(self, predict, label, length=3): @@ -109,5 +119,202 @@ def test_all_to_all(): context.set_context(save_graphs=False) +def test_all_to_all_success(): + """ + Feature: AlltoAll + Description: on 8p, a 4d tensor split at dim 2 and concat at dim 3 + Expectation: success + """ + context.set_auto_parallel_context(device_num=8, global_rank=0) + + class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.alltoallv = _AlltoAll(split_count=8, split_dim=2, concat_dim=3) + + def construct(self, x1): + out = self.alltoallv(x1) + return out + + net = Net() + _executor.compile(net, _x1) + + +def test_all_to_all_invalid_split_count_value_failed(): + """ + Feature: AlltoAll + Description: split_count should be equal to rank size, but not + Expectation: throw ValueError + """ + context.set_auto_parallel_context(device_num=8, global_rank=0) + + class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.alltoallv = _AlltoAll(split_count=7, split_dim=2, concat_dim=3) + + def construct(self, x1): + out = self.alltoallv(x1) + return out + + with pytest.raises(ValueError): + net = Net() + _executor.compile(net, _x1) + + +def test_all_to_all_invalid_split_count_type_failed(): + """ + Feature: AlltoAll + Description: split_count should be int, but a list is given + Expectation: throw TypeError + """ + context.set_auto_parallel_context(device_num=8, global_rank=0) + + class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.alltoallv = _AlltoAll(split_count=[8], split_dim=2, concat_dim=3) + + def construct(self, x1): + out = self.alltoallv(x1) + return out + + with pytest.raises(TypeError): + net = Net() + _executor.compile(net, _x1) + + +def test_all_to_all_invalid_split_dim_value_failed(): + """ + Feature: AlltoAll + Description: split_dim over input shape + Expectation: throw IndexError + """ + context.set_auto_parallel_context(device_num=8, global_rank=0) + + class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.alltoallv = _AlltoAll(split_count=8, split_dim=4, concat_dim=3) + + def construct(self, x1): + out = self.alltoallv(x1) + return out + + with pytest.raises(IndexError): + net = Net() + _executor.compile(net, _x1) + + +def test_all_to_all_invalid_split_dim_type_failed(): + """ + Feature: AlltoAll + Description: split_dim should be int, but a tuple is given + Expectation: throw TypeError + """ + context.set_auto_parallel_context(device_num=8, global_rank=0) + + class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.alltoallv = _AlltoAll(split_count=8, split_dim=(3,), concat_dim=3) + + def construct(self, x1): + out = self.alltoallv(x1) + return out + + with pytest.raises(TypeError): + net = Net() + _executor.compile(net, _x1) + + +def test_all_to_all_invalid_concat_dim_value_failed(): + """ + Feature: AlltoAll + Description: concat_dim over input shape + Expectation: throw IndexError + """ + context.set_auto_parallel_context(device_num=8, global_rank=0) + + class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.alltoallv = _AlltoAll(split_count=8, split_dim=3, concat_dim=4) + + def construct(self, x1): + out = self.alltoallv(x1) + return out + + with pytest.raises(IndexError): + net = Net() + _executor.compile(net, _x1) + + +def test_all_to_all_invalid_concat_dim_type_failed(): + """ + Feature: AlltoAll + Description: concat_dim should be int, but a tuple is given + Expectation: throw TypeError + """ + context.set_auto_parallel_context(device_num=8, global_rank=0) + + class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.alltoallv = _AlltoAll(split_count=8, split_dim=3, concat_dim=([3],)) + + def construct(self, x1): + out = self.alltoallv(x1) + return out + + with pytest.raises(TypeError): + net = Net() + _executor.compile(net, _x1) + + +def test_all_to_all_invalid_split_count_cannot_be_divisible_failed(): + """ + Feature: AlltoAll + Description: shape at split_dim should be divisible by split_count, but not + Expectation: throw ValueError + """ + context.set_auto_parallel_context(device_num=3, global_rank=0) + + class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.alltoallv = _AlltoAll(split_count=3, split_dim=3, concat_dim=3) + + def construct(self, x1): + out = self.alltoallv(x1) + return out + + with pytest.raises(ValueError): + net = Net() + _executor.compile(net, _x1) + + +def test_all_to_all_invalid_group_type_failed(): + """ + Feature: AlltoAll + Description: group should be str, but a tuple is given + Expectation: throw TypeError + """ + context.set_auto_parallel_context(device_num=8, global_rank=0) + + class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.alltoallv = _AlltoAll(split_count=8, split_dim=3, concat_dim=3, group=3) + + def construct(self, x1): + out = self.alltoallv(x1) + return out + + with pytest.raises(TypeError): + net = Net() + _executor.compile(net, _x1) + + if __name__ == '__main__': test_all_to_all() diff --git a/tests/ut/python/parallel/test_neighborexchange.py b/tests/ut/python/parallel/test_neighborexchange.py index 787dd86704..f1d0003f51 100644 --- a/tests/ut/python/parallel/test_neighborexchange.py +++ b/tests/ut/python/parallel/test_neighborexchange.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ +import pytest import numpy as np import mindspore as ms import mindspore.context as context @@ -22,39 +23,6 @@ from mindspore.nn import TrainOneStepCell, Momentum from mindspore.ops import operations as P from mindspore.ops.operations._inner_ops import NeighborExchange - -class MatMulNet(nn.Cell): - def __init__(self, weight1): - super(MatMulNet, self).__init__() - self.matmul = P.MatMul() - self.mul = P.Mul() - self.alltoallv = NeighborExchange(send_rank_ids=[0], recv_rank_ids=[1, 2], recv_shapes=([32, 32], [32, 64]), - send_shapes=([32, 32], [32, 16]), recv_type=ms.float32) - self.weight1 = Parameter(weight1, "w1") - - def construct(self, x1, x2): - out = self.matmul(x1, x2) - out = self.mul(out, self.weight1) - out = self.alltoallv((out, x1)) - return out[0] - - -class MatMulNet2(nn.Cell): - def __init__(self, weight1): - super(MatMulNet2, self).__init__() - self.matmul = P.MatMul() - self.mul = P.Mul() - self.alltoallv = NeighborExchange(send_rank_ids=[0], recv_rank_ids=[1, 2], recv_shapes=([32, 32], [32, 64]), - send_shapes=([32, 32],), recv_type=ms.float32) - self.weight1 = Parameter(weight1, "w1") - - def construct(self, x1, x2): - out = self.matmul(x1, x2) - out = self.mul(out, self.weight1) - out = self.alltoallv((out,)) - return out[0] - - _w1 = Tensor(np.ones([32, 32]), dtype=ms.float32) _x1 = Tensor(np.ones([32, 16]), dtype=ms.float32) _x2 = Tensor(np.ones([16, 32]), dtype=ms.float32) @@ -68,13 +36,361 @@ def compile_net(net): _executor.compile(train_net, _x1, _x2) -def test_NeighborExchange_two_inputs(): +def test_NeighborExchange_two_inputs_success(): + """ + Feature: NeighborExchange + Description: two inputs and two outputs, with valid arguments + Expectation: success + """ context.set_auto_parallel_context(device_num=8, global_rank=0) + + class MatMulNet(nn.Cell): + def __init__(self, weight1): + super(MatMulNet, self).__init__() + self.matmul = P.MatMul() + self.mul = P.Mul() + self.alltoallv = NeighborExchange(send_rank_ids=[0, 1], recv_rank_ids=[1, 2], + recv_shapes=([32, 32], [32, 64]), + send_shapes=([32, 32], [32, 16]), recv_type=ms.float32) + self.weight1 = Parameter(weight1, "w1") + + def construct(self, x1, x2): + out = self.matmul(x1, x2) + out = self.mul(out, self.weight1) + out = self.alltoallv((out, x1)) + return out[0] + net = MatMulNet(_w1) compile_net(net) -def test_NeighborExchange_single_input(): +def test_NeighborExchange_single_input_success(): + """ + Feature: NeighborExchange + Description: one inputs and two outputs, with valid arguments + Expectation: success + """ context.set_auto_parallel_context(device_num=8, global_rank=0) + + class MatMulNet2(nn.Cell): + def __init__(self, weight1): + super(MatMulNet2, self).__init__() + self.matmul = P.MatMul() + self.mul = P.Mul() + self.alltoallv = NeighborExchange(send_rank_ids=[0], recv_rank_ids=[1, 2], recv_shapes=([32, 32], [32, 64]), + send_shapes=([32, 32],), recv_type=ms.float32) + self.weight1 = Parameter(weight1, "w1") + + def construct(self, x1, x2): + out = self.matmul(x1, x2) + out = self.mul(out, self.weight1) + out = self.alltoallv((out,)) + return out[0] + net = MatMulNet2(_w1) compile_net(net) + + +def test_NeighborExchage_empty_send_empty_recv_success(): + """ + Feature: NeighborExchange + Description: empty inputs and empty outputs, with valid arguments + Expectation: success + """ + context.set_auto_parallel_context(device_num=8, global_rank=0) + + class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.alltoallv = NeighborExchange(send_rank_ids=[], recv_rank_ids=[], + recv_shapes=(), + send_shapes=(), recv_type=ms.float32, group=("str",)) + + def construct(self, x1): + self.alltoallv() + return x1 + + net = Net() + with pytest.raises(TypeError): + _executor.compile(net, _x1) + + +def test_NeighborExchage_recv_shape_num_diff_with_recv_rank_size_failed(): + """ + Feature: NeighborExchange + Description: send_rank_ids and send_shapes are set as 1 input, but gives 2 + Expectation: throw ValueError + """ + context.set_auto_parallel_context(device_num=8, global_rank=0) + + class Net(nn.Cell): + def __init__(self, weight1): + super(Net, self).__init__() + self.matmul = P.MatMul() + self.mul = P.Mul() + self.alltoallv = NeighborExchange(send_rank_ids=[0], recv_rank_ids=[1, 2], recv_shapes=([32, 32],), + send_shapes=([32, 32],), recv_type=ms.float32) + self.weight1 = Parameter(weight1, "w1") + + def construct(self, x1, x2): + out = self.matmul(x1, x2) + out = self.mul(out, self.weight1) + out = self.alltoallv((out,)) + return out[0] + + net = Net(_w1) + with pytest.raises(ValueError): + compile_net(net) + + +def test_NeighborExchage_send_shape_num_diff_with_send_rank_size_failed(): + """ + Feature: NeighborExchange + Description: send_rank_ids is set as 2 inputs, but send_shapes are set as 1 input + Expectation: throw ValueError + """ + context.set_auto_parallel_context(device_num=8, global_rank=0) + + class Net(nn.Cell): + def __init__(self, weight1): + super(Net, self).__init__() + self.matmul = P.MatMul() + self.mul = P.Mul() + self.alltoallv = NeighborExchange(send_rank_ids=[0, 1], recv_rank_ids=[1, 2], + recv_shapes=([32, 32], [32, 32]), + send_shapes=([32, 32],), recv_type=ms.float32) + self.weight1 = Parameter(weight1, "w1") + + def construct(self, x1, x2): + out = self.matmul(x1, x2) + out = self.mul(out, self.weight1) + out = self.alltoallv((out,)) + return out[0] + + net = Net(_w1) + with pytest.raises(ValueError): + compile_net(net) + + +def test_NeighborExchage_send_shape_num_diff_with_input_num_failed(): + """ + Feature: NeighborExchange + Description: send_rank_ids and send_shapes are set as 2 inputs, but has only 1 input + Expectation: throw Exception + """ + context.set_auto_parallel_context(device_num=8, global_rank=0) + + class Net(nn.Cell): + def __init__(self, weight1): + super(Net, self).__init__() + self.matmul = P.MatMul() + self.mul = P.Mul() + self.alltoallv = NeighborExchange(send_rank_ids=[0, 1], recv_rank_ids=[1, 2], + recv_shapes=([32, 32], [32, 32]), + send_shapes=([32, 32], [32, 32]), recv_type=ms.float32) + self.weight1 = Parameter(weight1, "w1") + + def construct(self, x1, x2): + out = self.matmul(x1, x2) + out = self.mul(out, self.weight1) + out = self.alltoallv((out,)) + return out[0] + + net = Net(_w1) + with pytest.raises(Exception): + compile_net(net) + + +def test_NeighborExchage_send_shape_diff_with_input_shape_failed(): + """ + Feature: NeighborExchange + Description: send_shapes is set as [16, 16], but input is [32, 32] + Expectation: throw Exception + """ + context.set_auto_parallel_context(device_num=8, global_rank=0) + + class Net(nn.Cell): + def __init__(self, weight1): + super(Net, self).__init__() + self.matmul = P.MatMul() + self.mul = P.Mul() + self.alltoallv = NeighborExchange(send_rank_ids=[0], recv_rank_ids=[1, 2], recv_shapes=([32, 32], [32, 64]), + send_shapes=([16, 16],), recv_type=ms.float32) + self.weight1 = Parameter(weight1, "w1") + + def construct(self, x1, x2): + out = self.matmul(x1, x2) + out = self.mul(out, self.weight1) + out = self.alltoallv((out,)) + return out[0] + + net = Net(_w1) + with pytest.raises(Exception): + compile_net(net) + + +def test_NeighborExchage_attr_check_send_rank_ids_is_tuple_failed(): + """ + Feature: NeighborExchange + Description: send_rank_ids should be list, but a tuple is given + Expectation: throw TypeError + """ + context.set_auto_parallel_context(device_num=8, global_rank=0) + + class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.alltoallv = NeighborExchange(send_rank_ids=(0), recv_rank_ids=[1, 2], recv_shapes=([32, 32], [32, 64]), + send_shapes=([32, 16],), recv_type=ms.float32) + + def construct(self, x1): + out = self.alltoallv((x1,)) + return out[0] + + net = Net() + with pytest.raises(TypeError): + _executor.compile(net, _x1) + + +def test_NeighborExchage_attr_check_send_rank_ids_is_float_failed(): + """ + Feature: NeighborExchange + Description: send_rank_ids should be int, but a float is given + Expectation: throw TypeError + """ + context.set_auto_parallel_context(device_num=8, global_rank=0) + + class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.alltoallv = NeighborExchange(send_rank_ids=[1.0], recv_rank_ids=[1, 2], + recv_shapes=([32, 32], [32, 64]), + send_shapes=([32, 16],), recv_type=ms.float32) + + def construct(self, x1): + out = self.alltoallv((x1,)) + return out[0] + + net = Net() + with pytest.raises(TypeError): + _executor.compile(net, _x1) + + +def test_NeighborExchage_attr_check_recv_rank_ids_is_tuple_failed(): + """ + Feature: NeighborExchange + Description: recv_rank_ids should be list, but a tuple is given + Expectation: throw TypeError + """ + context.set_auto_parallel_context(device_num=8, global_rank=0) + + class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.alltoallv = NeighborExchange(send_rank_ids=[0], recv_rank_ids=([1, 2],), + recv_shapes=([32, 32], [32, 64]), + send_shapes=([32, 16],), recv_type=ms.float32) + + def construct(self, x1): + out = self.alltoallv((x1,)) + return out[0] + + net = Net() + with pytest.raises(TypeError): + _executor.compile(net, _x1) + + +def test_NeighborExchage_attr_check_recv_rank_ids_is_float_failed(): + """ + Feature: NeighborExchange + Description: recv_rank_ids should be int, but a float is given + Expectation: throw TypeError + """ + context.set_auto_parallel_context(device_num=8, global_rank=0) + + class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.alltoallv = NeighborExchange(send_rank_ids=[1], recv_rank_ids=[1, 2.0], + recv_shapes=([32, 32], [32, 64]), + send_shapes=([32, 16],), recv_type=ms.float32) + + def construct(self, x1): + out = self.alltoallv((x1,)) + return out[0] + + net = Net() + with pytest.raises(TypeError): + _executor.compile(net, _x1) + + +def test_NeighborExchage_attr_check_send_shape_not_tuple_failed(): + """ + Feature: NeighborExchange + Description: send_shapes should be tuple(list), but a list is given + Expectation: throw TypeError + """ + context.set_auto_parallel_context(device_num=8, global_rank=0) + + class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.alltoallv = NeighborExchange(send_rank_ids=[1], recv_rank_ids=[1, 2], + recv_shapes=([32, 32], [32, 64]), + send_shapes=([32, 16]), recv_type=ms.float32) + + def construct(self, x1): + out = self.alltoallv((x1,)) + return out[0] + + net = Net() + with pytest.raises(TypeError): + _executor.compile(net, _x1) + + +def test_NeighborExchage_attr_check_recv_type_numpy_failed(): + """ + Feature: NeighborExchange + Description: recv_type should be mindspore type, but a numpy type is given + Expectation: throw TypeError + """ + context.set_auto_parallel_context(device_num=8, global_rank=0) + + class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.alltoallv = NeighborExchange(send_rank_ids=[1], recv_rank_ids=[1, 2], + recv_shapes=([32, 32], [32, 64]), + send_shapes=([32, 16],), recv_type=np.float32) + + def construct(self, x1): + out = self.alltoallv((x1,)) + return out[0] + + net = Net() + with pytest.raises(TypeError): + _executor.compile(net, _x1) + + +def test_NeighborExchage_attr_invalid_grpup_failed(): + """ + Feature: NeighborExchange + Description: group should be str, but a tuple is given + Expectation: throw TypeError + """ + context.set_auto_parallel_context(device_num=8, global_rank=0) + + class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.alltoallv = NeighborExchange(send_rank_ids=[1], recv_rank_ids=[1, 2], + recv_shapes=([32, 32], [32, 64]), + send_shapes=([32, 16],), recv_type=ms.float32, group=("str",)) + + def construct(self, x1): + out = self.alltoallv((x1,)) + return out[0] + + net = Net() + with pytest.raises(TypeError): + _executor.compile(net, _x1)