From 1bad79cb575bbe62841412351442275c6aeaf38a Mon Sep 17 00:00:00 2001 From: huangxinjing Date: Mon, 1 Nov 2021 09:36:42 +0800 Subject: [PATCH] Add mode check Add parallel mode check --- mindspore/ccsrc/frontend/parallel/context.h | 5 +++++ mindspore/ccsrc/frontend/parallel/step_parallel.cc | 14 +++++++------- mindspore/ccsrc/pipeline/jit/init.cc | 1 + mindspore/common/parameter.py | 3 ++- mindspore/communication/_comm_helper.py | 2 ++ mindspore/context.py | 12 ++++++++---- mindspore/parallel/_auto_parallel_context.py | 7 +++++++ 7 files changed, 32 insertions(+), 12 deletions(-) diff --git a/mindspore/ccsrc/frontend/parallel/context.h b/mindspore/ccsrc/frontend/parallel/context.h index 00b1ef237c4..9562498e3db 100644 --- a/mindspore/ccsrc/frontend/parallel/context.h +++ b/mindspore/ccsrc/frontend/parallel/context.h @@ -125,6 +125,9 @@ class ParallelContext { } bool enable_parallel_optimizer() const { return enable_parallel_optimizer_; } + void set_hccl_test_available(bool hccl_test_available) { hccl_test_available_ = hccl_test_available; } + bool hccl_test_available() const { return hccl_test_available_; } + bool set_communi_parallel_mode(const std::string &communi_parallel_mode); std::string communi_parallel_mode() const { return communi_parallel_mode_; } void set_sharding_propagation(const bool); @@ -173,6 +176,8 @@ class ParallelContext { // Enable AllToAll or not. If false, use AllGather and Split. bool enable_all2all_; std::vector> dataset_strategy_; + bool dataset_repeat_dim_right_ = false; + bool hccl_test_available_ = false; }; } // namespace parallel diff --git a/mindspore/ccsrc/frontend/parallel/step_parallel.cc b/mindspore/ccsrc/frontend/parallel/step_parallel.cc index c72f83df70e..10c8c88ea8c 100644 --- a/mindspore/ccsrc/frontend/parallel/step_parallel.cc +++ b/mindspore/ccsrc/frontend/parallel/step_parallel.cc @@ -2901,15 +2901,15 @@ CommInfo GetCommInfo() { device_num = UintToInt(world_rank_size); MS_LOG(INFO) << "Get device num from communication model, the device num is " << device_num; } -#if defined(ENABLE_GPU) - if (ParallelContext::GetInstance()->device_num_is_set() && backend == kGPUDevice) { - if (world_rank_size != device_num) { - MS_LOG(EXCEPTION) << "The device_num " << device_num - << " set in the context is not consist with the word group size " << world_rank_size; - } +#if ENABLE_D || ENABLE_GPU + if (ParallelContext::GetInstance()->device_num_is_set() && world_rank_size != device_num && + !ParallelContext::GetInstance()->hccl_test_available()) { + // hccl_test_available is used when we compile graphs in real ascend card environment, but with hccl_test. + MS_LOG(EXCEPTION) << "The device_num " << device_num << " set in the context is not consist with " + << world_rank_size << " devices you have " + << ". Please check your rank_table file(for Ascend) or host file(for GPU)."; } #endif - uint32_t rank_id = 0; if (!ParallelContext::GetInstance()->global_rank_is_set()) { if (!CommManager::GetInstance().GetRankID(world_group, &rank_id)) { diff --git a/mindspore/ccsrc/pipeline/jit/init.cc b/mindspore/ccsrc/pipeline/jit/init.cc index f41c1768085..b26cd17ad2f 100644 --- a/mindspore/ccsrc/pipeline/jit/init.cc +++ b/mindspore/ccsrc/pipeline/jit/init.cc @@ -135,6 +135,7 @@ PYBIND11_MODULE(_c_expression, m) { (void)py::class_>(m, "AutoParallelContext") .def_static("get_instance", &ParallelContext::GetInstance, "Get auto parallel context instance.") .def("get_device_num", &ParallelContext::device_num, "Get device num.") + .def("set_hccl_test_avaible", &ParallelContext::set_hccl_test_available, "Set hccl test available.") .def("set_device_num", &ParallelContext::set_device_num, "Set device num.") .def("get_device_num_is_set", &ParallelContext::device_num_is_set, "Get device num is set.") .def("get_global_rank", &ParallelContext::global_rank, "Get global rank.") diff --git a/mindspore/common/parameter.py b/mindspore/common/parameter.py index 6b741a4d7a8..e2962313171 100644 --- a/mindspore/common/parameter.py +++ b/mindspore/common/parameter.py @@ -321,7 +321,8 @@ class Parameter(Tensor_): @comm_fusion.setter def comm_fusion(self, comm_fusion_): if context.get_context("mode") == context.PYNATIVE_MODE and "auto_parallel" in _get_parallel_mode(): - raise RuntimeError("`comm_fusion` does not support PYNATIVE_MODE") + raise RuntimeError( + "`comm_fusion` does not support PYNATIVE_MODE in AUTO_PARALLEL and SEMI_AUTO_PARALLEL mode.") Validator.check_non_negative_int(comm_fusion_) self.param_info.comm_fusion = comm_fusion_ diff --git a/mindspore/communication/_comm_helper.py b/mindspore/communication/_comm_helper.py index 9e5d8513319..b9dc8b452c8 100644 --- a/mindspore/communication/_comm_helper.py +++ b/mindspore/communication/_comm_helper.py @@ -19,6 +19,7 @@ from mindspore import log as logger from ._hccl_management import load_lib as hccl_load_lib _HCCL_AVAILABLE = False +_HCCL_TEST_AVAILABLE = False _NCCL_AVAILABLE = False _MPI_AVAILABLE = False try: @@ -45,6 +46,7 @@ else: try: import hccl_test.manage.api as hccl _HCCL_AVAILABLE = True + _HCCL_TEST_AVAILABLE = True except ImportError: _HCCL_AVAILABLE = False diff --git a/mindspore/context.py b/mindspore/context.py index 0575fa282fc..5a29a94cbb1 100644 --- a/mindspore/context.py +++ b/mindspore/context.py @@ -180,6 +180,10 @@ class _Context: if mode == PYNATIVE_MODE: if self.enable_debug_runtime: self.set_backend_policy("vm") + parallel_mode = _get_auto_parallel_context("parallel_mode") + if parallel_mode not in (ParallelMode.DATA_PARALLEL, ParallelMode.STAND_ALONE): + raise ValueError(f"Pynative Only support STAND_ALONE and DATA_PARALLEL for ParallelMode," + f"but got {parallel_mode.upper()}.") self._context_switches.push(True, None) elif mode == GRAPH_MODE: if self.enable_debug_runtime: @@ -356,9 +360,8 @@ def set_auto_parallel_context(**kwargs): Attribute name is required for setting attributes. If a program has tasks on different parallel modes, before setting a new parallel mode for the next task, interface mindspore.context.reset_auto_parallel_context() should be called to reset - the configuration. - Setting or changing parallel modes must be called before creating any Initializer, otherwise, - it may have RuntimeError when compiling the network. + the configuration. Setting or changing parallel modes must be called before creating any Initializer, + otherwise, it may have RuntimeError when compiling the network. Some configurations are parallel mode specific, see the below table for details: @@ -383,7 +386,8 @@ def set_auto_parallel_context(**kwargs): gradient_fp32_sync (bool): Run allreduce of gradients in fp32. "stand_alone", "data_parallel" and "hybrid_parallel" do not support gradient_fp32_sync. Default: True. parallel_mode (str): There are five kinds of parallel modes, "stand_alone", "data_parallel", - "hybrid_parallel", "semi_auto_parallel" and "auto_parallel". Default: "stand_alone". + "hybrid_parallel", "semi_auto_parallel" and "auto_parallel". Note the pynative mode only supports + the "stand_alone" and "data_parallel" mode. Default: "stand_alone". - stand_alone: Only one processor is working. diff --git a/mindspore/parallel/_auto_parallel_context.py b/mindspore/parallel/_auto_parallel_context.py index d15b39b3018..da9dd6a09a4 100644 --- a/mindspore/parallel/_auto_parallel_context.py +++ b/mindspore/parallel/_auto_parallel_context.py @@ -72,6 +72,8 @@ class _AutoParallelContext: self.check_context_handle() if device_num < 1 or device_num > 4096: raise ValueError("Device num must be in [1, 4096], but got {}".format(device_num)) + from mindspore.communication._comm_helper import _HCCL_TEST_AVAILABLE + self._context_handle.set_hccl_test_avaible(_HCCL_TEST_AVAILABLE) self._context_handle.set_device_num(device_num) def get_device_num(self): @@ -188,6 +190,11 @@ class _AutoParallelContext: ValueError: If parallel mode is not supported. """ self.check_context_handle() + run_mode = context.get_context("mode") + if run_mode == context.PYNATIVE_MODE and parallel_mode not in ( + context.ParallelMode.DATA_PARALLEL, context.ParallelMode.STAND_ALONE): + raise ValueError(f"Pynative Only support STAND_ALONE and DATA_PARALLEL for ParallelMode, " + f"but got {parallel_mode.upper()}.") ret = self._context_handle.set_parallel_mode(parallel_mode) if ret is False: raise ValueError("Parallel mode does not support {}".format(parallel_mode))