Add mode check

Add parallel mode check

mindspore/ccsrc/frontend/parallel/context.h

@@ -125,6 +125,9 @@ class ParallelContext {
   }
   bool enable_parallel_optimizer() const { return enable_parallel_optimizer_; }
 
+  void set_hccl_test_available(bool hccl_test_available) { hccl_test_available_ = hccl_test_available; }
+  bool hccl_test_available() const { return hccl_test_available_; }
+
   bool set_communi_parallel_mode(const std::string &communi_parallel_mode);
   std::string communi_parallel_mode() const { return communi_parallel_mode_; }
   void set_sharding_propagation(const bool);
@@ -173,6 +176,8 @@ class ParallelContext {
   // Enable AllToAll or not. If false, use AllGather and Split.
   bool enable_all2all_;
   std::vector<std::vector<int64_t>> dataset_strategy_;
+  bool dataset_repeat_dim_right_ = false;
+  bool hccl_test_available_ = false;
 };
 
 }  // namespace parallel

mindspore/ccsrc/frontend/parallel/step_parallel.cc

@@ -2901,15 +2901,15 @@ CommInfo GetCommInfo() {
     device_num = UintToInt(world_rank_size);
     MS_LOG(INFO) << "Get device num from communication model, the device num is  " << device_num;
   }
-#if defined(ENABLE_GPU)
-  if (ParallelContext::GetInstance()->device_num_is_set() && backend == kGPUDevice) {
-    if (world_rank_size != device_num) {
-      MS_LOG(EXCEPTION) << "The device_num " << device_num
-                        << " set in the context is not consist with the word group size " << world_rank_size;
-    }
+#if ENABLE_D || ENABLE_GPU
+  if (ParallelContext::GetInstance()->device_num_is_set() && world_rank_size != device_num &&
+      !ParallelContext::GetInstance()->hccl_test_available()) {
+    // hccl_test_available is used when we compile graphs in real ascend card environment, but with hccl_test.
+    MS_LOG(EXCEPTION) << "The device_num " << device_num << " set in the context is not consist with "
+                      << world_rank_size << " devices you have "
+                      << ". Please check your rank_table file(for Ascend) or host file(for GPU).";
   }
 #endif
-
   uint32_t rank_id = 0;
   if (!ParallelContext::GetInstance()->global_rank_is_set()) {
     if (!CommManager::GetInstance().GetRankID(world_group, &rank_id)) {

mindspore/ccsrc/pipeline/jit/init.cc

@@ -135,6 +135,7 @@ PYBIND11_MODULE(_c_expression, m) {
   (void)py::class_<ParallelContext, std::shared_ptr<ParallelContext>>(m, "AutoParallelContext")
     .def_static("get_instance", &ParallelContext::GetInstance, "Get auto parallel context instance.")
     .def("get_device_num", &ParallelContext::device_num, "Get device num.")
+    .def("set_hccl_test_avaible", &ParallelContext::set_hccl_test_available, "Set hccl test available.")
     .def("set_device_num", &ParallelContext::set_device_num, "Set device num.")
     .def("get_device_num_is_set", &ParallelContext::device_num_is_set, "Get device num is set.")
     .def("get_global_rank", &ParallelContext::global_rank, "Get global rank.")

mindspore/common/parameter.py

@@ -321,7 +321,8 @@ class Parameter(Tensor_):
     @comm_fusion.setter
     def comm_fusion(self, comm_fusion_):
         if context.get_context("mode") == context.PYNATIVE_MODE and "auto_parallel" in _get_parallel_mode():
-            raise RuntimeError("`comm_fusion` does not support PYNATIVE_MODE")
+            raise RuntimeError(
+                "`comm_fusion` does not support PYNATIVE_MODE in AUTO_PARALLEL and SEMI_AUTO_PARALLEL mode.")
         Validator.check_non_negative_int(comm_fusion_)
         self.param_info.comm_fusion = comm_fusion_
 

mindspore/communication/_comm_helper.py

@@ -19,6 +19,7 @@ from mindspore import log as logger
 from ._hccl_management import load_lib as hccl_load_lib
 
 _HCCL_AVAILABLE = False
+_HCCL_TEST_AVAILABLE = False
 _NCCL_AVAILABLE = False
 _MPI_AVAILABLE = False
 try:
@@ -45,6 +46,7 @@ else:
     try:
         import hccl_test.manage.api as hccl
         _HCCL_AVAILABLE = True
+        _HCCL_TEST_AVAILABLE = True
     except ImportError:
         _HCCL_AVAILABLE = False
 

mindspore/context.py

@@ -180,6 +180,10 @@ class _Context:
         if mode == PYNATIVE_MODE:
             if self.enable_debug_runtime:
                 self.set_backend_policy("vm")
+            parallel_mode = _get_auto_parallel_context("parallel_mode")
+            if parallel_mode not in (ParallelMode.DATA_PARALLEL, ParallelMode.STAND_ALONE):
+                raise ValueError(f"Pynative Only support STAND_ALONE and DATA_PARALLEL for ParallelMode,"
+                                 f"but got {parallel_mode.upper()}.")
             self._context_switches.push(True, None)
         elif mode == GRAPH_MODE:
             if self.enable_debug_runtime:
@@ -356,9 +360,8 @@ def set_auto_parallel_context(**kwargs):
         Attribute name is required for setting attributes.
         If a program has tasks on different parallel modes, before setting a new parallel mode for the
         next task, interface mindspore.context.reset_auto_parallel_context() should be called to reset
-        the configuration.
-        Setting or changing parallel modes must be called before creating any Initializer, otherwise,
-        it may have RuntimeError when compiling the network.
+        the configuration. Setting or changing parallel modes must be called before creating any Initializer,
+        otherwise, it may have RuntimeError when compiling the network.
 
     Some configurations are parallel mode specific, see the below table for details:
 
@@ -383,7 +386,8 @@ def set_auto_parallel_context(**kwargs):
         gradient_fp32_sync (bool): Run allreduce of gradients in fp32. "stand_alone", "data_parallel"
                      and "hybrid_parallel" do not support gradient_fp32_sync. Default: True.
         parallel_mode (str): There are five kinds of parallel modes, "stand_alone", "data_parallel",
-                     "hybrid_parallel", "semi_auto_parallel" and "auto_parallel". Default: "stand_alone".
+                     "hybrid_parallel", "semi_auto_parallel" and "auto_parallel". Note the pynative mode only supports
+                     the "stand_alone" and "data_parallel" mode. Default: "stand_alone".
 
                      - stand_alone: Only one processor is working.
 

mindspore/parallel/_auto_parallel_context.py

@@ -72,6 +72,8 @@ class _AutoParallelContext:
         self.check_context_handle()
         if device_num < 1 or device_num > 4096:
             raise ValueError("Device num must be in [1, 4096], but got {}".format(device_num))
+        from mindspore.communication._comm_helper import _HCCL_TEST_AVAILABLE
+        self._context_handle.set_hccl_test_avaible(_HCCL_TEST_AVAILABLE)
         self._context_handle.set_device_num(device_num)
 
     def get_device_num(self):
@@ -188,6 +190,11 @@ class _AutoParallelContext:
             ValueError: If parallel mode is not supported.
         """
         self.check_context_handle()
+        run_mode = context.get_context("mode")
+        if run_mode == context.PYNATIVE_MODE and parallel_mode not in (
+                context.ParallelMode.DATA_PARALLEL, context.ParallelMode.STAND_ALONE):
+            raise ValueError(f"Pynative Only support STAND_ALONE and DATA_PARALLEL for ParallelMode, "
+                             f"but got {parallel_mode.upper()}.")
         ret = self._context_handle.set_parallel_mode(parallel_mode)
         if ret is False:
             raise ValueError("Parallel mode does not support {}".format(parallel_mode))